[security] Fix ISEC-V2026-03/04/05 authorization findings and move to Python 3.11 only

.github/workflows/commitflow-py3.yml

@@ -11,10 +11,6 @@ jobs:
   hue-build-and-tests:
     runs-on: ubuntu-latest
 
-    strategy:
-      matrix:
-        python-version: ['3.9', '3.10', '3.11']
-
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -25,10 +21,10 @@ jobs:
         run: |
           ./tools/ci/check_for_commit_message.sh
 
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python 3.11
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: '3.11'
 
       - name: Cache pip
         uses: actions/cache@v4
@@ -51,29 +47,20 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y build-essential asciidoc libkrb5-dev libldap2-dev libsasl2-dev libxslt-dev libsasl2-modules-gssapi-mit libsnappy-dev libgdbm-dev
 
-          export PYTHON_VER=python${{ matrix.python-version }}
+          export PYTHON_VER=python3.11
           export SKIP_PYTHONDEV_CHECK=true
           export ROOT=$PWD
           make apps test_prep
 
       - name: Run unit tests
         run: |
-          if [[ "${{ matrix.python-version }}" == "3.11" ]]; then
-            ./build/env/bin/pytest
-          else
-            ./build/venvs/python${{ matrix.python-version }}/bin/pytest
-          fi
+          ./build/env/bin/pytest
 
       - name: Run python lint check
         run: |
-          if [[ "${{ matrix.python-version }}" == "3.11" ]]; then
-            PYTHONWARNINGS=always ./build/env/bin/hue runruff check
-          else
-            PYTHONWARNINGS=always ./build/venvs/python${{ matrix.python-version }}/bin/hue runruff check
-          fi
+          PYTHONWARNINGS=always ./build/env/bin/hue runruff check
 
       - name: Upload reports
-        if: matrix.python-version == '3.11'
         uses: actions/upload-artifact@v4
         with:
           name: hue-reports

Makefile.vars

@@ -37,10 +37,9 @@ ifneq ($(wildcard $(ROOT)/Makefile.buildvars),)
 endif
 
 
-VER_ERR_MSG = "Variable PYTHON_VER is $(PYTHON_VER) but it only supports >= python3.9. If not set, defaults to python3.11."
+VER_ERR_MSG = "Variable PYTHON_VER is $(PYTHON_VER) but it only supports python3.11. If not set, defaults to python3.11."
 
-# Major rewrite to support multi-python environments.
-PYTHON_VERSIONS ?= python3.9 python3.10 python3.11
+PYTHON_VERSIONS ?= python3.11
 PYTHON_VER ?= python3.11
 
 ifeq ($(filter $(PYTHON_VER), $(PYTHON_VERSIONS)),)
@@ -52,28 +51,18 @@ endif
 ##############################
 BLD_DIR := $(ROOT)/build
 
-# Default case for Python 3.11
 ifndef BLD_DIR_ENV
-  ifeq ($(PYTHON_VER),python3.11)
-    BLD_DIR_ENV := $(BLD_DIR)/env
-  else
-    BLD_DIR_ENV := $(BLD_DIR)/venvs/$(PYTHON_VER)
-  endif
+  BLD_DIR_ENV := $(BLD_DIR)/env
 endif
 
 ifdef INSTALL_DIR
   INST_DIR := $(INSTALL_DIR)/build
-
-  ifeq ($(BLD_DIR_ENV),"$(BLD_DIR)/env")
-    INST_DIR_ENV := $(INST_DIR)/env
-  else
-    INST_DIR_ENV := $(INST_DIR)/venvs/$(PYTHON_VER)
-  endif
+  INST_DIR_ENV := $(INST_DIR)/env
 endif
 
 BLD_DIR_BIN := $(BLD_DIR_ENV)/bin
 
-# Adds detection and configuration for python3.9 through python3.11.
+# Adds detection and configuration for python3.11.
 # Sets up per-version BLD_DIR_ENV and INST_DIR_ENV variables.
 # Introduces version-specific requirement file generation via generate_requirements.py.
 SYS_PYTHON := $(shell IFS=:; for p in $$PATH; do if [ -x "$$p/$(PYTHON_VER)" ]; then echo "$$p/$(PYTHON_VER)"; break; fi; done)
@@ -88,7 +77,7 @@ $(info "PYTHON_VER is $(PYTHON_VER).")
 
 ifeq ($(shell echo $(PYTHON_VER) | head -c 8),python3.)
   MINOR_VER = $(shell $(SYS_PYTHON) --version | sed -E 's/Python [0-9]+\.([0-9]+).*/\1/')
-  ifeq ($(shell test $(MINOR_VER) -lt 9; echo $$?),0)
+  ifeq ($(shell test $(MINOR_VER) -lt 11; echo $$?),0)
     $(error "$(VER_ERR_MSG)")
   endif
   EXT_ENV_INSTALL = ext-env-pip-install
@@ -103,7 +92,7 @@ ifndef SKIP_PYTHONDEV_CHECK
 endif
 
 ifeq ($(SYS_PYTHON),)
-  $(error "Error: Need python version >= 3.8")
+  $(error "Error: Need python version >= 3.11")
 endif
 
 HADOOP_HOME ?= /usr/lib/hadoop
@@ -164,18 +153,8 @@ VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
 ##############################
 # Set specific versions for some libraries
 ##############################
-ifeq ($(PYTHON_VER),python3.9)
-  PIP_VERSION := 22.2.2
-  VIRTUAL_ENV_VERSION := 21.7.16
-  VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
-else ifeq ($(PYTHON_VER),python3.10)
-  PIP_VERSION := 22.2.2
-  VIRTUAL_ENV_VERSION := 21.7.16
-  VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
-else ifeq ($(PYTHON_VER),python3.11)
-  PIP_VERSION := 22.2.2
-  VIRTUAL_ENV_VERSION := 21.7.16
-  VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
-endif
+PIP_VERSION := 22.2.2
+VIRTUAL_ENV_VERSION := 21.7.16
+VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
 
 RELOCATABLE := $(shell which virtualenv-make-relocatable)

desktop/core/generate_requirements.py

@@ -15,10 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-Dynamic requirements generator for the Hue multi-Python build system.
+Dynamic requirements generator for the Hue build system (Python 3.11).
 
-Generate architecture- and Python-version-specific requirement files at
-build time, replacing a static requirements.txt. This tool supports:
+Generate architecture-specific requirement files at build time, replacing
+a static requirements.txt. This tool supports:
 
 - Local editable paths for Hue extensions.
 - Binary wheels for specific platforms (e.g., ppc64le).
@@ -116,75 +116,31 @@ def __init__(self):
       "six==1.16.0",
       "slack-sdk==3.31.0",
       "SQLAlchemy==1.3.8",
-      # sqlparse is pinned per Python version below: 0.6.0 fixes three DoS CVEs but needs Python >= 3.10.
+      # sqlparse is pinned in the architecture-specific requirements below: 0.6.0 fixes three DoS CVEs.
       "tablib==0.13.0",
       "tabulate==0.8.9",
       "thrift-sasl==0.4.3",
       "trino==0.329.0",
       "git+https://github.com/gethue/thrift.git",
     ]
 
+    python_311_arch_extras = [
+      "async-timeout==5.0.1",
+      "lxml==4.9.1",
+      "Markdown==3.8.1",
+      "numpy==1.24.4",
+      "pandas==2.0.3",
+      "pure-sasl==0.6.2",
+      "setuptools==80.9.0",
+      "sqlparse==0.6.0",
+    ]
+
     self.x86_64_requirements = {
-      "default": [
-        "lxml==4.9.1",
-        "Markdown==3.8.1",
-        "numpy==1.24.4",
-        "pandas==2.0.3",
-        "sasl==0.3.1",
-        "setuptools==70.0.0",
-        "sqlparse==0.6.0",
-      ],
-      "3.9": [
-        "decorator==5.1.1",
-        "lxml==4.9.1",
-        "Markdown==3.8.1",
-        "numpy==1.24.4",
-        "pandas==2.0.3",
-        "sasl==0.3.1",
-        "setuptools==80.9.0",
-        "sqlparse==0.5.3",  # 0.6.0 requires Python >= 3.10
-      ],
-      "3.11": [
-        "async-timeout==5.0.1",
-        "lxml==4.9.1",
-        "Markdown==3.8.1",
-        "numpy==1.24.4",
-        "pandas==2.0.3",
-        "pure-sasl==0.6.2",
-        "setuptools==80.9.0",
-        "sqlparse==0.6.0",
-      ],
+      "3.11": python_311_arch_extras,
     }
 
     self.aarch64_requirements = {
-      "default": [
-        "lxml==4.9.1",
-        "Markdown==3.8.1",
-        "numpy==1.24.4",
-        "pandas==2.0.3",
-        "setuptools==70.0.0",
-        "sqlparse==0.6.0",
-      ],
-      "3.9": [
-        "decorator==5.1.1",
-        "lxml==4.9.1",
-        "Markdown==3.8.1",
-        "numpy==1.24.4",
-        "pandas==2.0.3",
-        "sasl==0.3.1",
-        "setuptools==80.9.0",
-        "sqlparse==0.5.3",  # 0.6.0 requires Python >= 3.10
-      ],
-      "3.11": [
-        "async-timeout==5.0.1",
-        "lxml==4.9.1",
-        "Markdown==3.8.1",
-        "numpy==1.24.4",
-        "pandas==2.0.3",
-        "pure-sasl==0.6.2",
-        "setuptools==80.9.0",
-        "sqlparse==0.6.0",
-      ],
+      "3.11": python_311_arch_extras,
     }
     self.arch_requirements_map = {
       "x86_64": self.x86_64_requirements,
@@ -205,7 +161,12 @@ def generate_requirements(self):
     if self.arch not in self.arch_requirements_map:
       raise ValueError(f"Unsupported architecture: {self.arch}")
     arch_reqs = self.arch_requirements_map[self.arch]
-    self.requirements.extend(arch_reqs.get(self.python_version_string, arch_reqs["default"]))
+    if self.python_version_string not in arch_reqs:
+      raise ValueError(
+        f"Unsupported Python version {self.python_version_string} for architecture {self.arch}. "
+        f"Supported: {', '.join(sorted(arch_reqs.keys()))}"
+      )
+    self.requirements.extend(arch_reqs[self.python_version_string])
     self.requirements.extend(self.copy_local_requirements(self.python_version_string))
     with open(f"{this_dir}/requirements-{self.arch}-{self.python_version_string}.txt", "w") as f:
       f.write("\n".join(self.requirements))

desktop/core/generate_requirements_test.py

@@ -67,13 +67,13 @@ def test_init_sets_properties(self, mock_version_info, mock_machine, mock_this_d
 
     mock_machine.return_value = "x86_64"
     mock_version_info.major = 3
-    mock_version_info.minor = 9
+    mock_version_info.minor = 11
     mock_this_dir.return_value = self.temp_dir
 
     generator = RequirementsGenerator()
 
     assert generator.arch == "x86_64"
-    assert generator.python_version_string == "3.9"
+    assert generator.python_version_string == "3.11"
     assert isinstance(generator.requirements, list)
     assert isinstance(generator.local_requirements, list)
     assert "x86_64" in generator.arch_requirements_map
@@ -87,7 +87,7 @@ def test_copy_local_requirements(self, mock_copytree, mock_this_dir):
 
     # Need to make this_dir return the string value directly, not a MagicMock
     mock_this_dir.__str__.return_value = self.temp_dir
-    python_version_string = "3.9"
+    python_version_string = "3.11"
 
     with mock.patch.object(RequirementsGenerator, "__init__", return_value=None):
       generator = RequirementsGenerator()
@@ -112,11 +112,11 @@ def test_generate_requirements_x86_64(self, mock_open, mock_this_dir):
     with mock.patch.object(RequirementsGenerator, "__init__", return_value=None):
       generator = RequirementsGenerator()
       generator.arch = "x86_64"
-      generator.python_version_string = "3.9"
+      generator.python_version_string = "3.11"
       generator.requirements = ["Django==4.1.13"]
       generator.local_requirements = []
       generator.arch_requirements_map = {
-        "x86_64": {"default": ["cryptography==42.0.8"], "3.9": ["Markdown==3.8", "numpy==1.24.4", "setuptools==80.9.0"]}
+        "x86_64": {"3.11": ["Markdown==3.8", "numpy==1.24.4", "setuptools==80.9.0"]}
       }
 
       # Mock copy_local_requirements for test isolation
@@ -125,7 +125,7 @@ def test_generate_requirements_x86_64(self, mock_open, mock_this_dir):
       generator.generate_requirements()
 
       # Verify file creation with correct path
-      mock_open.assert_called_once_with(f"{self.temp_dir}/requirements-x86_64-3.9.txt", "w")
+      mock_open.assert_called_once_with(f"{self.temp_dir}/requirements-x86_64-3.11.txt", "w")
 
       # Verify correct requirements were written
       expected_requirements = "\n".join(["Django==4.1.13", "Markdown==3.8", "numpy==1.24.4", "setuptools==80.9.0"])
@@ -141,9 +141,9 @@ def test_generate_requirements_unsupported_arch(self, mock_this_dir):
     with mock.patch.object(RequirementsGenerator, "__init__", return_value=None):
       generator = RequirementsGenerator()
       generator.arch = "unsupported_arch"
-      generator.python_version_string = "3.9"
+      generator.python_version_string = "3.11"
       generator.requirements = []
-      generator.arch_requirements_map = {"x86_64": {"default": []}, "aarch64": {"default": []}}
+      generator.arch_requirements_map = {"x86_64": {"3.11": []}, "aarch64": {"3.11": []}}
 
       with pytest.raises(ValueError) as exc_info:
         generator.generate_requirements()
@@ -160,11 +160,11 @@ def test_get_file_name(self, mock_this_dir):
     with mock.patch.object(RequirementsGenerator, "__init__", return_value=None):
       generator = RequirementsGenerator()
       generator.arch = "x86_64"
-      generator.python_version_string = "3.9"
+      generator.python_version_string = "3.11"
 
       result = generator.get_file_name()
 
-      assert result == f"{self.temp_dir}/requirements-x86_64-3.9.txt"
+      assert result == f"{self.temp_dir}/requirements-x86_64-3.11.txt"
 
   @mock.patch("generate_requirements.this_dir")
   @mock.patch("builtins.open", new_callable=mock.mock_open)
@@ -177,21 +177,21 @@ def test_generate_requirements_with_local_requirements(self, mock_open, mock_thi
     with mock.patch.object(RequirementsGenerator, "__init__", return_value=None):
       generator = RequirementsGenerator()
       generator.arch = "x86_64"
-      generator.python_version_string = "3.9"
+      generator.python_version_string = "3.11"
       generator.requirements = []
       generator.local_requirements = ["boto-2.49.0", "django-axes-5.13.0"]
-      generator.arch_requirements_map = {"x86_64": {"default": [], "3.9": ["Markdown==3.8", "setuptools==80.9.0"]}}
+      generator.arch_requirements_map = {"x86_64": {"3.11": ["Markdown==3.8", "setuptools==80.9.0"]}}
 
       # Mock copy_local_requirements to return file paths
-      local_reqs = [f"file://{self.temp_dir}/3.9/boto-2.49.0", f"file://{self.temp_dir}/3.9/django-axes-5.13.0"]
+      local_reqs = [f"file://{self.temp_dir}/3.11/boto-2.49.0", f"file://{self.temp_dir}/3.11/django-axes-5.13.0"]
       generator.copy_local_requirements = mock.MagicMock(return_value=local_reqs)
 
       generator.generate_requirements()
 
-      mock_open.assert_called_once_with(f"{self.temp_dir}/requirements-x86_64-3.9.txt", "w")
+      mock_open.assert_called_once_with(f"{self.temp_dir}/requirements-x86_64-3.11.txt", "w")
 
       expected_requirements = "\n".join(
-        ["Markdown==3.8", "setuptools==80.9.0", f"file://{self.temp_dir}/3.9/boto-2.49.0", f"file://{self.temp_dir}/3.9/django-axes-5.13.0"]
+        ["Markdown==3.8", "setuptools==80.9.0", f"file://{self.temp_dir}/3.11/boto-2.49.0", f"file://{self.temp_dir}/3.11/django-axes-5.13.0"]
       )
 
       mock_open().write.assert_called_once_with(expected_requirements)