From a8172765de0d96d4789d42e55b733e9abb494345 Mon Sep 17 00:00:00 2001 From: Harsh Gupta <42064744+Harshg999@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:04:42 +0530 Subject: [PATCH 1/6] [desktop] Enforce authorization on configuration API (ISEC-V2026-03) The DefaultConfiguration REST endpoints trusted a client-supplied user_id and configuration id without checking ownership or admin role. Any authenticated user could read or change another user's effective settings, modify cluster-wide default/group configuration, or delete arbitrary configuration rows. - Add configuration/authorization.py with helpers for the global admin check, per-user target resolution and delete authorization. - Require a Hue admin for GET/POST /desktop/api/configurations/. - Resolve and authorize user_id on GET/POST /desktop/api/configurations/user/. Non-admins can only use their own id; admins can use any valid id. - Authorize before deleting on /desktop/api/configurations/delete/: default and group rows need an admin, user rows need the owner or an admin. - Point CONFIG_SAVE_API at /desktop/api/configurations/user/. The previous /configurations/save/ route does not exist. - Add TestConfigurationAuthorization regression tests covering the cross-user, non-admin and admin cases. --- desktop/core/src/desktop/configuration/api.py | 26 ++-- .../desktop/configuration/authorization.py | 95 ++++++++++++ .../core/src/desktop/configuration/tests.py | 139 ++++++++++++++++++ desktop/core/src/desktop/js/api/urls.js | 2 +- 4 files changed, 249 insertions(+), 13 deletions(-) create mode 100644 desktop/core/src/desktop/configuration/authorization.py diff --git a/desktop/core/src/desktop/configuration/api.py b/desktop/core/src/desktop/configuration/api.py index 4aadfca2b90..d46ae630c9e 100644 --- a/desktop/core/src/desktop/configuration/api.py +++ b/desktop/core/src/desktop/configuration/api.py @@ -19,12 +19,17 @@ import json import logging -from django.contrib.auth.models import Group, User +from django.contrib.auth.models import Group from django.db import transaction from django.db.models import Q from django.utils.translation import gettext as _ from django.views.decorators.http import require_POST +from desktop.configuration.authorization import ( + assert_can_delete_configuration, + require_configuration_admin, + resolve_target_user, +) from desktop.lib.django_util import JsonResponse from desktop.lib.exceptions_renderable import PopupException from desktop.lib.i18n import force_unicode @@ -60,6 +65,8 @@ def decorator(*args, **kwargs): @api_error_handler def default_configurations(request): + require_configuration_admin(request.user) + if request.method == 'GET': # get configurable apps configurations = _get_default_configurations() @@ -87,13 +94,10 @@ def app_configuration_for_user(request): app = request.GET.get('app') user_id = request.GET.get('user_id') - if not app or not user_id: + if not app: raise PopupException(_('app_configuration_for_user requires app and user_id')) - user = User.objects.get(id=user_id) - - if not user: - raise PopupException(_('Could not find user with User ID: %s') % user_id) + user = resolve_target_user(request, user_id) config = DefaultConfiguration.objects.get_configuration_for_user(app, user) @@ -106,16 +110,13 @@ def app_configuration_for_user(request): user_id = request.POST.get('user_id') properties = json.loads(request.POST.get('properties')) - if not app or not user_id or not properties: + if not app or not properties: raise PopupException(_('app_configuration_for_user requires app, user_id, and properties')) - try: - user = User.objects.get(id=int(user_id)) - except User.DoesNotExist as e: - raise PopupException(_('Could not find user with ID: %s') % user_id) + user = resolve_target_user(request, user_id) config = _save_configuration(app, properties, is_default=False, user=user) - LOG.info('Saved user configuration for app: %s and group_id: %s' % (app, user_id)) + LOG.info('Saved user configuration for app: %s and user_id: %s' % (app, user.id)) response = { 'status': 0, @@ -138,6 +139,7 @@ def delete_default_configuration(request): try: config_id = int(config_id) config = DefaultConfiguration.objects.get(id=config_id) + assert_can_delete_configuration(request, config) config.delete() except DefaultConfiguration.DoesNotExist as e: raise PopupException(_('Could not find configuration with ID: %d') % config_id) diff --git a/desktop/core/src/desktop/configuration/authorization.py b/desktop/core/src/desktop/configuration/authorization.py new file mode 100644 index 00000000000..42d362425f5 --- /dev/null +++ b/desktop/core/src/desktop/configuration/authorization.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# Licensed to Cloudera, Inc. under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. Cloudera, Inc. licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Authorization for DefaultConfiguration REST endpoints. + +Non-admin users may only read and write their own user-specific configuration rows. +Default and group-wide configuration management requires a Hue admin (``is_admin``). +""" + +from django.contrib.auth.models import User +from django.utils.translation import gettext as _ + +from desktop.auth.backend import is_admin +from desktop.lib.exceptions_renderable import PopupException + + +def require_configuration_admin(user): + """ + :raises PopupException: When ``user`` is not a Hue admin. + """ + if not is_admin(user): + raise PopupException(_("You must be an admin."), error_code=401) + + +def resolve_target_user(request, user_id): + """ + Resolve the User record that the caller is allowed to access for per-user configuration. + + Admins may specify any valid user id. Non-admins may only use their own id. + + :param request: Django request with authenticated ``request.user``. + :param user_id: Raw ``user_id`` from GET/POST (string or int). + :return: ``User`` instance. + :raises PopupException: Invalid id, unknown user, or cross-user access without admin rights. + """ + if user_id is None or str(user_id).strip() == '': + raise PopupException(_('app_configuration_for_user requires app and user_id')) + + try: + requested_id = int(user_id) + except (TypeError, ValueError): + raise PopupException(_('Invalid user_id.')) + + if is_admin(request.user): + try: + return User.objects.get(id=requested_id) + except User.DoesNotExist: + raise PopupException(_('Could not find user with User ID: %s') % requested_id) + + if requested_id != request.user.id: + raise PopupException( + _('You do not have permission to access configuration for this user.'), + error_code=403, + ) + + return request.user + + +def assert_can_delete_configuration(request, config): + """ + Ensure the caller may delete a ``DefaultConfiguration`` row. + + Default and group configurations require admin. User-specific rows may be deleted by the + owner or an admin. + + :raises PopupException: When the caller lacks permission. + """ + if config.is_default or config.groups.exists(): + require_configuration_admin(request.user) + return + + if config.user_id is not None: + if config.user_id != request.user.id and not is_admin(request.user): + raise PopupException( + _('You do not have permission to delete this configuration.'), + error_code=403, + ) + return + + require_configuration_admin(request.user) diff --git a/desktop/core/src/desktop/configuration/tests.py b/desktop/core/src/desktop/configuration/tests.py index cbcac0be106..13a8d1d41bd 100644 --- a/desktop/core/src/desktop/configuration/tests.py +++ b/desktop/core/src/desktop/configuration/tests.py @@ -248,3 +248,142 @@ def test_get_default_configurations(self): assert content['configuration']['user'] == self.user.username, content assert content['configuration']['group_ids'] == [], content assert content['configuration']['properties'] == user_properties, content + + +@pytest.mark.django_db +class TestConfigurationAuthorization(object): + """ + Regression tests for ISEC-V2026-03: DefaultConfiguration API must not allow IDOR on user_id or + unprivileged access to global default/group configuration. + """ + + def setup_method(self): + self.client = make_logged_in_client(username="test_admin", groupname="default", recreate=False, is_superuser=True) + self.client_user = make_logged_in_client(username="test_user", groupname="default", recreate=False, is_superuser=False) + + self.admin = User.objects.get(username="test_admin") + self.user = User.objects.get(username="test_user") + + self.group = get_default_user_group() + + grant_access(self.admin.username, self.admin.username, "desktop") + grant_access(self.user.username, self.user.username, "desktop") + + def teardown_method(self): + DefaultConfiguration.objects.all().delete() + + def test_non_admin_cannot_list_or_update_global_configurations(self): + response = self.client_user.get("/desktop/api/configurations/") + content = json.loads(response.content) + assert content['status'] == -1 + assert 'admin' in content['message'].lower() + + response = self.client_user.post( + "/desktop/api/configurations/", + {'configuration': json.dumps({'hive': {'default': []}})}, + ) + content = json.loads(response.content) + assert content['status'] == -1 + + def test_non_admin_cannot_read_other_users_effective_configuration(self): + response = self.client_user.get("/desktop/api/configurations/user", { + 'app': 'hive', + 'user_id': self.admin.id, + }) + content = json.loads(response.content) + assert content['status'] == -1 + assert 'permission' in content['message'].lower() + + def test_non_admin_cannot_save_configuration_for_other_user(self): + response = self.client_user.post("/desktop/api/configurations/user", { + 'app': 'hive', + 'user_id': self.admin.id, + 'properties': json.dumps([{'key': 'settings', 'value': []}]), + }) + content = json.loads(response.content) + assert content['status'] == -1 + assert DefaultConfiguration.objects.filter(app='hive', user=self.admin).count() == 0 + + def test_non_admin_can_read_and_save_own_user_configuration(self): + user_properties = [{'settings': [{'key': 'hive.execution.engine', 'value': 'tez'}]}] + + response = self.client_user.post("/desktop/api/configurations/user", { + 'app': 'hive', + 'user_id': self.user.id, + 'properties': json.dumps(user_properties), + }) + content = json.loads(response.content) + assert content['status'] == 0, content + + response = self.client_user.get("/desktop/api/configurations/user", { + 'app': 'hive', + 'user_id': self.user.id, + }) + content = json.loads(response.content) + assert content['status'] == 0, content + assert content['configuration']['user'] == self.user.username + assert content['configuration']['properties'] == user_properties + + def test_admin_can_read_other_users_effective_configuration(self): + response = self.client.get("/desktop/api/configurations/user", { + 'app': 'hive', + 'user_id': self.user.id, + }) + content = json.loads(response.content) + assert content['status'] == 0, content + + def test_non_admin_cannot_delete_default_configuration(self): + default_config = DefaultConfiguration.objects.create(app='hive', is_default=True, properties='[]') + + response = self.client_user.post("/desktop/api/configurations/delete/", {'id': default_config.id}) + content = json.loads(response.content) + assert content['status'] == -1 + assert DefaultConfiguration.objects.filter(id=default_config.id).exists() + + def test_non_admin_cannot_delete_other_users_configuration(self): + other_user_config = DefaultConfiguration.objects.create( + app='hive', + is_default=False, + user=self.admin, + properties='[]', + ) + + response = self.client_user.post("/desktop/api/configurations/delete/", {'id': other_user_config.id}) + content = json.loads(response.content) + assert content['status'] == -1 + assert DefaultConfiguration.objects.filter(id=other_user_config.id).exists() + + def test_user_can_delete_own_configuration(self): + own_config = DefaultConfiguration.objects.create( + app='hive', + is_default=False, + user=self.user, + properties='[]', + ) + + response = self.client_user.post("/desktop/api/configurations/delete/", {'id': own_config.id}) + content = json.loads(response.content) + assert content['status'] == 0, content + assert not DefaultConfiguration.objects.filter(id=own_config.id).exists() + + def test_non_admin_cannot_delete_group_configuration(self): + group_config = DefaultConfiguration.objects.create(app='hive', is_default=False, properties='[]') + group_config.groups.add(self.group) + + response = self.client_user.post("/desktop/api/configurations/delete/", {'id': group_config.id}) + content = json.loads(response.content) + assert content['status'] == -1 + assert DefaultConfiguration.objects.filter(id=group_config.id).exists() + + def test_admin_can_delete_any_configuration(self): + user_config = DefaultConfiguration.objects.create( + app='hive', + is_default=False, + user=self.user, + properties='[]', + ) + + response = self.client.post("/desktop/api/configurations/delete/", {'id': user_config.id}) + content = json.loads(response.content) + assert content['status'] == 0, content + assert not DefaultConfiguration.objects.filter(id=user_config.id).exists() diff --git a/desktop/core/src/desktop/js/api/urls.js b/desktop/core/src/desktop/js/api/urls.js index 511a02f005f..c39a6511707 100644 --- a/desktop/core/src/desktop/js/api/urls.js +++ b/desktop/core/src/desktop/js/api/urls.js @@ -30,7 +30,7 @@ export const GIT_API_PREFIX = '/desktop/api/vcs/contents/'; export const S3_API_PREFIX = '/filebrowser/view=' + encodeURIComponent('s3a://'); export const GS_API_PREFIX = '/filebrowser/view=' + encodeURIComponent('gs://'); export const IMPALA_INVALIDATE_API = '/impala/api/invalidate'; -export const CONFIG_SAVE_API = '/desktop/api/configurations/save/'; +export const CONFIG_SAVE_API = '/desktop/api/configurations/user/'; export const CONFIG_APPS_API = '/desktop/api/configurations'; export const SOLR_COLLECTIONS_API = '/indexer/api/indexes/list/'; export const SOLR_FIELDS_API = '/indexer/api/index/list/'; From c9dc2924f3ea3ed0f283d1d09a0e9086bc861336 Mon Sep 17 00:00:00 2001 From: Harsh Gupta <42064744+Harshg999@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:36:38 +0530 Subject: [PATCH 2/6] [ci] Require Python 3.11 and fix GitHub Actions on Ubuntu 26.04 GitHub-hosted ubuntu-latest now resolves to Ubuntu 26.04, where actions/setup-python no longer publishes prebuilt CPython 3.9. Python 3 CI failed at the setup-python step with "version 3.9 was not found for Ubuntu 26.04". Python 3.9 and 3.10 are also end of life. Align the supported, tested and documented Python version on 3.11 only, following the same direction as #4376 (drop EOL interpreters from the matrix). - commitflow-py3.yml: remove the 3.9/3.10/3.11 matrix and run a single job on Python 3.11. Always use build/env for pytest, ruff and artifacts. - Makefile.vars: PYTHON_VERSIONS is python3.11 only, require minor version >= 11 and simplify the pip/virtualenv version settings. - generate_requirements.py: drop the Python 3.9-specific pins on x86_64 and aarch64 (3.11 entries unchanged). - generate_requirements_test.py: update the tests for the 3.11 paths. - Docs: installation dependencies, developer guide, SDK and the ci-cd design note now state Python 3.11 as the required version. Follow-up (separate commit): tools/cloudera and the legacy container Dockerfiles still reference older Python versions. --- .github/workflows/commitflow-py3.yml | 23 ++++----------- Makefile.vars | 27 ++++++------------ desktop/core/generate_requirements.py | 20 ------------- desktop/core/generate_requirements_test.py | 28 +++++++++---------- docs/designs/ci-cd.md | 5 ++-- .../installation/dependencies/_index.md | 7 ++--- .../content/developer/development/_index.md | 9 +++--- docs/sdk/sdk.md | 2 +- 8 files changed, 36 insertions(+), 85 deletions(-) diff --git a/.github/workflows/commitflow-py3.yml b/.github/workflows/commitflow-py3.yml index 75ef2cef2ef..6aaa978f466 100644 --- a/.github/workflows/commitflow-py3.yml +++ b/.github/workflows/commitflow-py3.yml @@ -11,10 +11,6 @@ jobs: hue-build-and-tests: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.9', '3.10', '3.11'] - steps: - name: Checkout uses: actions/checkout@v4 @@ -25,10 +21,10 @@ jobs: run: | ./tools/ci/check_for_commit_message.sh - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.11 uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: '3.11' - name: Cache pip uses: actions/cache@v4 @@ -51,29 +47,20 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential asciidoc libkrb5-dev libldap2-dev libsasl2-dev libxslt-dev libsasl2-modules-gssapi-mit libsnappy-dev libgdbm-dev - export PYTHON_VER=python${{ matrix.python-version }} + export PYTHON_VER=python3.11 export SKIP_PYTHONDEV_CHECK=true export ROOT=$PWD make apps test_prep - name: Run unit tests run: | - if [[ "${{ matrix.python-version }}" == "3.11" ]]; then - ./build/env/bin/pytest - else - ./build/venvs/python${{ matrix.python-version }}/bin/pytest - fi + ./build/env/bin/pytest - name: Run python lint check run: | - if [[ "${{ matrix.python-version }}" == "3.11" ]]; then - PYTHONWARNINGS=always ./build/env/bin/hue runruff check - else - PYTHONWARNINGS=always ./build/venvs/python${{ matrix.python-version }}/bin/hue runruff check - fi + PYTHONWARNINGS=always ./build/env/bin/hue runruff check - name: Upload reports - if: matrix.python-version == '3.11' uses: actions/upload-artifact@v4 with: name: hue-reports diff --git a/Makefile.vars b/Makefile.vars index 0936870b8f7..b971422f214 100644 --- a/Makefile.vars +++ b/Makefile.vars @@ -37,10 +37,9 @@ ifneq ($(wildcard $(ROOT)/Makefile.buildvars),) endif -VER_ERR_MSG = "Variable PYTHON_VER is $(PYTHON_VER) but it only supports >= python3.9. If not set, defaults to python3.11." +VER_ERR_MSG = "Variable PYTHON_VER is $(PYTHON_VER) but it only supports python3.11. If not set, defaults to python3.11." -# Major rewrite to support multi-python environments. -PYTHON_VERSIONS ?= python3.9 python3.10 python3.11 +PYTHON_VERSIONS ?= python3.11 PYTHON_VER ?= python3.11 ifeq ($(filter $(PYTHON_VER), $(PYTHON_VERSIONS)),) @@ -73,7 +72,7 @@ endif BLD_DIR_BIN := $(BLD_DIR_ENV)/bin -# Adds detection and configuration for python3.9 through python3.11. +# Adds detection and configuration for python3.11. # Sets up per-version BLD_DIR_ENV and INST_DIR_ENV variables. # Introduces version-specific requirement file generation via generate_requirements.py. SYS_PYTHON := $(shell IFS=:; for p in $$PATH; do if [ -x "$$p/$(PYTHON_VER)" ]; then echo "$$p/$(PYTHON_VER)"; break; fi; done) @@ -88,7 +87,7 @@ $(info "PYTHON_VER is $(PYTHON_VER).") ifeq ($(shell echo $(PYTHON_VER) | head -c 8),python3.) MINOR_VER = $(shell $(SYS_PYTHON) --version | sed -E 's/Python [0-9]+\.([0-9]+).*/\1/') - ifeq ($(shell test $(MINOR_VER) -lt 9; echo $$?),0) + ifeq ($(shell test $(MINOR_VER) -lt 11; echo $$?),0) $(error "$(VER_ERR_MSG)") endif EXT_ENV_INSTALL = ext-env-pip-install @@ -103,7 +102,7 @@ ifndef SKIP_PYTHONDEV_CHECK endif ifeq ($(SYS_PYTHON),) - $(error "Error: Need python version >= 3.8") + $(error "Error: Need python version >= 3.11") endif HADOOP_HOME ?= /usr/lib/hadoop @@ -164,18 +163,8 @@ VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1 ############################## # Set specific versions for some libraries ############################## -ifeq ($(PYTHON_VER),python3.9) - PIP_VERSION := 22.2.2 - VIRTUAL_ENV_VERSION := 21.7.16 - VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1 -else ifeq ($(PYTHON_VER),python3.10) - PIP_VERSION := 22.2.2 - VIRTUAL_ENV_VERSION := 21.7.16 - VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1 -else ifeq ($(PYTHON_VER),python3.11) - PIP_VERSION := 22.2.2 - VIRTUAL_ENV_VERSION := 21.7.16 - VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1 -endif +PIP_VERSION := 22.2.2 +VIRTUAL_ENV_VERSION := 21.7.16 +VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1 RELOCATABLE := $(shell which virtualenv-make-relocatable) \ No newline at end of file diff --git a/desktop/core/generate_requirements.py b/desktop/core/generate_requirements.py index fd1aa5d79c7..6d0cbc2d4e8 100755 --- a/desktop/core/generate_requirements.py +++ b/desktop/core/generate_requirements.py @@ -134,16 +134,6 @@ def __init__(self): "setuptools==70.0.0", "sqlparse==0.6.0", ], - "3.9": [ - "decorator==5.1.1", - "lxml==4.9.1", - "Markdown==3.8.1", - "numpy==1.24.4", - "pandas==2.0.3", - "sasl==0.3.1", - "setuptools==80.9.0", - "sqlparse==0.5.3", # 0.6.0 requires Python >= 3.10 - ], "3.11": [ "async-timeout==5.0.1", "lxml==4.9.1", @@ -165,16 +155,6 @@ def __init__(self): "setuptools==70.0.0", "sqlparse==0.6.0", ], - "3.9": [ - "decorator==5.1.1", - "lxml==4.9.1", - "Markdown==3.8.1", - "numpy==1.24.4", - "pandas==2.0.3", - "sasl==0.3.1", - "setuptools==80.9.0", - "sqlparse==0.5.3", # 0.6.0 requires Python >= 3.10 - ], "3.11": [ "async-timeout==5.0.1", "lxml==4.9.1", diff --git a/desktop/core/generate_requirements_test.py b/desktop/core/generate_requirements_test.py index fe95a67dbfb..62a7565f825 100644 --- a/desktop/core/generate_requirements_test.py +++ b/desktop/core/generate_requirements_test.py @@ -67,13 +67,13 @@ def test_init_sets_properties(self, mock_version_info, mock_machine, mock_this_d mock_machine.return_value = "x86_64" mock_version_info.major = 3 - mock_version_info.minor = 9 + mock_version_info.minor = 11 mock_this_dir.return_value = self.temp_dir generator = RequirementsGenerator() assert generator.arch == "x86_64" - assert generator.python_version_string == "3.9" + assert generator.python_version_string == "3.11" assert isinstance(generator.requirements, list) assert isinstance(generator.local_requirements, list) assert "x86_64" in generator.arch_requirements_map @@ -87,7 +87,7 @@ def test_copy_local_requirements(self, mock_copytree, mock_this_dir): # Need to make this_dir return the string value directly, not a MagicMock mock_this_dir.__str__.return_value = self.temp_dir - python_version_string = "3.9" + python_version_string = "3.11" with mock.patch.object(RequirementsGenerator, "__init__", return_value=None): generator = RequirementsGenerator() @@ -112,11 +112,11 @@ def test_generate_requirements_x86_64(self, mock_open, mock_this_dir): with mock.patch.object(RequirementsGenerator, "__init__", return_value=None): generator = RequirementsGenerator() generator.arch = "x86_64" - generator.python_version_string = "3.9" + generator.python_version_string = "3.11" generator.requirements = ["Django==4.1.13"] generator.local_requirements = [] generator.arch_requirements_map = { - "x86_64": {"default": ["cryptography==42.0.8"], "3.9": ["Markdown==3.8", "numpy==1.24.4", "setuptools==80.9.0"]} + "x86_64": {"default": ["cryptography==42.0.8"], "3.11": ["Markdown==3.8", "numpy==1.24.4", "setuptools==80.9.0"]} } # Mock copy_local_requirements for test isolation @@ -125,7 +125,7 @@ def test_generate_requirements_x86_64(self, mock_open, mock_this_dir): generator.generate_requirements() # Verify file creation with correct path - mock_open.assert_called_once_with(f"{self.temp_dir}/requirements-x86_64-3.9.txt", "w") + mock_open.assert_called_once_with(f"{self.temp_dir}/requirements-x86_64-3.11.txt", "w") # Verify correct requirements were written expected_requirements = "\n".join(["Django==4.1.13", "Markdown==3.8", "numpy==1.24.4", "setuptools==80.9.0"]) @@ -141,7 +141,7 @@ def test_generate_requirements_unsupported_arch(self, mock_this_dir): with mock.patch.object(RequirementsGenerator, "__init__", return_value=None): generator = RequirementsGenerator() generator.arch = "unsupported_arch" - generator.python_version_string = "3.9" + generator.python_version_string = "3.11" generator.requirements = [] generator.arch_requirements_map = {"x86_64": {"default": []}, "aarch64": {"default": []}} @@ -160,11 +160,11 @@ def test_get_file_name(self, mock_this_dir): with mock.patch.object(RequirementsGenerator, "__init__", return_value=None): generator = RequirementsGenerator() generator.arch = "x86_64" - generator.python_version_string = "3.9" + generator.python_version_string = "3.11" result = generator.get_file_name() - assert result == f"{self.temp_dir}/requirements-x86_64-3.9.txt" + assert result == f"{self.temp_dir}/requirements-x86_64-3.11.txt" @mock.patch("generate_requirements.this_dir") @mock.patch("builtins.open", new_callable=mock.mock_open) @@ -177,21 +177,21 @@ def test_generate_requirements_with_local_requirements(self, mock_open, mock_thi with mock.patch.object(RequirementsGenerator, "__init__", return_value=None): generator = RequirementsGenerator() generator.arch = "x86_64" - generator.python_version_string = "3.9" + generator.python_version_string = "3.11" generator.requirements = [] generator.local_requirements = ["boto-2.49.0", "django-axes-5.13.0"] - generator.arch_requirements_map = {"x86_64": {"default": [], "3.9": ["Markdown==3.8", "setuptools==80.9.0"]}} + generator.arch_requirements_map = {"x86_64": {"default": [], "3.11": ["Markdown==3.8", "setuptools==80.9.0"]}} # Mock copy_local_requirements to return file paths - local_reqs = [f"file://{self.temp_dir}/3.9/boto-2.49.0", f"file://{self.temp_dir}/3.9/django-axes-5.13.0"] + local_reqs = [f"file://{self.temp_dir}/3.11/boto-2.49.0", f"file://{self.temp_dir}/3.11/django-axes-5.13.0"] generator.copy_local_requirements = mock.MagicMock(return_value=local_reqs) generator.generate_requirements() - mock_open.assert_called_once_with(f"{self.temp_dir}/requirements-x86_64-3.9.txt", "w") + mock_open.assert_called_once_with(f"{self.temp_dir}/requirements-x86_64-3.11.txt", "w") expected_requirements = "\n".join( - ["Markdown==3.8", "setuptools==80.9.0", f"file://{self.temp_dir}/3.9/boto-2.49.0", f"file://{self.temp_dir}/3.9/django-axes-5.13.0"] + ["Markdown==3.8", "setuptools==80.9.0", f"file://{self.temp_dir}/3.11/boto-2.49.0", f"file://{self.temp_dir}/3.11/django-axes-5.13.0"] ) mock_open().write.assert_called_once_with(expected_requirements) diff --git a/docs/designs/ci-cd.md b/docs/designs/ci-cd.md index 575ca91f9d4..57e6d097311 100644 --- a/docs/designs/ci-cd.md +++ b/docs/designs/ci-cd.md @@ -12,10 +12,9 @@ Be fast and simple: ### Python -Similar to the [Docker Hue](/tools/docker/hue). Currently doing only Python 3.8 and 3.9, but we should test the Python 3.10+ build and run the tests. +Similar to the [Docker Hue](/tools/docker/hue). GitHub Actions validates the build and unit tests on Python 3.11. -* Python 3.8 -* Python 3.9 +* Python 3.11 The docker images could be forked for QA and comes with a series of flags. diff --git a/docs/docs-site/content/administrator/installation/dependencies/_index.md b/docs/docs-site/content/administrator/installation/dependencies/_index.md index 8895bd7549c..10afd73f902 100644 --- a/docs/docs-site/content/administrator/installation/dependencies/_index.md +++ b/docs/docs-site/content/administrator/installation/dependencies/_index.md @@ -12,8 +12,6 @@ You will need Python, some OS packages, a Database and Node.js. Hue employs some Python modules which use native code and requires certain development libraries be installed on your system. To install from the tarball, you'll need these library development packages and tools installed on your system: Versions supported: -* Python 3.8 -* Python 3.9 * Python 3.11 ``` # If you are using Python 3.11, set PYTHON_VER before the build, like @@ -117,7 +115,7 @@ There is more details on this [Apply Temporary Workaround for Oracle 12 Client]( With Python 3, also: - sudo apt-get install python3.8-dev python3-distutils + sudo apt-get install python3.11-dev python3-distutils ### CentOS/RHEL @@ -186,8 +184,7 @@ This is a verified step-by-step guide on how to get Hue up and running on a fres - For Apple Silicon (M1, M2, M3), Python 3.11 is recommended as of June 2025. You can install Python 3.11 either via Homebrew or by direct download from python.org—both options will work, but the direct download is preferred and tested. ->**Note:** By default, Hue will use the highest available Python version on your system. If you want to use a specific lower Python version (such as Python 3.8 or 3.9), set the `PYTHON_VER` environment variable before building. -For Python 3.11, use `./build/env/bin/hue` as the command path. For Python versions lower than 3.11, the command path will be `./build/venvs/python3.x/bin/hue`, where `python3.x` matches your `PYTHON_VER` setting. Adjust all related commands accordingly if you use a lower Python version. Both options will work as long as you use the correct path for your Python version. +>**Note:** Hue requires Python 3.11. Set `export PYTHON_VER=python3.11` before building and use `./build/env/bin/hue` for Hue commands. 8. Install node and npm (unless already present) using nvm or brew diff --git a/docs/docs-site/content/developer/development/_index.md b/docs/docs-site/content/developer/development/_index.md index 1a585a7c6b4..580e5ef2dd1 100644 --- a/docs/docs-site/content/developer/development/_index.md +++ b/docs/docs-site/content/developer/development/_index.md @@ -13,7 +13,7 @@ This section goes into greater detail on how to build and reuse the components o ### Dependencies * The OS specific packages are listed in the [install guide](/administrator/installation/dependencies/) -* Python 3.8/3.9 and Django 3.2 +* Python 3.11 * Vue.js 3 * Node.js ([20.0+](https://deb.nodesource.com/setup_20.x)) @@ -21,8 +21,7 @@ This section goes into greater detail on how to build and reuse the components o Build once: - # If you are using Python 3.8, set PYTHON_VER before the build, like - export PYTHON_VER=python3.8 + export PYTHON_VER=python3.11 # Mac user might need to set export SKIP_PYTHONDEV_CHECK=true @@ -217,7 +216,7 @@ Once we get ship it from at least two reviewer, we can push the changes to maste * The push will auto run the tests and push it to master * It can be seen on https://github.com/cloudera/hue/actions - * Two builds should be green - One for Python 3.8 and another for Python 3.9 + * Python 3 CI should be green on Python 3.11 * If successful, the change would be auto merged to master * On failure, we will get a mail * Runs usually take 10-20 min @@ -752,7 +751,7 @@ Publish Github NPM package and Docker images at https://github.com/orgs/cloudera Building the tarball release: git checkout -b release-4.11.0 release-4.11.0 - export PYTHON_VER=python3.8 + export PYTHON_VER=python3.11 make prod You might need to upgrade the [Mysqlclient](https://docs.gethue.com/administrator/installation/dependencies/#mysql--mariadb) if seeing: diff --git a/docs/sdk/sdk.md b/docs/sdk/sdk.md index 3e0f3b3a359..fec57d74372 100644 --- a/docs/sdk/sdk.md +++ b/docs/sdk/sdk.md @@ -958,7 +958,7 @@ namespace. See ``apps/about/src/about/urls.py`` for an example. ### Dependencies * The OS specific dependencies listed [here](https://docs.gethue.com/admin-manual/manual.html) -* Python 3.8 or Python 3.9 +* Python 3.11 * Django (3.2 included with our distribution) * Hadoop (Apache Hadoop 2+) * Java (Java 1.8) From ed880718da3684b281a162e45bc45a99f8c66702 Mon Sep 17 00:00:00 2001 From: Harsh Gupta <42064744+Harshg999@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:06:17 +0530 Subject: [PATCH 3/6] [desktop] Fix ruff lint in configuration API and tests CI runs `hue runruff check` with PYTHONWARNINGS=always. The configuration authorization changes introduced import order, unused exception bindings and style issues in api.py and tests.py. Apply the ruff fixes (I001, F401, F841, E303, E711, E712) so the Python 3 and ARM64 CI lint steps pass. No behavior change. --- .../src/desktop/configuration/__init__.py | 2 +- desktop/core/src/desktop/configuration/api.py | 7 +++---- .../core/src/desktop/configuration/tests.py | 19 ++++++++----------- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/desktop/core/src/desktop/configuration/__init__.py b/desktop/core/src/desktop/configuration/__init__.py index a533c04a920..d053ec41e3c 100644 --- a/desktop/core/src/desktop/configuration/__init__.py +++ b/desktop/core/src/desktop/configuration/__init__.py @@ -13,4 +13,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. diff --git a/desktop/core/src/desktop/configuration/api.py b/desktop/core/src/desktop/configuration/api.py index d46ae630c9e..0674ab916da 100644 --- a/desktop/core/src/desktop/configuration/api.py +++ b/desktop/core/src/desktop/configuration/api.py @@ -15,7 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import json import logging @@ -39,7 +38,7 @@ try: from oozie.models2 import WorkflowConfiguration as OozieWorkflowConfiguration -except (ImportError, RuntimeError) as e: +except (ImportError, RuntimeError): OozieWorkflowConfiguration = None @@ -141,7 +140,7 @@ def delete_default_configuration(request): config = DefaultConfiguration.objects.get(id=config_id) assert_can_delete_configuration(request, config) config.delete() - except DefaultConfiguration.DoesNotExist as e: + except DefaultConfiguration.DoesNotExist: raise PopupException(_('Could not find configuration with ID: %d') % config_id) return JsonResponse({ @@ -222,7 +221,7 @@ def _update_default_and_group_configurations(configurations): try: groups = Group.objects.filter(id__in=group_ids) _save_configuration(app, properties, is_default=False, groups=groups) - except Group.DoesNotExist as e: + except Group.DoesNotExist: raise PopupException(_('Could not find one or more groups with IDs: %s') % ', '.join(group_ids)) except Exception as e: raise PopupException(_('Failed to update configurations: %s') % e) diff --git a/desktop/core/src/desktop/configuration/tests.py b/desktop/core/src/desktop/configuration/tests.py index 13a8d1d41bd..83e58804808 100644 --- a/desktop/core/src/desktop/configuration/tests.py +++ b/desktop/core/src/desktop/configuration/tests.py @@ -16,14 +16,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from builtins import object import json +from builtins import object + import pytest from desktop.lib.django_test_util import make_logged_in_client from desktop.lib.test_utils import grant_access from desktop.models import DefaultConfiguration - from useradmin.models import get_default_user_group, User @@ -42,11 +42,9 @@ def setup_method(self): grant_access(self.admin.username, self.admin.username, "desktop") grant_access(self.user.username, self.user.username, "desktop") - def teardown_method(self): DefaultConfiguration.objects.all().delete() - def test_update_default_and_group_configurations(self): configuration = { 'hive': { @@ -119,7 +117,6 @@ def test_update_default_and_group_configurations(self): config = DefaultConfiguration.objects.get(app='hive', groups__in=[self.group]) assert config.properties_list == configuration['hive']['groups'][0]['properties'], config.properties_list - def test_get_default_configurations(self): app = 'hive' properties = [ @@ -176,7 +173,7 @@ def test_get_default_configurations(self): 'user_id': self.user.id}) content = json.loads(response.content) assert content['status'] == 0, content - assert content['configuration'] == None, content + assert content['configuration'] is None, content # Creating a default configuration returns default response = self.client.post("/desktop/api/configurations/", {'configuration': json.dumps(configuration)}) @@ -187,8 +184,8 @@ def test_get_default_configurations(self): content = json.loads(response.content) assert content['status'] == 0, content assert content['configuration']['app'] == 'hive', content - assert content['configuration']['is_default'] == True, content - assert content['configuration']['user'] == None, content + assert content['configuration']['is_default'], content + assert content['configuration']['user'] is None, content assert content['configuration']['group_ids'] == [], content assert content['configuration']['properties'] == properties, content @@ -222,8 +219,8 @@ def test_get_default_configurations(self): content = json.loads(response.content) assert content['status'] == 0, content assert content['configuration']['app'] == 'hive', content - assert content['configuration']['is_default'] == False, content - assert content['configuration']['user'] == None, content + assert not content['configuration']['is_default'], content + assert content['configuration']['user'] is None, content assert content['configuration']['group_ids'] == [self.group.id], content assert content['configuration']['properties'] == group_properties, content @@ -244,7 +241,7 @@ def test_get_default_configurations(self): content = json.loads(response.content) assert content['status'] == 0, content assert content['configuration']['app'] == 'hive', content - assert content['configuration']['is_default'] == False, content + assert not content['configuration']['is_default'], content assert content['configuration']['user'] == self.user.username, content assert content['configuration']['group_ids'] == [], content assert content['configuration']['properties'] == user_properties, content From 1b8deeb66cc89a90e4bc20b5ab4dd7237da24476 Mon Sep 17 00:00:00 2001 From: Harsh Gupta <42064744+Harshg999@users.noreply.github.com> Date: Tue, 27 Oct 2026 13:47:40 +0530 Subject: [PATCH 4/6] [core] Complete Python 3.11-only build and packaging cleanup Follow-up to the CI/Makefile minimum of Python 3.11: remove the remaining multi-Python and EOL interpreter paths across the Cloudera builds, containers, scripts and requirement generation, so the supported, tested and documented versions line up. Cloudera packaging (tools/cloudera): - build_hue_cloudera.sh: single python3.11 build/release. All supported OS targets use PYTHON311_OSES and always build into build/env. - build_hue_common.sh: drop install_python39 and the Python 3.9 install hooks from every OS installer. install_sqlite_python only bootstraps 3.11.12. Runtime and developer tooling: - python_helper.sh and hue.sh: resolve build/env/bin only and reject other HUE_PYTHON_VERSION values. - Makefile.vars: always use build/env and inst/env (no build/venvs layout). - runruff.py: always use build/env/bin/ruff. - relocatable.sh: document and gate on Python 3.11. - pyproject.toml: ruff target-version is py311 and pytest norecursedirs drops the removed 3.8-3.10 directories. Requirements: - generate_requirements.py: one 3.11 arch-specific pin set. Fail on an unsupported Python minor version instead of falling back to legacy "default" pins. Refresh the stale version comments. Containers: - UBI compile/base/hue Dockerfiles: python3.11 packages and pip paths. Install psycopg2-binary with pip instead of a cp38 wheel. Tests: - generate_requirements_test.py updated. Not changed: historical gethue posts, vendored third-party CI configs and tools/docker/hue/Dockerfile.py2 (legacy Python 2 image). --- Makefile.vars | 14 +-- desktop/core/generate_requirements.py | 65 ++++------- desktop/core/generate_requirements_test.py | 6 +- .../desktop/management/commands/runruff.py | 11 +- pyproject.toml | 4 +- tools/cloudera/build_hue_cloudera.sh | 104 +++++------------ tools/cloudera/build_hue_common.sh | 108 +++--------------- tools/container/base/hue/Dockerfile | 7 +- tools/container/build.sh | 2 +- tools/container/compile/hue/Dockerfile | 6 +- tools/container/hue/Dockerfile | 6 +- tools/docker/hue/Dockerfile | 2 +- tools/relocatable.sh | 6 +- tools/scripts/hue.sh | 3 +- tools/scripts/python/python_helper.sh | 97 ++-------------- 15 files changed, 104 insertions(+), 337 deletions(-) diff --git a/Makefile.vars b/Makefile.vars index b971422f214..d82967eff62 100644 --- a/Makefile.vars +++ b/Makefile.vars @@ -51,23 +51,13 @@ endif ############################## BLD_DIR := $(ROOT)/build -# Default case for Python 3.11 ifndef BLD_DIR_ENV - ifeq ($(PYTHON_VER),python3.11) - BLD_DIR_ENV := $(BLD_DIR)/env - else - BLD_DIR_ENV := $(BLD_DIR)/venvs/$(PYTHON_VER) - endif + BLD_DIR_ENV := $(BLD_DIR)/env endif ifdef INSTALL_DIR INST_DIR := $(INSTALL_DIR)/build - - ifeq ($(BLD_DIR_ENV),"$(BLD_DIR)/env") - INST_DIR_ENV := $(INST_DIR)/env - else - INST_DIR_ENV := $(INST_DIR)/venvs/$(PYTHON_VER) - endif + INST_DIR_ENV := $(INST_DIR)/env endif BLD_DIR_BIN := $(BLD_DIR_ENV)/bin diff --git a/desktop/core/generate_requirements.py b/desktop/core/generate_requirements.py index 6d0cbc2d4e8..0943381bf9c 100755 --- a/desktop/core/generate_requirements.py +++ b/desktop/core/generate_requirements.py @@ -15,10 +15,10 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -Dynamic requirements generator for the Hue multi-Python build system. +Dynamic requirements generator for the Hue build system (Python 3.11). -Generate architecture- and Python-version-specific requirement files at -build time, replacing a static requirements.txt. This tool supports: +Generate architecture-specific requirement files at build time, replacing +a static requirements.txt. This tool supports: - Local editable paths for Hue extensions. - Binary wheels for specific platforms (e.g., ppc64le). @@ -116,7 +116,7 @@ def __init__(self): "six==1.16.0", "slack-sdk==3.31.0", "SQLAlchemy==1.3.8", - # sqlparse is pinned per Python version below: 0.6.0 fixes three DoS CVEs but needs Python >= 3.10. + # sqlparse is pinned in the architecture-specific requirements below: 0.6.0 fixes three DoS CVEs. "tablib==0.13.0", "tabulate==0.8.9", "thrift-sasl==0.4.3", @@ -124,47 +124,23 @@ def __init__(self): "git+https://github.com/gethue/thrift.git", ] + python_311_arch_extras = [ + "async-timeout==5.0.1", + "lxml==4.9.1", + "Markdown==3.8.1", + "numpy==1.24.4", + "pandas==2.0.3", + "pure-sasl==0.6.2", + "setuptools==80.9.0", + "sqlparse==0.6.0", + ] + self.x86_64_requirements = { - "default": [ - "lxml==4.9.1", - "Markdown==3.8.1", - "numpy==1.24.4", - "pandas==2.0.3", - "sasl==0.3.1", - "setuptools==70.0.0", - "sqlparse==0.6.0", - ], - "3.11": [ - "async-timeout==5.0.1", - "lxml==4.9.1", - "Markdown==3.8.1", - "numpy==1.24.4", - "pandas==2.0.3", - "pure-sasl==0.6.2", - "setuptools==80.9.0", - "sqlparse==0.6.0", - ], + "3.11": python_311_arch_extras, } self.aarch64_requirements = { - "default": [ - "lxml==4.9.1", - "Markdown==3.8.1", - "numpy==1.24.4", - "pandas==2.0.3", - "setuptools==70.0.0", - "sqlparse==0.6.0", - ], - "3.11": [ - "async-timeout==5.0.1", - "lxml==4.9.1", - "Markdown==3.8.1", - "numpy==1.24.4", - "pandas==2.0.3", - "pure-sasl==0.6.2", - "setuptools==80.9.0", - "sqlparse==0.6.0", - ], + "3.11": python_311_arch_extras, } self.arch_requirements_map = { "x86_64": self.x86_64_requirements, @@ -185,7 +161,12 @@ def generate_requirements(self): if self.arch not in self.arch_requirements_map: raise ValueError(f"Unsupported architecture: {self.arch}") arch_reqs = self.arch_requirements_map[self.arch] - self.requirements.extend(arch_reqs.get(self.python_version_string, arch_reqs["default"])) + if self.python_version_string not in arch_reqs: + raise ValueError( + f"Unsupported Python version {self.python_version_string} for architecture {self.arch}. " + f"Supported: {', '.join(sorted(arch_reqs.keys()))}" + ) + self.requirements.extend(arch_reqs[self.python_version_string]) self.requirements.extend(self.copy_local_requirements(self.python_version_string)) with open(f"{this_dir}/requirements-{self.arch}-{self.python_version_string}.txt", "w") as f: f.write("\n".join(self.requirements)) diff --git a/desktop/core/generate_requirements_test.py b/desktop/core/generate_requirements_test.py index 62a7565f825..fb781617792 100644 --- a/desktop/core/generate_requirements_test.py +++ b/desktop/core/generate_requirements_test.py @@ -116,7 +116,7 @@ def test_generate_requirements_x86_64(self, mock_open, mock_this_dir): generator.requirements = ["Django==4.1.13"] generator.local_requirements = [] generator.arch_requirements_map = { - "x86_64": {"default": ["cryptography==42.0.8"], "3.11": ["Markdown==3.8", "numpy==1.24.4", "setuptools==80.9.0"]} + "x86_64": {"3.11": ["Markdown==3.8", "numpy==1.24.4", "setuptools==80.9.0"]} } # Mock copy_local_requirements for test isolation @@ -143,7 +143,7 @@ def test_generate_requirements_unsupported_arch(self, mock_this_dir): generator.arch = "unsupported_arch" generator.python_version_string = "3.11" generator.requirements = [] - generator.arch_requirements_map = {"x86_64": {"default": []}, "aarch64": {"default": []}} + generator.arch_requirements_map = {"x86_64": {"3.11": []}, "aarch64": {"3.11": []}} with pytest.raises(ValueError) as exc_info: generator.generate_requirements() @@ -180,7 +180,7 @@ def test_generate_requirements_with_local_requirements(self, mock_open, mock_thi generator.python_version_string = "3.11" generator.requirements = [] generator.local_requirements = ["boto-2.49.0", "django-axes-5.13.0"] - generator.arch_requirements_map = {"x86_64": {"default": [], "3.11": ["Markdown==3.8", "setuptools==80.9.0"]}} + generator.arch_requirements_map = {"x86_64": {"3.11": ["Markdown==3.8", "setuptools==80.9.0"]}} # Mock copy_local_requirements to return file paths local_reqs = [f"file://{self.temp_dir}/3.11/boto-2.49.0", f"file://{self.temp_dir}/3.11/django-axes-5.13.0"] diff --git a/desktop/core/src/desktop/management/commands/runruff.py b/desktop/core/src/desktop/management/commands/runruff.py index 8bee910a579..493db493686 100644 --- a/desktop/core/src/desktop/management/commands/runruff.py +++ b/desktop/core/src/desktop/management/commands/runruff.py @@ -15,11 +15,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys +import argparse import logging import os.path -import argparse import subprocess +import sys from django.core.management.base import BaseCommand from django.utils.translation import gettext as _ @@ -107,12 +107,7 @@ def handle(self, *args, **options): *args: Variable arguments. **options: Keyword arguments. """ - PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}" - - if PYTHON_VERSION == "3.11": - ruff_package = paths.get_build_dir('env', 'bin', 'ruff') - else: - ruff_package = paths.get_build_dir('venvs', f'python{PYTHON_VERSION}', 'bin', 'ruff') + ruff_package = paths.get_build_dir('env', 'bin', 'ruff') if not os.path.exists(ruff_package): msg = _( diff --git a/pyproject.toml b/pyproject.toml index e1303674784..ae00c5f0970 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ addopts = """ --cov-report html:reports/code-cov/coverage-html \ --cov-report xml:reports/code-cov/coverage.xml \ """ -norecursedirs = "desktop/core/ext-py3 desktop/core/3.11 desktop/core/3.10 desktop/core/3.9 desktop/core/3.8" +norecursedirs = "desktop/core/ext-py3 desktop/core/3.11" python_files = "tests.py test_*.py *_tests.py tests_* *_test.py" markers = [ "integration: live server based tests", @@ -88,7 +88,7 @@ source = [ precision = 2 [tool.ruff] -target-version = "py38" +target-version = "py311" line-length = 140 indent-width = 2 force-exclude = true diff --git a/tools/cloudera/build_hue_cloudera.sh b/tools/cloudera/build_hue_cloudera.sh index 7c9b62e1113..4ca8cf64baa 100755 --- a/tools/cloudera/build_hue_cloudera.sh +++ b/tools/cloudera/build_hue_cloudera.sh @@ -37,13 +37,12 @@ BUILDTYPE=$1 DOCKEROS=$2 HUE_JAR_VERSION=$3 -PYTHON39_OSES=("centos7" "redhat8" "redhat8-arm64" "redhat9" "redhat9-arm64" "sles12" "ubuntu20") -PYTHON311_OSES=("redhat8" "redhat8-arm64" "redhat9" "sles15" "ubuntu22" "ubuntu24" "redhat9-arm64") +# All Cloudera packaging targets build Hue with Python 3.11 (see Makefile.vars). +PYTHON311_OSES=( + "centos7" "redhat8" "redhat8-arm64" "redhat9" "redhat9-arm64" + "sles12" "sles15" "ubuntu18" "ubuntu20" "ubuntu22" "ubuntu24" +) -LATEST_PYTHON="python3.11" -PYTHON_VERSIONS=("python3.11" "python3.9") - -export REQ_PYTHON39="3.9.16" export REQ_PYTHON311="3.11.12" export DESKTOP_VERSION=$DOCKEROS @@ -55,31 +54,16 @@ export LD_RUN_PATH="/usr/local/lib:$ORACLE_INSTANTCLIENT19_PATH:$LD_RUN_PATH" export ORIGINAL_PATH=$PATH setup_python_env() { - local ver="$1" os="$2" - - case "$ver" in - python3.9) - if is_supported_os PYTHON39_OSES "$os"; then - export PYTHON39_PATH=${PYTHON39_PATH:=/opt/python/3.9.16} - if is_supported_python_version "$PYTHON39_PATH/bin/python3.9" $REQ_PYTHON39; then - export PATH="$PYTHON39_PATH/bin:$PATH" - else - unset PYTHON39_PATH - fi - fi - ;; - python3.11) - if is_supported_os PYTHON311_OSES "$os"; then - export PYTHON311_PATH=${PYTHON311_PATH:=/opt/python/3.11.12} - if is_supported_python_version "$PYTHON311_PATH/bin/python3.11" $REQ_PYTHON311; then - export PATH="$PYTHON311_PATH/bin:$PATH" - else - unset PYTHON311_PATH - fi - fi - ;; - *) echo "Unsupported python version: $ver"; exit 1 ;; - esac + local os="$1" + + if is_supported_os PYTHON311_OSES "$os"; then + export PYTHON311_PATH=${PYTHON311_PATH:=/opt/python/3.11.12} + if is_supported_python_version "$PYTHON311_PATH/bin/python3.11" $REQ_PYTHON311; then + export PATH="$PYTHON311_PATH/bin:$PATH" + else + unset PYTHON311_PATH + fi + fi } install_prerequisite() { @@ -104,60 +88,34 @@ install_prerequisite() { } should_build_python() { - local ver="$1" - local os="$2" - - case "$ver" in - python3.11) - is_supported_os PYTHON311_OSES "$os" - return $? - ;; - python3.9) - is_supported_os PYTHON39_OSES "$os" - return $? - ;; - *) - return 1 - ;; - esac - - return 1 + local os="$1" + is_supported_os PYTHON311_OSES "$os" } -for PYTHON_VER in "${PYTHON_VERSIONS[@]}"; do - setup_python_env "$PYTHON_VER" "$DOCKEROS" -done +setup_python_env "$DOCKEROS" big_console_header "Hue PreRequisite Start for $DOCKEROS" install_prerequisite "$DOCKEROS" big_console_header "Hue PreRequisite End for $DOCKEROS" -THISPATH=$PATH -for PYTHON_VER in "${PYTHON_VERSIONS[@]}"; do +if should_build_python "$DOCKEROS"; then HUE_SRC=$(realpath "$WORK_DIR/../..") cd "$HUE_SRC" || exit 1 - if should_build_python "$PYTHON_VER" "$DOCKEROS"; then - BLD_DIR="${BLD_DIR:-$HUE_SRC/build}" - BLD_DIR_ENV="$BLD_DIR/venvs/${PYTHON_VER}" - [[ "$PYTHON_VER" == "python3.11" || "$DOCKEROS" =~ (sles12|centos7|ubuntu20) ]] && BLD_DIR_ENV="$BLD_DIR/env" - echo "BLD_DIR_ENV=${BLD_DIR_ENV}" + BLD_DIR="${BLD_DIR:-$HUE_SRC/build}" + BLD_DIR_ENV="$BLD_DIR/env" + echo "BLD_DIR_ENV=${BLD_DIR_ENV}" - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ORACLE_INSTANTCLIENT19_PATH} - export PATH=$PYTHON39_PATH/bin:$PYTHON311_PATH/bin:/opt/sqlite3/bin:/usr/bin:$THISPATH + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ORACLE_INSTANTCLIENT19_PATH} + export PATH="${PYTHON311_PATH:+$PYTHON311_PATH/bin:}/opt/sqlite3/bin:/usr/bin:$ORIGINAL_PATH" - big_console_header "Hue Build Start for" "$PYTHON_VER" "$@" - BLD_DIR_ENV="$BLD_DIR_ENV" PYTHON_VER="$PYTHON_VER" make apps docs - BLD_DIR_ENV="$BLD_DIR_ENV" PYTHON_VER="$PYTHON_VER" make relocatable-env - BLD_DIR_ENV="$BLD_DIR_ENV" PYTHON_VER="$PYTHON_VER" make huecheck - big_console_header "Hue Build End for" "$PYTHON_VER" "$@" - fi -done + big_console_header "Hue Build Start for python3.11" "$@" + BLD_DIR_ENV="$BLD_DIR_ENV" PYTHON_VER="python3.11" make apps docs + BLD_DIR_ENV="$BLD_DIR_ENV" PYTHON_VER="python3.11" make relocatable-env + BLD_DIR_ENV="$BLD_DIR_ENV" PYTHON_VER="python3.11" make huecheck + big_console_header "Hue Build End for python3.11" "$@" +fi big_console_header "Hue PROD Build Start for" "$@" -if [[ "$DOCKEROS" =~ (centos7|sles12|ubuntu18) ]]; then - PYTHON_VER="python3.9" make release -else - PYTHON_VER="python3.11" make release -fi +PYTHON_VER="python3.11" make release big_console_header "Hue PROD Build End for" "$@" diff --git a/tools/cloudera/build_hue_common.sh b/tools/cloudera/build_hue_common.sh index 2d686b49917..f3873fac0b1 100644 --- a/tools/cloudera/build_hue_common.sh +++ b/tools/cloudera/build_hue_common.sh @@ -154,51 +154,6 @@ function install_python311() { popd > /dev/null } -function install_python39() { - local PYTHON_VERSION=${1:-"3.9.16"} - local INSTALL_PREFIX=$2 - local LD_LIBRARY_PATH=$3 - local CPPFLAGS=$4 - local LDFLAGS=$5 - pushd . > /dev/null - - cd $HOME - local PYTHON_TGZ="Python-${PYTHON_VERSION}.tgz" - local PYTHON_SRC_DIR="Python-${PYTHON_VERSION}" - - export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/mit/bin:$PATH - - echo "Installing Python ${PYTHON_VERSION}..." - sudo PYTHON_VERSION=${PYTHON_VERSION} INSTALL_PREFIX=${INSTALL_PREFIX} \ - PYTHON_TGZ=${PYTHON_TGZ} PYTHON_SRC_DIR=${PYTHON_SRC_DIR} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} \ - CPPFLAGS=${CPPFLAGS} LDFLAGS=${LDFLAGS} PATH=${PATH} \ - -- sh -c 'cd /tmp && mkdir -p "${INSTALL_PREFIX}" && \ - curl -o "${PYTHON_TGZ}" "https://www.python.org/ftp/python/${PYTHON_VERSION}/${PYTHON_TGZ}" && \ - tar zxf "${PYTHON_TGZ}" && \ - cd "${PYTHON_SRC_DIR}" && \ - ./configure --prefix="${INSTALL_PREFIX}" --enable-shared --enable-optimizations --with-lto && \ - make altinstall' - - # echo "Stripping debug symbols to reduce size..." - # sudo -- sh -c '/usr/bin/strip "${INSTALL_PREFIX}/bin/python3.9" && \ - # /usr/bin/strip "${INSTALL_PREFIX}/lib/libpython3.9.so.1.0"' - - # Install pip - sudo PYTHON_VERSION=${PYTHON_VERSION} INSTALL_PREFIX=${INSTALL_PREFIX} \ - PYTHON_TGZ=${PYTHON_TGZ} PYTHON_SRC_DIR=${PYTHON_SRC_DIR} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} \ - CPPFLAGS=${CPPFLAGS} LDFLAGS=${LDFLAGS} PATH=${PATH} \ - -- sh -c 'cd /tmp && curl -sS https://bootstrap.pypa.io/get-pip.py | "${INSTALL_PREFIX}/bin/python3.9"' - - export PYTHON39_PATH="${INSTALL_PREFIX}" - export pip_bin=${PYTHON39_PATH}/bin/pip3.9 - export VIRTUAL_ENV_VERSION="21.7.16" - # Pip modules install - sudo pip_bin=${pip_bin} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} -- sh -c '${pip_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable==0.0.1 mysqlclient==2.1.1' - sudo pip_bin=${pip_bin} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} PG_CONFIG=${PG_CONFIG} -- sh -c 'export PG_CONFIG="$PG_CONFIG"; ${pip_bin} install psycopg2==2.9.6' - sudo pip_bin=${pip_bin} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} -- sh -c 'ln -fs ${pip_bin} $(dirname ${pip_bin})/pip' - popd > /dev/null -} - function install_sqlite_python() { local DOCKEROS="${1:-ubuntu22}" local PYTHON_VERSION="${2:-3.11.12}" @@ -207,24 +162,13 @@ function install_sqlite_python() { pushd . > /dev/null - if [[ "$PYTHON_VERSION" == "$REQ_PYTHON311" ]]; then - if [[ -z ${PYTHON311_PATH+x} ]]; then - install_build_dependencies "$DOCKEROS" - install_sqlite3 "$SQLITE3_PATH" - export LD_LIBRARY_PATH="${INSTALL_PREFIX}/lib:${SQLITE3_PATH}/lib:${LD_LIBRARY_PATH}" - export CPPFLAGS="-I${SQLITE3_PATH}/include" - export LDFLAGS="-L${SQLITE3_PATH}/lib" - install_python311 "$PYTHON_VERSION" "$INSTALL_PREFIX" "$LD_LIBRARY_PATH" "$CPPFLAGS" "$LDFLAGS" - fi - elif [[ "$PYTHON_VERSION" == "$REQ_PYTHON39" ]]; then - if [[ -z ${PYTHON39_PATH+x} ]]; then - install_build_dependencies "$DOCKEROS" - install_sqlite3 "$SQLITE3_PATH" - export LD_LIBRARY_PATH="${INSTALL_PREFIX}/lib:${SQLITE3_PATH}/lib:${LD_LIBRARY_PATH}" - export CPPFLAGS="-I${SQLITE3_PATH}/include" - export LDFLAGS="-L${SQLITE3_PATH}/lib" - install_python39 "$PYTHON_VERSION" "$INSTALL_PREFIX" "$LD_LIBRARY_PATH" "$CPPFLAGS" "$LDFLAGS" - fi + if [[ -z ${PYTHON311_PATH+x} ]]; then + install_build_dependencies "$DOCKEROS" + install_sqlite3 "$SQLITE3_PATH" + export LD_LIBRARY_PATH="${INSTALL_PREFIX}/lib:${SQLITE3_PATH}/lib:${LD_LIBRARY_PATH}" + export CPPFLAGS="-I${SQLITE3_PATH}/include" + export LDFLAGS="-L${SQLITE3_PATH}/lib" + install_python311 "$PYTHON_VERSION" "$INSTALL_PREFIX" "$LD_LIBRARY_PATH" "$CPPFLAGS" "$LDFLAGS" fi popd > /dev/null @@ -266,9 +210,8 @@ function centos7_install() { sudo -- sh -c 'cd /tmp && curl -fsSL https://rpm.nodesource.com/setup_16.x | sudo bash - && \ yum install -y nodejs npm' - unset PYTHON39_PATH - if [[ -z ${PYTHON39_PATH+x} ]]; then - install_sqlite_python "$os" "$REQ_PYTHON39" + if [[ -z ${PYTHON311_PATH+x} ]]; then + install_sqlite_python "$os" "$REQ_PYTHON311" fi fi } @@ -308,9 +251,6 @@ function redhat8_install() { install_sqlite_python "$os" "$REQ_PYTHON311" fi - if [[ -z ${PYTHON39_PATH+x} ]]; then - install_sqlite_python "$os" "$REQ_PYTHON39" - fi fi } @@ -342,9 +282,6 @@ function redhat8_arm64_install() { install_sqlite_python "$os" "$REQ_PYTHON311" fi - if [[ -z ${PYTHON39_PATH+x} ]]; then - install_sqlite_python "$os" "$REQ_PYTHON39" - fi fi } @@ -382,9 +319,6 @@ function redhat9_install() { install_sqlite_python "$os" "$REQ_PYTHON311" fi - if [[ -z ${PYTHON39_PATH+x} ]]; then - install_sqlite_python "$os" "$REQ_PYTHON39" - fi fi } @@ -422,11 +356,6 @@ function redhat9_arm64_install() { if [[ -z ${PYTHON311_PATH+x} ]]; then install_sqlite_python "$os" "$REQ_PYTHON311" fi - - unset PYTHON39_PATH - if [[ -z ${PYTHON39_PATH+x} ]]; then - install_sqlite_python "$os" "$REQ_PYTHON39" - fi fi } @@ -462,9 +391,8 @@ function sles12_install() { # Node-v20-LTS is not supported by old OS'es - Redhat7_ppc, Centos7, Ubuntu18, Sles12. So upgrading to node-v16 sudo -- sh -c 'zypper install -y npm14 nodejs14' - unset PYTHON39_PATH - if [[ -z ${PYTHON39_PATH+x} ]]; then - install_sqlite_python "$os" "$REQ_PYTHON39" + if [[ -z ${PYTHON311_PATH+x} ]]; then + install_sqlite_python "$os" "$REQ_PYTHON311" fi fi } @@ -504,9 +432,6 @@ function sles15_install() { install_sqlite_python "$os" "$REQ_PYTHON311" fi - if [[ -z ${PYTHON39_PATH+x} ]]; then - install_sqlite_python "$os" "$REQ_PYTHON39" - fi fi } @@ -557,9 +482,6 @@ function ubuntu18_install() { install_sqlite_python "$os" "$REQ_PYTHON311" fi - if [[ -z ${PYTHON39_PATH+x} ]]; then - install_sqlite_python "$os" "$REQ_PYTHON39" - fi fi } @@ -606,9 +528,8 @@ function ubuntu20_install() { cd /tmp && curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash - && \ apt-get install -y nodejs' - unset PYTHON39_PATH - if [[ -z ${PYTHON39_PATH+x} ]]; then - install_sqlite_python "$os" "$REQ_PYTHON39" + if [[ -z ${PYTHON311_PATH+x} ]]; then + install_sqlite_python "$os" "$REQ_PYTHON311" fi fi } @@ -654,9 +575,6 @@ function ubuntu22_install() { install_sqlite_python "$os" "$REQ_PYTHON311" fi - if [[ -z ${PYTHON39_PATH+x} ]]; then - install_sqlite_python "$os" "$REQ_PYTHON39" - fi fi } diff --git a/tools/container/base/hue/Dockerfile b/tools/container/base/hue/Dockerfile index 943df54943a..65be4ae28d3 100644 --- a/tools/container/base/hue/Dockerfile +++ b/tools/container/base/hue/Dockerfile @@ -26,8 +26,8 @@ RUN set -eux; \ ncurses-devel \ nmap-ncat \ procps-ng \ - python38 \ - python38-devel \ + python3.11 \ + python3.11-devel \ rsync \ openldap-devel \ openssl \ @@ -53,8 +53,7 @@ RUN set -eux; \ rm -rf /var/cache/yum RUN set -eux; \ - /usr/bin/pip3.8 install supervisor \ - && curl -s https://files.pythonhosted.org/packages/45/78/4621eb7085162bc4d2252ad92af1cc5ccacbd417a50e2ee74426331aad18/psycopg2_binary-2.9.3-cp38-cp38-musllinux_1_1_x86_64.whl -o /tmp/psycopg2_binary-2.9.3-cp38-cp38-musllinux_1_1_x86_64.whl \ + python3.11 -m pip install supervisor psycopg2-binary \ && dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm \ && yum install -y postgresql15 \ && curl -sL https://rpm.nodesource.com/setup_14.x | bash - \ diff --git a/tools/container/build.sh b/tools/container/build.sh index 823013cbf73..56a2efa5dfb 100755 --- a/tools/container/build.sh +++ b/tools/container/build.sh @@ -63,7 +63,7 @@ docker_hue_build() { cp -a $BUILD_DIR/${HUEUSER} $HUE_DIR rm -f $HUE_DIR/${HUEUSER}/desktop/conf/* - # Include tools/scripts for multi-python wrapper and helpers + # Include tools/scripts for Hue CLI wrappers and helpers mkdir -p $HUE_DIR/${HUEUSER}/tools/scripts cp -a $HUE_SRC/tools/scripts/* $HUE_DIR/${HUEUSER}/tools/scripts/ chmod +x $HUE_DIR/${HUEUSER}/tools/scripts/*.sh 2>/dev/null || true diff --git a/tools/container/compile/hue/Dockerfile b/tools/container/compile/hue/Dockerfile index bf92438571d..aba309550a6 100644 --- a/tools/container/compile/hue/Dockerfile +++ b/tools/container/compile/hue/Dockerfile @@ -4,7 +4,7 @@ LABEL description="Hue Project https://github.com/cloudera/hue" # Set the environment variable ENV NAME="basehue" -ENV PYTHON_VER=python3.8 +ENV PYTHON_VER=python3.11 # Required for building Hue RUN set -eux; \ @@ -34,8 +34,8 @@ RUN set -eux; \ openldap-devel \ openssl \ openssl-devel \ - python38 \ - python38-devel \ + python3.11 \ + python3.11-devel \ rsync \ sqlite-devel \ sudo \ diff --git a/tools/container/hue/Dockerfile b/tools/container/hue/Dockerfile index a0ec6b329f3..da1db2d80b5 100644 --- a/tools/container/hue/Dockerfile +++ b/tools/container/hue/Dockerfile @@ -40,8 +40,8 @@ RUN set -eux; \ python3-dev \ build-base \ postgresql-dev && \ - $HUE_BIN/python3.8 -m pip install --upgrade pip --force && \ - $HUE_BIN/pip install psycopg2 --no-cache-dir + $HUE_BIN/python3.11 -m pip install --upgrade pip --force && \ + $HUE_BIN/pip install psycopg2-binary --no-cache-dir # Setup directory structure RUN mkdir -p ${HUE_LOG_DIR} && chown -R ${HUEUSER}:${HUEUSER} ${HUE_LOG_DIR} @@ -67,7 +67,7 @@ ADD supervisor-files/hue_redis.conf /etc/supervisor.d/ RUN chown -R ${HUEUSER}:${HUEUSER} /etc/supervisord.conf && chown -R ${HUEUSER}:${HUEUSER} /etc/supervisor.d # Remove chardet package -RUN rm -rf ${HUE_HOME}/build/env/lib/python3.8/site-packages/pip/_vendor/chardet +RUN rm -rf ${HUE_HOME}/build/env/lib/python3.11/site-packages/pip/_vendor/chardet EXPOSE 8888 9111 6379 diff --git a/tools/docker/hue/Dockerfile b/tools/docker/hue/Dockerfile index b37df36f717..e97f7f70b5b 100644 --- a/tools/docker/hue/Dockerfile +++ b/tools/docker/hue/Dockerfile @@ -57,7 +57,7 @@ RUN pip3 install --upgrade --no-cache-dir setuptools virtualenv pip && \ rm -rf /var/lib/apt/lists/* && \ cp -r /hue/tools/docker/hue/conf3/* /usr/share/hue/desktop/conf/ && \ cp /hue/tools/docker/hue/startup.sh /usr/share/hue/startup.sh && \ - # Include tools/scripts for multi-python wrapper and helpers + # Include tools/scripts for Hue CLI wrappers and helpers mkdir -p /usr/share/hue/tools/scripts && \ cp -r /hue/tools/scripts/* /usr/share/hue/tools/scripts/ && \ chmod +x /usr/share/hue/tools/scripts/*.sh || true && \ diff --git a/tools/relocatable.sh b/tools/relocatable.sh index 59b1ccc48e1..ab8d0088a21 100644 --- a/tools/relocatable.sh +++ b/tools/relocatable.sh @@ -24,7 +24,7 @@ usage() { Usage: ./tools/relocatable.sh [python_version] Example: - ./tools/relocatable.sh python3.9 + ./tools/relocatable.sh python3.11 " exit 1 } @@ -95,8 +95,8 @@ export PATH=$(dirname $ENV_PYTHON):$PATH PYVER=$($ENV_PYTHON -V 2>&1 | awk '{print $2}' | cut -d '.' -f 1,2) BIN_DIR=$(dirname $SYS_PYTHON) # Step 1. Fix virtualenv -if [[ "$PYVER" == "3."[8-9]* || "$PYVER" == "3.11" ]]; then - echo "Python version is 3.8 or greater" +if [[ "$PYVER" == "3.11" ]]; then + echo "Python version is 3.11" pushd . cd $HUE_ROOT $SYS_PYTHON $BIN_DIR/virtualenv-make-relocatable "$BLD_DIR_ENV" diff --git a/tools/scripts/hue.sh b/tools/scripts/hue.sh index 22f7cd146b9..6215509aef0 100755 --- a/tools/scripts/hue.sh +++ b/tools/scripts/hue.sh @@ -15,8 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# wrapper script to identify the latest python and pass the execution -# to the corresponding hue script in the `build` directory. +# Wrapper script that runs Hue using the Python 3.11 virtualenv at build/env/bin. set -ex diff --git a/tools/scripts/python/python_helper.sh b/tools/scripts/python/python_helper.sh index 4af47fdb917..5a44a00faac 100755 --- a/tools/scripts/python/python_helper.sh +++ b/tools/scripts/python/python_helper.sh @@ -15,93 +15,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Some helper functions to find the latest python bin and venv path +# Resolve Hue's Python 3.11 virtualenv bin directory (build/env/bin). set -ex -# Find the latest Python binary in build/env/bin -LATEST_PYTHON=$([ -d "$HUE_HOME_DIR/build/env/bin" ] && find "$HUE_HOME_DIR/build/env/bin" -name "python3*" -exec basename {} \; | sort -V | tail -n 1 || echo "") +HUE_SUPPORTED_PYTHON_VERSION="3.11" -# Extract version from the latest python binary (e.g., python3.11 → 3.11) -LATEST_PYTHON_VERSION=$(echo "$LATEST_PYTHON" | grep -oE '[0-9]+\.[0-9]+' || echo "") +if [ -n "$HUE_PYTHON_VERSION" ]; then + export SELECTED_PYTHON_VERSION=$(echo "$HUE_PYTHON_VERSION" | grep -oE '[0-9]+\.[0-9]+') +else + export SELECTED_PYTHON_VERSION="$HUE_SUPPORTED_PYTHON_VERSION" +fi -# Find all supported python versions from build/venvs and include latest version -readarray -t SUPPORTED_VERSIONS < <( - ( - [ -d "$HUE_HOME_DIR/build/venvs" ] && find "$HUE_HOME_DIR/build/venvs" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | grep -oE '[0-9]+\.[0-9]+' - echo "$LATEST_PYTHON_VERSION" - ) | sort -Vr | uniq -) +if [ "$SELECTED_PYTHON_VERSION" != "$HUE_SUPPORTED_PYTHON_VERSION" ]; then + echo "Hue requires Python ${HUE_SUPPORTED_PYTHON_VERSION}; got ${SELECTED_PYTHON_VERSION} from HUE_PYTHON_VERSION." >&2 + exit 1 +fi -# Create array of supported versions -SUPPORTED_PYTHON_VERSIONS=("${SUPPORTED_VERSIONS[@]}") - -_python_bin_path() { - # Searches for the provided python version in various locations on the node - PYTHON_VERSION="$1" - PYTHON_VERSION_NO_DOT="${PYTHON_VERSION//./}" - - # First check in PATH if it exists - if [ -n "$PATH" ]; then - IFS=: read -ra PATH_DIRS <<< "$PATH" - for DIR in "${PATH_DIRS[@]}"; do - # Skip cm-agent/bin directories - if [[ "$DIR" == *"cm-agent/bin"* ]]; then - continue - fi - PY_PATH="$DIR/python$PYTHON_VERSION" - if [[ -x "$PY_PATH" ]]; then - echo "$PY_PATH" - return 0 - fi - done - fi - - # Fall back to hardcoded directories - SEARCH_DIRS=("/usr/local/bin" "/bin" "/usr/bin" "/opt/rh/rh-${PYTHON_VERSION_NO_DOT}/root/usr/bin") - for DIR in "${SEARCH_DIRS[@]}"; do - PY_PATH="$DIR/python$PYTHON_VERSION" - if [[ -x "$PY_PATH" ]]; then - echo "$PY_PATH" - return 0 - fi - done - return 1 -} - -_choose_python_version() { - # returns the latest py version, e.g., 3.11, 3.9 or 3.8 - # if HUE_PYTHON_VERSION is set, use it - if [ -n "$HUE_PYTHON_VERSION" ]; then - echo "$HUE_PYTHON_VERSION" | grep -oE '[0-9]+\.[0-9]+' - return 0 - else - for PYTHON_VERSION in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do - if _python_bin_path "$PYTHON_VERSION" > /dev/null; then - echo "$PYTHON_VERSION" - return 0 - fi - done - fi - - echo "No supported Python versions found in expected locations." - return 1 -} - -_venv_path() { - # returns the path to the env/bin of the latest python, - # relative to the top-level hue directory. - local version="$1" - - if [ -z "$version" ]; then - return 1 # Return error if no version provided/found - fi - - if [ "$version" = "$LATEST_PYTHON_VERSION" ]; then - echo "build/env/bin" - else - echo "build/venvs/python${version}/bin" - fi -} - -export SELECTED_PYTHON_VERSION="$(_choose_python_version)" -export VENV_BIN_PATH=${HUE_HOME_DIR}/$(_venv_path "$SELECTED_PYTHON_VERSION") \ No newline at end of file +export VENV_BIN_PATH="${HUE_HOME_DIR}/build/env/bin" From 2da71d905625a2d47e30a79c1820154cb9034043 Mon Sep 17 00:00:00 2001 From: Harsh Gupta <42064744+Harshg999@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:41:24 +0530 Subject: [PATCH 5/6] [desktop] Restrict check_config API to Hue admins (ISEC-V2026-04) GET /api/v1/check_config was available to any authenticated user. It returns the Hue configuration directory path and the configuration validation errors, which expose server paths and internal service topology. The legacy /desktop/debug/check_config view is already admin-only. - Gate check_config with deny_unless_hue_admin and return HTTP 403 with the shared "You must be a Hue admin" message for non-admins. The check uses the same is_admin helper as the legacy view. - Switch available_app_examples and install_app_examples to the same helper. Their behavior and message are unchanged. - Add TestCheckConfigPublicApi (anonymous, non-admin, admin), admin and non-admin unit tests for check_config, and unit tests for deny_unless_hue_admin. - Point the existing api2 admin tests at the shared is_admin patch target. Fix the unused imports and the undefined SkipTest in api_public_tests.py that ruff flagged once the file was touched. --- desktop/core/src/desktop/api2.py | 23 +++++-- desktop/core/src/desktop/api2_tests.py | 65 ++++++++++++------- desktop/core/src/desktop/api_public_tests.py | 37 ++++++++++- .../desktop/configuration/authorization.py | 38 ++++++++--- .../core/src/desktop/configuration/tests.py | 18 ++++- 5 files changed, 136 insertions(+), 45 deletions(-) diff --git a/desktop/core/src/desktop/api2.py b/desktop/core/src/desktop/api2.py index 52ea1e82a5b..ade081c468e 100644 --- a/desktop/core/src/desktop/api2.py +++ b/desktop/core/src/desktop/api2.py @@ -53,6 +53,7 @@ IMPORTER, TASK_SERVER_V2, ) +from desktop.configuration.authorization import deny_unless_hue_admin from desktop.lib.conf import BoundContainer, GLOBAL_CONFIG, is_anonymous from desktop.lib.connectors.models import Connector from desktop.lib.django_util import JsonResponse, login_notrequired, render @@ -1463,9 +1464,9 @@ def _paginate(request, queryset): @api_error_handler def available_app_examples(request): """Returns a dictionary of available apps whose examples can be installed by the admin user.""" - - if not is_admin(request.user): - return JsonResponse({'message': "You must be a Hue admin to access this endpoint."}, status=403) + denied = deny_unless_hue_admin(request) + if denied: + return denied supported_app_examples = {'hive', 'impala', 'hbase', 'pig', 'oozie', 'notebook', 'search', 'spark', 'jobsub'} @@ -1481,8 +1482,9 @@ def install_app_examples(request): if not app_name: return JsonResponse({'message': "Missing parameter: app_name is required."}, status=400) - if not is_admin(request.user): - return JsonResponse({'message': "You must be a Hue admin to access this endpoint."}, status=403) + denied = deny_unless_hue_admin(request) + if denied: + return denied # Define supported apps and their setup functions setup_functions = { @@ -1563,7 +1565,16 @@ def _setup_search_examples(request): @api_error_handler def check_config(request): - """Returns the configuration directory and the list of validation errors.""" + """ + Return the Hue configuration directory and configuration validation errors. + + Hue admins only; non-admins must not receive server paths or internal service topology + (see ``/desktop/debug/check_config``, which uses the same ``is_admin`` gate). + """ + denied = deny_unless_hue_admin(request) + if denied: + return denied + response = { 'hue_config_dir': os.path.realpath(os.getenv("HUE_CONF_DIR", get_desktop_root("conf"))), 'config_errors': _get_config_errors(request, cache=False), diff --git a/desktop/core/src/desktop/api2_tests.py b/desktop/core/src/desktop/api2_tests.py index bdf63652a77..58e570f52b4 100644 --- a/desktop/core/src/desktop/api2_tests.py +++ b/desktop/core/src/desktop/api2_tests.py @@ -816,7 +816,7 @@ class MockApp: nice_name: str def test_available_app_examples_success(self): - with patch("desktop.api2.is_admin") as mock_is_admin: + with patch("desktop.configuration.authorization.is_admin") as mock_is_admin: with patch("desktop.api2.appmanager.get_apps") as mock_get_apps: request = Mock(method="GET", user=Mock()) @@ -833,7 +833,7 @@ def test_available_app_examples_success(self): assert json.loads(response.content) == {"apps": {"hive": "Hive", "impala": "Impala"}} def test_available_app_examples_non_admin(self): - with patch("desktop.api2.is_admin") as mock_is_admin: + with patch("desktop.configuration.authorization.is_admin") as mock_is_admin: request = Mock(method="GET", user=Mock()) mock_is_admin.return_value = False @@ -843,7 +843,7 @@ def test_available_app_examples_non_admin(self): assert json.loads(response.content) == {"message": "You must be a Hue admin to access this endpoint."} def test_available_app_examples_no_apps(self): - with patch("desktop.api2.is_admin") as mock_is_admin: + with patch("desktop.configuration.authorization.is_admin") as mock_is_admin: with patch("desktop.api2.appmanager.get_apps") as mock_get_apps: mock_is_admin.return_value = True mock_get_apps.return_value = [] @@ -856,7 +856,7 @@ def test_available_app_examples_no_apps(self): assert json.loads(response.content) == {"apps": {}} def test_available_app_examples_only_unsupported_apps(self): - with patch("desktop.api2.is_admin") as mock_is_admin: + with patch("desktop.configuration.authorization.is_admin") as mock_is_admin: with patch("desktop.api2.appmanager.get_apps") as mock_get_apps: mock_is_admin.return_value = True @@ -889,7 +889,7 @@ def test_install_app_examples_unsupported_app_name(self): assert json.loads(response.content.decode("utf-8"))["message"] == "Unsupported app name: test_app" def test_install_app_examples_non_admin_user(self): - with patch("desktop.api2.is_admin") as mock_is_admin: + with patch("desktop.configuration.authorization.is_admin") as mock_is_admin: mock_is_admin.return_value = False request = Mock(method="POST", POST={"app_name": "hive"}, user=Mock()) @@ -899,7 +899,7 @@ def test_install_app_examples_non_admin_user(self): assert json.loads(response.content.decode("utf-8"))["message"] == "You must be a Hue admin to access this endpoint." def test_install_app_examples_success_hive(self): - with patch("desktop.api2.is_admin") as mock_is_admin: + with patch("desktop.configuration.authorization.is_admin") as mock_is_admin: with patch("desktop.api2._setup_hive_impala_examples") as mock_setup_hive_impala_examples: mock_is_admin.return_value = True mock_setup_hive_impala_examples.return_value = None @@ -911,7 +911,7 @@ def test_install_app_examples_success_hive(self): assert json.loads(response.content.decode("utf-8"))["message"] == "Successfully installed examples for hive." def test_install_app_examples_success_impala(self): - with patch("desktop.api2.is_admin") as mock_is_admin: + with patch("desktop.configuration.authorization.is_admin") as mock_is_admin: with patch("desktop.api2._setup_hive_impala_examples") as mock_setup_hive_impala_examples: mock_is_admin.return_value = True mock_setup_hive_impala_examples.return_value = None @@ -1001,25 +1001,40 @@ def test_setup_search_examples_without_log_analytics_demo(self): class TestCheckConfigAPI: def test_check_config_success(self): - with patch("desktop.api2.os.path.realpath") as mock_hue_conf_dir: + with patch("desktop.configuration.authorization.is_admin") as mock_is_admin: + with patch("desktop.api2.os.path.realpath") as mock_hue_conf_dir: + with patch("desktop.api2._get_config_errors") as mock_get_config_errors: + mock_is_admin.return_value = True + request = Mock(method="GET", user=Mock()) + mock_hue_conf_dir.return_value = "/test/hue/conf" + mock_get_config_errors.return_value = [ + {"name": "Hive", "message": "The application won't work without a running HiveServer2."}, + {"name": "Impala", "message": "No available Impalad to send queries to."}, + {"name": "Spark", "message": "The app won't work without a running Livy Spark Server"}, + ] + + response = check_config(request) + response_data = json.loads(response.content) + + assert response.status_code == 200 + assert response_data == { + "hue_config_dir": "/test/hue/conf", + "config_errors": [ + {"name": "Hive", "message": "The application won't work without a running HiveServer2."}, + {"name": "Impala", "message": "No available Impalad to send queries to."}, + {"name": "Spark", "message": "The app won't work without a running Livy Spark Server"}, + ], + } + mock_get_config_errors.assert_called_once() + + def test_check_config_non_admin(self): + with patch("desktop.configuration.authorization.is_admin") as mock_is_admin: with patch("desktop.api2._get_config_errors") as mock_get_config_errors: - request = Mock(method="GET") - mock_hue_conf_dir.return_value = "/test/hue/conf" - mock_get_config_errors.return_value = [ - {"name": "Hive", "message": "The application won't work without a running HiveServer2."}, - {"name": "Impala", "message": "No available Impalad to send queries to."}, - {"name": "Spark", "message": "The app won't work without a running Livy Spark Server"}, - ] + mock_is_admin.return_value = False + request = Mock(method="GET", user=Mock()) response = check_config(request) - response_data = json.loads(response.content) - assert response.status_code == 200 - assert response_data == { - "hue_config_dir": "/test/hue/conf", - "config_errors": [ - {"name": "Hive", "message": "The application won't work without a running HiveServer2."}, - {"name": "Impala", "message": "No available Impalad to send queries to."}, - {"name": "Spark", "message": "The app won't work without a running Livy Spark Server"}, - ], - } + assert response.status_code == 403 + assert json.loads(response.content) == {"message": "You must be a Hue admin to access this endpoint."} + mock_get_config_errors.assert_not_called() diff --git a/desktop/core/src/desktop/api_public_tests.py b/desktop/core/src/desktop/api_public_tests.py index 79b641700ed..8bf746ecb2e 100644 --- a/desktop/core/src/desktop/api_public_tests.py +++ b/desktop/core/src/desktop/api_public_tests.py @@ -16,9 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import json -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import Mock, patch import pytest from django.http import HttpResponse @@ -26,7 +25,9 @@ from desktop.api_public import get_django_request from desktop.conf import CUSTOM +from desktop.configuration.authorization import HUE_ADMIN_REQUIRED_API_MESSAGE from desktop.lib.django_test_util import Client, make_logged_in_client +from desktop.lib.test_utils import grant_access from useradmin.models import User @@ -53,6 +54,36 @@ def test_banners(self): done() +@pytest.mark.django_db +class TestCheckConfigPublicApi(object): + def setup_method(self): + self.unauthorized_client = Client() + self.admin_client = make_logged_in_client(username='check_config_admin', recreate=True, is_superuser=True) + self.user_client = make_logged_in_client(username='check_config_user', recreate=True, is_superuser=False) + grant_access('check_config_admin', 'check_config_admin', 'desktop') + grant_access('check_config_user', 'check_config_user', 'desktop') + + def test_check_config_requires_authentication(self): + response = self.unauthorized_client.get(reverse('api:core_check_config')) + assert response.status_code in (401, 403) + + def test_check_config_denied_for_non_admin(self): + with patch('desktop.api2._get_config_errors') as mock_get_config_errors: + response = self.user_client.get(reverse('api:core_check_config')) + + assert response.status_code == 403 + assert json.loads(response.content) == {'message': HUE_ADMIN_REQUIRED_API_MESSAGE} + mock_get_config_errors.assert_not_called() + + def test_check_config_allowed_for_admin(self): + with patch('desktop.api2._get_config_errors', return_value=[]): + with patch('desktop.api2.os.path.realpath', return_value='/hue/conf'): + response = self.admin_client.get(reverse('api:core_check_config')) + + assert response.status_code == 200 + assert json.loads(response.content) == {'hue_config_dir': '/hue/conf', 'config_errors': []} + + @pytest.mark.django_db class TestEditorApi(): TEST_INTERPRETER = { @@ -86,7 +117,7 @@ def test_editor_execute(self): execute.assert_called() if not execute.call_args.args[1]: - raise SkipTest() # Incorrect in Py3 CircleCi + pytest.skip('Incorrect in Py3 CircleCi') assert execute.call_args.args[1] == 'hive' json.loads(execute.call_args.args[0].POST['notebook']) json.loads(execute.call_args.args[0].POST['snippet']) diff --git a/desktop/core/src/desktop/configuration/authorization.py b/desktop/core/src/desktop/configuration/authorization.py index 42d362425f5..75e57a6c2cb 100644 --- a/desktop/core/src/desktop/configuration/authorization.py +++ b/desktop/core/src/desktop/configuration/authorization.py @@ -16,18 +16,38 @@ # limitations under the License. """ -Authorization for DefaultConfiguration REST endpoints. +Authorization for configuration-related REST and diagnostics APIs. -Non-admin users may only read and write their own user-specific configuration rows. -Default and group-wide configuration management requires a Hue admin (``is_admin``). +DefaultConfiguration endpoints: non-admins may only access their own user-specific rows; +default and group-wide configuration requires a Hue admin (``is_admin``). + +Admin-only diagnostics (e.g. ``check_config``) use :func:`deny_unless_hue_admin` for HTTP 403 +responses consistent with other public API v1 admin endpoints. """ from django.contrib.auth.models import User from django.utils.translation import gettext as _ from desktop.auth.backend import is_admin +from desktop.lib.django_util import JsonResponse from desktop.lib.exceptions_renderable import PopupException +# Shared with ``available_app_examples``, ``install_app_examples``, and ``check_config``. +HUE_ADMIN_REQUIRED_API_MESSAGE = "You must be a Hue admin to access this endpoint." + + +def deny_unless_hue_admin(request): + """ + Guard for JSON handlers exposed on the public API (DRF) that must match legacy admin views. + + Uses the same ``is_admin`` check as ``@admin_required`` on ``/desktop/debug/check_config``. + + :return: ``JsonResponse`` with HTTP 403 when the caller is not a Hue admin, otherwise ``None``. + """ + if not is_admin(request.user): + return JsonResponse({"message": HUE_ADMIN_REQUIRED_API_MESSAGE}, status=403) + return None + def require_configuration_admin(user): """ @@ -48,23 +68,23 @@ def resolve_target_user(request, user_id): :return: ``User`` instance. :raises PopupException: Invalid id, unknown user, or cross-user access without admin rights. """ - if user_id is None or str(user_id).strip() == '': - raise PopupException(_('app_configuration_for_user requires app and user_id')) + if user_id is None or str(user_id).strip() == "": + raise PopupException(_("app_configuration_for_user requires app and user_id")) try: requested_id = int(user_id) except (TypeError, ValueError): - raise PopupException(_('Invalid user_id.')) + raise PopupException(_("Invalid user_id.")) if is_admin(request.user): try: return User.objects.get(id=requested_id) except User.DoesNotExist: - raise PopupException(_('Could not find user with User ID: %s') % requested_id) + raise PopupException(_("Could not find user with User ID: %s") % requested_id) if requested_id != request.user.id: raise PopupException( - _('You do not have permission to access configuration for this user.'), + _("You do not have permission to access configuration for this user."), error_code=403, ) @@ -87,7 +107,7 @@ def assert_can_delete_configuration(request, config): if config.user_id is not None: if config.user_id != request.user.id and not is_admin(request.user): raise PopupException( - _('You do not have permission to delete this configuration.'), + _("You do not have permission to delete this configuration."), error_code=403, ) return diff --git a/desktop/core/src/desktop/configuration/tests.py b/desktop/core/src/desktop/configuration/tests.py index 83e58804808..a7f5709b576 100644 --- a/desktop/core/src/desktop/configuration/tests.py +++ b/desktop/core/src/desktop/configuration/tests.py @@ -18,9 +18,11 @@ import json from builtins import object +from unittest.mock import Mock, patch import pytest +from desktop.configuration.authorization import deny_unless_hue_admin, HUE_ADMIN_REQUIRED_API_MESSAGE from desktop.lib.django_test_util import make_logged_in_client from desktop.lib.test_utils import grant_access from desktop.models import DefaultConfiguration @@ -250,8 +252,10 @@ def test_get_default_configurations(self): @pytest.mark.django_db class TestConfigurationAuthorization(object): """ - Regression tests for ISEC-V2026-03: DefaultConfiguration API must not allow IDOR on user_id or - unprivileged access to global default/group configuration. + Regression tests for configuration authorization (ISEC-V2026-03, ISEC-V2026-04). + + DefaultConfiguration API must not allow IDOR on user_id or unprivileged access to global + default/group configuration. ``deny_unless_hue_admin`` must gate admin-only diagnostics APIs. """ def setup_method(self): @@ -384,3 +388,13 @@ def test_admin_can_delete_any_configuration(self): content = json.loads(response.content) assert content['status'] == 0, content assert not DefaultConfiguration.objects.filter(id=user_config.id).exists() + + def test_deny_unless_hue_admin_allows_admin(self): + with patch('desktop.configuration.authorization.is_admin', return_value=True): + assert deny_unless_hue_admin(Mock(user=Mock())) is None + + def test_deny_unless_hue_admin_blocks_non_admin(self): + with patch('desktop.configuration.authorization.is_admin', return_value=False): + response = deny_unless_hue_admin(Mock(user=Mock())) + assert response.status_code == 403 + assert json.loads(response.content)['message'] == HUE_ADMIN_REQUIRED_API_MESSAGE From 003aa8c54337edcc7924f221c6bddd1e473268e2 Mon Sep 17 00:00:00 2001 From: Harsh Gupta <42064744+Harshg999@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:29:04 +0530 Subject: [PATCH 6/6] [indexer] Enforce write access on importer save (ISEC-V2026-05) POST /indexer/api/importer/save (save_pipeline) had its permission decorator commented out, and _save_pipeline loaded the target with a raw Document2.objects.get(id=...) and no write check. Any authenticated user could overwrite another user's saved document by id. - Add notebook/saved_document_auth.py with get_writable_document_for_update(). It resolves the existing document by parentSavedQueryUuid (write permission required) or by id (can_write_or_exception), and returns None when a new document should be created. A missing document and a denied write raise the same message so document ids cannot be probed. Admins keep write access to other users' documents, as before. - Re-enable @check_document_modify_permission() on save_pipeline. - Make check_document_modify_permission, _save_pipeline and _save_notebook use the helper, so authorization still holds if a view decorator is missing or incomplete. - check_document_modify_permission now also checks parentSavedQueryUuid when no id is sent, and a read-only share can no longer be saved over. - Add tests: TestSavedDocumentAuth, TestSavePipeline (model and HTTP, cross-user denied, owner allowed) and test_save_notebook_denies_cross_user_update. Housekeeping needed to pass ruff on the touched files: replace the fetch_handle lambda in indexer api3.py with a nested function, drop unused exception bindings in notebook/decorators.py and api.py, and remove test_save_notebook_with_connector_on, which was skipped unless connectors were enabled and referenced an undefined variable. --- desktop/libs/indexer/src/indexer/api3.py | 17 +-- desktop/libs/indexer/src/indexer/models.py | 20 ++- .../src/indexer/save_pipeline_tests.py | 141 ++++++++++++++++++ desktop/libs/notebook/src/notebook/api.py | 22 ++- .../libs/notebook/src/notebook/api_tests.py | 43 ++---- .../libs/notebook/src/notebook/decorators.py | 22 +-- .../src/notebook/saved_document_auth.py | 67 +++++++++ .../src/notebook/saved_document_auth_tests.py | 69 +++++++++ 8 files changed, 328 insertions(+), 73 deletions(-) create mode 100644 desktop/libs/indexer/src/indexer/save_pipeline_tests.py create mode 100644 desktop/libs/notebook/src/notebook/saved_document_auth.py create mode 100644 desktop/libs/notebook/src/notebook/saved_document_auth_tests.py diff --git a/desktop/libs/indexer/src/indexer/api3.py b/desktop/libs/indexer/src/indexer/api3.py index 8ca922d9e6a..9d8c420119b 100644 --- a/desktop/libs/indexer/src/indexer/api3.py +++ b/desktop/libs/indexer/src/indexer/api3.py @@ -50,7 +50,7 @@ from indexer.utils import validate_local_upload_path from kafka.kafka_api import get_topic_data, get_topics from notebook.connectors.base import get_api, Notebook -from notebook.decorators import api_error_handler +from notebook.decorators import api_error_handler, check_document_modify_permission from notebook.models import escape_rows, MockedDjangoRequest LOG = logging.getLogger() @@ -627,14 +627,10 @@ def _small_indexing(user, fs, client, source, destination, index_name): searcher = CollectionManagerController(user) columns = [field['name'] for field in fields if field['name'] != 'hue_id'] # Assumes handle still live - fetch_handle = lambda rows, start_over: get_api( - request, snippet - ).fetch_result( - notebook, - snippet, - rows=rows, - start_over=start_over - ) + + def fetch_handle(rows, start_over): + return get_api(request, snippet).fetch_result(notebook, snippet, rows=rows, start_over=start_over) + searcher.update_data_from_hive( index_name, columns, @@ -719,8 +715,9 @@ def _large_indexing(request, file_format, collection_name, query=None, start_tim @api_error_handler @require_POST -# @check_document_modify_permission() +@check_document_modify_permission() def save_pipeline(request): + """Persist importer pipeline document state; requires write access to the target document.""" response = {'status': -1} notebook = json.loads(request.POST.get('notebook', '{}')) diff --git a/desktop/libs/indexer/src/indexer/models.py b/desktop/libs/indexer/src/indexer/models.py index 37bbf771fcd..3b4776e9bdc 100644 --- a/desktop/libs/indexer/src/indexer/models.py +++ b/desktop/libs/indexer/src/indexer/models.py @@ -15,20 +15,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -from desktop.models import Document2, Document +from desktop.models import Document, Document2 +from notebook.saved_document_auth import get_writable_document_for_update def _save_pipeline(pipeline, user): pipeline_type = pipeline.get('type', 'pipeline') save_as = False - if pipeline.get('parentSavedQueryUuid'): # We save into the original saved query, not into the query history - pipeline_doc = Document2.objects.get_by_uuid(user=user, uuid=pipeline['parentSavedQueryUuid']) - elif pipeline.get('id'): - pipeline_doc = Document2.objects.get(id=pipeline['id']) - else: + pipeline_doc = get_writable_document_for_update(pipeline, user) + if pipeline_doc is None: pipeline_doc = Document2.objects.create(name=pipeline['name'], uuid=pipeline['uuid'], type=pipeline_type, owner=user) - Document.objects.link(pipeline_doc, owner=pipeline_doc.owner, name=pipeline_doc.name, description=pipeline_doc.description, extra=pipeline_type) + Document.objects.link( + pipeline_doc, + owner=pipeline_doc.owner, + name=pipeline_doc.name, + description=pipeline_doc.description, + extra=pipeline_type, + ) save_as = True if pipeline.get('directoryUuid'): @@ -41,7 +45,7 @@ def _save_pipeline(pipeline, user): pipeline['id'] = pipeline_doc.id pipeline_doc1 = pipeline_doc._get_doc1(doc2_type=pipeline_type) pipeline_doc.update_data(pipeline) - #pipeline_doc.search = _get_statement(pipeline) + # pipeline_doc.search = _get_statement(pipeline) pipeline_doc.name = pipeline_doc1.name = pipeline['name'] pipeline_doc.description = pipeline_doc1.description = pipeline['description'] pipeline_doc.save() diff --git a/desktop/libs/indexer/src/indexer/save_pipeline_tests.py b/desktop/libs/indexer/src/indexer/save_pipeline_tests.py new file mode 100644 index 00000000000..9ab0f06eb53 --- /dev/null +++ b/desktop/libs/indexer/src/indexer/save_pipeline_tests.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python +# Licensed to Cloudera, Inc. under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. Cloudera, Inc. licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import pytest + +from desktop.lib.django_test_util import make_logged_in_client +from desktop.lib.exceptions_renderable import PopupException +from desktop.lib.test_utils import grant_access +from desktop.models import Document, Document2 +from indexer.models import _save_pipeline +from useradmin.models import User + + +@pytest.mark.django_db +class TestSavePipeline(object): + def setup_method(self): + self.owner_client = make_logged_in_client(username="pipeline_owner", recreate=True, is_superuser=False) + self.attacker_client = make_logged_in_client(username="pipeline_attacker", recreate=True, is_superuser=False) + grant_access("pipeline_owner", "pipeline_owner", "desktop") + grant_access("pipeline_attacker", "pipeline_attacker", "desktop") + self.owner = User.objects.get(username="pipeline_owner") + self.attacker = User.objects.get(username="pipeline_attacker") + + def _minimal_pipeline(self, **overrides): + payload = { + "name": "Importer pipeline", + "description": "Original description", + "type": "pipeline", + "uuid": "pipeline-test-uuid", + "isSaved": False, + } + payload.update(overrides) + return payload + + def test_save_pipeline_denies_cross_user_overwrite_via_model(self): + victim_doc = Document2.objects.create( + name="Victim pipeline", + uuid="victim-pipeline-uuid", + type="pipeline", + owner=self.owner, + description="trusted content", + ) + Document.objects.link(victim_doc, owner=self.owner, name=victim_doc.name, description=victim_doc.description, extra="pipeline") + victim_doc.update_data({"steps": []}) + victim_doc.save() + + attacker_payload = self._minimal_pipeline( + id=victim_doc.id, + name="Malicious pipeline", + description="attacker controlled", + ) + + with pytest.raises(PopupException): + _save_pipeline(attacker_payload, self.attacker) + + victim_doc.refresh_from_db() + assert victim_doc.owner_id == self.owner.id + assert victim_doc.name == "Victim pipeline" + assert victim_doc.description == "trusted content" + + def test_save_pipeline_allows_owner_update_via_model(self): + doc = Document2.objects.create( + name="My pipeline", + uuid="owner-pipeline-uuid", + type="pipeline", + owner=self.owner, + description="before", + ) + Document.objects.link(doc, owner=self.owner, name=doc.name, description=doc.description, extra="pipeline") + doc.update_data({"steps": []}) + doc.save() + + payload = self._minimal_pipeline(id=doc.id, name="Updated pipeline", description="after") + updated_doc, save_as = _save_pipeline(payload, self.owner) + + assert not save_as + assert updated_doc.id == doc.id + assert updated_doc.name == "Updated pipeline" + assert updated_doc.description == "after" + + def test_save_pipeline_http_denies_cross_user(self): + victim_doc = Document2.objects.create( + name="HTTP victim", + uuid="http-victim-uuid", + type="pipeline", + owner=self.owner, + ) + Document.objects.link(victim_doc, owner=self.owner, name=victim_doc.name, description="", extra="pipeline") + victim_doc.update_data({}) + victim_doc.save() + + notebook_json = json.dumps(self._minimal_pipeline(id=victim_doc.id, name="HTTP attack")) + response = self.attacker_client.post("/indexer/api/importer/save", {"notebook": notebook_json}) + data = json.loads(response.content) + + assert data["status"] == -1 + victim_doc.refresh_from_db() + assert victim_doc.name == "HTTP victim" + + def test_save_pipeline_http_allows_owner(self): + doc = Document2.objects.create( + name="HTTP owner doc", + uuid="http-owner-uuid", + type="pipeline", + owner=self.owner, + ) + Document.objects.link(doc, owner=self.owner, name=doc.name, description="", extra="pipeline") + doc.update_data({}) + doc.save() + + notebook_json = json.dumps(self._minimal_pipeline(id=doc.id, name="HTTP updated")) + response = self.owner_client.post("/indexer/api/importer/save", {"notebook": notebook_json}) + data = json.loads(response.content) + + assert data["status"] == 0 + doc.refresh_from_db() + assert doc.name == "HTTP updated" + + def test_save_pipeline_creates_new_document_for_owner(self): + payload = self._minimal_pipeline(uuid="brand-new-pipeline-uuid") + doc, save_as = _save_pipeline(payload, self.owner) + + assert save_as + assert doc.owner_id == self.owner.id + assert doc.name == "Importer pipeline" diff --git a/desktop/libs/notebook/src/notebook/api.py b/desktop/libs/notebook/src/notebook/api.py index 4f68b2c3b52..6b57b6de99a 100644 --- a/desktop/libs/notebook/src/notebook/api.py +++ b/desktop/libs/notebook/src/notebook/api.py @@ -19,8 +19,8 @@ import logging from urllib.parse import unquote as urllib_unquote -import sqlparse import opentracing.tracer +import sqlparse from django.db.models import Q from django.urls import reverse from django.utils.translation import gettext as _ @@ -31,14 +31,15 @@ from desktop.lib.django_util import JsonResponse from desktop.lib.exceptions_renderable import PopupException from desktop.lib.i18n import smart_str -from desktop.models import Document, Document2, FilesystemException, __paginate, _get_gist_document +from desktop.models import __paginate, _get_gist_document, Document, Document2, FilesystemException from indexer.fields import Field from indexer.file_format import HiveFormat from metadata.conf import OPTIMIZER from notebook.conf import EXAMPLES -from notebook.connectors.base import Notebook, QueryError, QueryExpired, SessionExpired, _get_snippet_name, patch_snippet_for_connector +from notebook.connectors.base import _get_snippet_name, Notebook, patch_snippet_for_connector, QueryError, QueryExpired, SessionExpired from notebook.decorators import api_error_handler, check_document_access_permission, check_document_modify_permission from notebook.models import _get_dialect_example, escape_rows, get_api, make_notebook, upgrade_session_properties +from notebook.saved_document_auth import get_writable_document_for_update LOG = logging.getLogger() @@ -142,7 +143,7 @@ def _execute_notebook(request, notebook, snippet): if snippet.get('interface') == 'sqlalchemy': interpreter.options['session'] = sessions[0] - with opentracing.tracer.start_span('interpreter') as span: + with opentracing.tracer.start_span('interpreter'): # interpreter.execute needs the sessions, but we don't want to persist them pre_execute_sessions = notebook['sessions'] notebook['sessions'] = sessions @@ -473,11 +474,8 @@ def _save_notebook(notebook, user): save_as = False - if notebook.get('parentSavedQueryUuid'): # We save into the original saved query, not into the query history - notebook_doc = Document2.objects.get_by_uuid(user=user, uuid=notebook['parentSavedQueryUuid']) - elif notebook.get('id'): - notebook_doc = Document2.objects.get(id=notebook['id']) - else: + notebook_doc = get_writable_document_for_update(notebook, user) + if notebook_doc is None: notebook_doc = Document2.objects.create(name=notebook['name'], uuid=notebook['uuid'], type=notebook_type, owner=user) Document.objects.link( notebook_doc, owner=notebook_doc.owner, name=notebook_doc.name, description=notebook_doc.description, extra=notebook_type @@ -646,7 +644,7 @@ def get_history(request): def clear_history(request): response = {'status': -1} - notebook = json.loads(request.POST.get('notebook', '{}')) + notebook = json.loads(request.POST.get('notebook', '{}')) # noqa: F841 doc_type = request.POST.get('doc_type') is_notification_manager = request.POST.get('is_notification_manager', 'false') == 'true' @@ -740,7 +738,7 @@ def autocomplete(request, server=None, database=None, table=None, column=None, n response = {'status': -1} # Passed by check_document_access_permission but unused by APIs - notebook = json.loads(request.POST.get('notebook', '{}')) + notebook = json.loads(request.POST.get('notebook', '{}')) # noqa: F841 cluster = json.loads(request.POST.get('cluster', '{}')) if cluster and cluster.get('type') in ('hive-compute', 'impala-compute', 'trino-compute'): snippet = cluster @@ -766,7 +764,7 @@ def get_sample_data(request, server=None, database=None, table=None, column=None response = {'status': -1} # Passed by check_document_access_permission but unused by APIs - notebook = json.loads(request.POST.get('notebook', '{}')) + notebook = json.loads(request.POST.get('notebook', '{}')) # noqa: F841 snippet = json.loads(request.POST.get('snippet', '{}')) is_async = json.loads(request.POST.get('async', 'false')) operation = json.loads(request.POST.get('operation', '"default"')) diff --git a/desktop/libs/notebook/src/notebook/api_tests.py b/desktop/libs/notebook/src/notebook/api_tests.py index 84da9a15147..827283c716e 100644 --- a/desktop/libs/notebook/src/notebook/api_tests.py +++ b/desktop/libs/notebook/src/notebook/api_tests.py @@ -34,7 +34,7 @@ from desktop.models import Directory, Document, Document2 from hadoop import cluster as originalCluster from notebook.api import _historify -from notebook.conf import ENABLE_ALL_INTERPRETERS, INTERPRETERS, INTERPRETERS_SHOWN_ON_WHEEL, get_ordered_interpreters +from notebook.conf import ENABLE_ALL_INTERPRETERS, get_ordered_interpreters, INTERPRETERS, INTERPRETERS_SHOWN_ON_WHEEL from notebook.connectors.base import Api, Notebook, QueryError, QueryExpired from notebook.decorators import api_error_handler from useradmin.models import User @@ -146,6 +146,19 @@ def test_save_notebook(self): assert doc.search == "select * from default.web_logs where app = 'metastore';" assert doc.type == "query-hive" + def test_save_notebook_denies_cross_user_update(self): + grant_access(self.user_not_me.username, self.user_not_me.username, 'desktop') + notebook_cp = self.notebook.copy() + notebook_cp['name'] = 'Attacker overwrite' + notebook_json = json.dumps(notebook_cp) + + response = self.client_not_me.post(reverse('notebook:save_notebook'), {'notebook': notebook_json}) + data = json.loads(response.content) + + assert data['status'] == -1 + self.doc2.refresh_from_db() + assert self.doc2.name == self.notebook['name'] + def test_type_when_saving_an_actual_notebook(self): notebook_json = ( """ @@ -208,32 +221,6 @@ def test_save_notebook_with_connector_off(self): doc = Document2.objects.get(pk=data['id']) assert 'query-mysql' == doc.type - def test_save_notebook_with_connector_on(self): - if not ENABLE_CONNECTORS.get(): - pytest.skip("Skipping Test") - - notebook_cp = self.notebook.copy() - notebook_cp.pop('id') - - connector = Connector.objects.create(name='MySql', dialect='mysql') - - notebook_cp['snippets'][0]['connector'] = { - 'name': 'MySql', - 'dialect': 'mysql', - 'type': str(connector.id), - 'optimizer': 'api', - } - - try: - response = self.client.post(reverse('notebook:save_notebook'), {'notebook': notebook_json}) - data = json.loads(response.content) - finally: - connector.delete() - - assert 0 == data['status'], data - doc = Document2.objects.get(pk=data['id']) - assert 'query-mysql' == doc.type - def test_historify(self): # Starts with no history assert 0 == Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count() @@ -969,7 +956,7 @@ def test_open_saved_impala_query_when_no_hive_interepreter(self): name='open_saved_query_with_hive_not_present', type='query-impala', owner=self.user, data={} ) - with patch('desktop.middleware.fsmanager') as fsmanager: + with patch('desktop.middleware.fsmanager'): response = self.client.get(reverse('notebook:editor'), {'editor': doc.id, 'is_embeddable': True}) assert 200 == response.status_code finally: diff --git a/desktop/libs/notebook/src/notebook/decorators.py b/desktop/libs/notebook/src/notebook/decorators.py index 75b218d06b3..19c73b4d313 100644 --- a/desktop/libs/notebook/src/notebook/decorators.py +++ b/desktop/libs/notebook/src/notebook/decorators.py @@ -15,11 +15,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import re -import sys import json -import math import logging +import math +import re from django.forms import ValidationError from django.http import Http404 @@ -32,7 +31,7 @@ from desktop.lib.exceptions_renderable import PopupException from desktop.lib.i18n import smart_str from desktop.lib.rest.http_client import RestException -from desktop.models import Document, Document2, FilesystemException +from desktop.models import Document2, FilesystemException from notebook.conf import check_has_missing_permission from notebook.connectors.base import ( AuthenticationRequired, @@ -43,6 +42,7 @@ SessionExpired, ) from notebook.models import _get_editor_type +from notebook.saved_document_auth import get_writable_document_for_update LOG = logging.getLogger() @@ -96,15 +96,7 @@ def check_document_modify_permission(): def inner(view_func): def decorate(request, *args, **kwargs): notebook = json.loads(request.POST.get('notebook', '{}')) - - try: - if notebook.get('id'): - doc2 = Document2.objects.get(uuid=notebook['parentSavedQueryUuid']) if notebook.get('parentSavedQueryUuid') else \ - Document2.objects.get(id=notebook['id']) - doc2.can_write_or_exception(request.user) - except Document.DoesNotExist: - raise PopupException(_('Document %(id)s does not exist') % {'id': notebook.get('id')}) - + get_writable_document_for_update(notebook, request.user) return view_func(request, *args, **kwargs) return wraps(view_func)(decorate) return inner @@ -117,7 +109,7 @@ def wrapper(*args, **kwargs): try: return f(*args, **kwargs) - except SessionExpired as e: + except SessionExpired: response['status'] = -2 except QueryExpired as e: if ENABLE_HUE_5.get(): @@ -135,7 +127,7 @@ def wrapper(*args, **kwargs): LOG.exception('Error validation %s' % f) response['status'] = -1 response['message'] = e.message - except OperationTimeout as e: + except OperationTimeout: response['status'] = -4 except FilesystemException as e: response['status'] = 2 diff --git a/desktop/libs/notebook/src/notebook/saved_document_auth.py b/desktop/libs/notebook/src/notebook/saved_document_auth.py new file mode 100644 index 00000000000..8725b842678 --- /dev/null +++ b/desktop/libs/notebook/src/notebook/saved_document_auth.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# Licensed to Cloudera, Inc. under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. Cloudera, Inc. licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Resolve existing documents for notebook/pipeline save operations with write permission. + +Used by save API decorators and by ``_save_notebook`` / ``_save_pipeline`` so authorization +cannot be bypassed when a view decorator is missing or incomplete. +""" + +from django.utils.translation import gettext as _ + +from desktop.lib.exceptions_renderable import PopupException +from desktop.models import Document2, FilesystemException + + +def get_writable_document_for_update(document_payload, user): + """ + Resolve a ``Document2`` the caller may update when saving JSON notebook/pipeline state. + + Resolution order matches ``_save_notebook`` and ``_save_pipeline``: + + 1. ``parentSavedQueryUuid`` — save into the parent saved query (not query history). + 2. ``id`` — update an existing document by primary key. + + :param document_payload: Parsed ``notebook`` or pipeline dict from the client. + :param user: Authenticated user performing the save. + :return: ``Document2`` when updating an existing document, or ``None`` when creating a new one. + :raises PopupException: When the document is missing or the user lacks write permission. + + ``check_document_modify_permission`` calls this before save handlers and ignores a ``None`` return + (new document). ``_save_notebook`` and ``_save_pipeline`` use the return value to choose update vs create. + """ + if document_payload.get('parentSavedQueryUuid'): + try: + return Document2.objects.get_by_uuid( + user=user, + uuid=document_payload['parentSavedQueryUuid'], + perm_type='write', + ) + except FilesystemException as ex: + raise PopupException(str(ex)) + + if document_payload.get('id'): + try: + document = Document2.objects.get(id=document_payload['id']) + except Document2.DoesNotExist: + # Same message as a denied write so callers cannot probe which document ids exist. + raise PopupException(_("Document does not exist or you don't have the permission to access it.")) + document.can_write_or_exception(user) + return document + + return None diff --git a/desktop/libs/notebook/src/notebook/saved_document_auth_tests.py b/desktop/libs/notebook/src/notebook/saved_document_auth_tests.py new file mode 100644 index 00000000000..88091c39237 --- /dev/null +++ b/desktop/libs/notebook/src/notebook/saved_document_auth_tests.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# Licensed to Cloudera, Inc. under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. Cloudera, Inc. licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from desktop.lib.django_test_util import make_logged_in_client +from desktop.lib.exceptions_renderable import PopupException +from desktop.lib.test_utils import grant_access +from desktop.models import Document2 +from notebook.saved_document_auth import get_writable_document_for_update +from useradmin.models import User + + +@pytest.mark.django_db +class TestSavedDocumentAuth(object): + def setup_method(self): + make_logged_in_client(username="save_doc_owner", recreate=True, is_superuser=False) + make_logged_in_client(username="save_doc_other", recreate=True, is_superuser=False) + make_logged_in_client(username="save_doc_admin", recreate=True, is_superuser=True) + grant_access("save_doc_owner", "save_doc_owner", "desktop") + grant_access("save_doc_other", "save_doc_other", "desktop") + self.owner = User.objects.get(username="save_doc_owner") + self.other = User.objects.get(username="save_doc_other") + self.admin = User.objects.get(username="save_doc_admin") + + def test_get_writable_document_for_update_returns_none_for_new_document(self): + assert get_writable_document_for_update({"name": "new", "uuid": "abc"}, self.owner) is None + + def test_get_writable_document_for_update_allows_owner_by_id(self): + doc = Document2.objects.create(name="mine", uuid="owner-doc-uuid", type="pipeline", owner=self.owner) + resolved = get_writable_document_for_update({"id": doc.id}, self.owner) + assert resolved.id == doc.id + + def test_get_writable_document_for_update_denies_other_user_by_id(self): + doc = Document2.objects.create(name="victim", uuid="victim-doc-uuid", type="pipeline", owner=self.owner) + with pytest.raises(PopupException): + get_writable_document_for_update({"id": doc.id}, self.other) + + def test_get_writable_document_for_update_parent_uuid_requires_write(self): + parent = Document2.objects.create(name="parent", uuid="parent-uuid", type="query-hive", owner=self.owner) + with pytest.raises(PopupException): + get_writable_document_for_update({"parentSavedQueryUuid": parent.uuid}, self.other) + + def test_get_writable_document_for_update_allows_admin_for_other_users_private_document(self): + doc = Document2.objects.create(name="private", uuid="private-doc-uuid", type="query-hive", owner=self.owner) + resolved = get_writable_document_for_update({"id": doc.id}, self.admin) + assert resolved.id == doc.id + + def test_get_writable_document_for_update_missing_and_denied_documents_share_one_message(self): + doc = Document2.objects.create(name="victim", uuid="probe-doc-uuid", type="pipeline", owner=self.owner) + with pytest.raises(PopupException) as denied: + get_writable_document_for_update({"id": doc.id}, self.other) + with pytest.raises(PopupException) as missing: + get_writable_document_for_update({"id": doc.id + 100000}, self.other) + assert denied.value.message == missing.value.message