[security] Pin certifi, upgrade pip/virtualenv and drop bundled CA lists (eMudhra)

Makefile

@@ -45,7 +45,7 @@
 # ==========
 # Here we summarize the flow of the build logic. Lines represent dependency.
 #
-#                  virtual-bootstrap.py
+#                      virtualenv
 #                            |
 #                  virtual-env (./env)
 #                            |
@@ -137,7 +137,7 @@ $(info "SYS_PIP is $(SYS_PIP)")
 $(info "ENV_PIP is $(ENV_PIP)")
 $(info "PYTHON_INCLUDE_DIR is $(PYTHON_INCLUDE_DIR)")
 $(info "PYTHON_H is $(PYTHON_H)")
-$(info "PIP_VERSION is $(PIP_VERSION)")
+$(info "PIP_BOOTSTRAP_REQUIREMENTS_FILE is $(PIP_BOOTSTRAP_REQUIREMENTS_FILE)")
 $(info "VIRTUAL_ENV_VERSION is $(VIRTUAL_ENV_VERSION)")
 $(info "RELOCATABLE is $(RELOCATABLE)")
 $(info "LOCAL_PY_BIN is $(LOCAL_PY_BIN)")
@@ -155,10 +155,15 @@ virtual-env: $(BLD_DIR_ENV)/bin/python
 $(BLD_DIR_ENV)/bin/python:
 	@echo "--- Creating virtual environment for $(PYTHON_VER) ---"
 	@mkdir -p $(BLD_DIR_ENV)
-	@$(SYS_PYTHON) -m pip install --upgrade pip==$(PIP_VERSION)
+	@$(SYS_PYTHON) -m pip install --upgrade --require-hashes -r $(PIP_BOOTSTRAP_REQUIREMENTS_FILE)
 	@$(SYS_PYTHON) -m pip install virtualenv==$(VIRTUAL_ENV_VERSION) virtualenv-make-relocatable==$(VIRTUAL_ENV_RELOCATABLE_VERSION)
-	@$(SYS_PYTHON) -m virtualenv --copies -p $(PYTHON_VER) $(BLD_DIR_ENV)
-	@$(ENV_PYTHON) -m pip install virtualenv==$(VIRTUAL_ENV_VERSION) virtualenv-make-relocatable==$(VIRTUAL_ENV_RELOCATABLE_VERSION)
+	@$(SYS_PYTHON) -m virtualenv $(VIRTUAL_ENV_COPY_OPT) -p $(PYTHON_VER) $(BLD_DIR_ENV)
+	@# virtualenv seeds the pip wheel it embeds, which can be older than the bootstrap pip; upgrade it so the
+	@# shipped env does not keep an old pip (and its vendored, outdated CA bundle).
+	@$(ENV_PYTHON) -m pip install --upgrade --require-hashes -r $(PIP_BOOTSTRAP_REQUIREMENTS_FILE)
+	@# virtualenv is only needed on the build host (above). The shipped env needs just the relocator,
+	@# which does not import virtualenv.
+	@$(ENV_PYTHON) -m pip install virtualenv-make-relocatable==$(VIRTUAL_ENV_RELOCATABLE_VERSION)
 	@$(eval RELOCATABLE := $(shell which virtualenv-make-relocatable))
 	@echo "REQUIREMENT_FILE is $(REQUIREMENT_FILE)"
 	@echo "--- Using build constraints to prevent setuptools 82+ (cx-Oracle compatibility) ---"

Makefile.tarball

@@ -31,7 +31,6 @@ PROD_INCLUDES := \
 	desktop/conf \
 	desktop/desktop.db \
 	tools/app_reg \
-	tools/virtual-bootstrap \
 	tools/relocatable.py \
 	tools/load-balancer \
 	tools/scripts \

Makefile.vars

@@ -146,15 +146,18 @@ LOCAL_PY_BIN := $(LOCAL_PY_BASE)/bin
 export PATH := $(PATH):$(LOCAL_PY_BIN)
 ##############################
 # Set specific versions for some libraries
-##############################
-PIP_VERSION := 22.2.2
-VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
-
-##############################
-# Set specific versions for some libraries
-##############################
-PIP_VERSION := 22.2.2
-VIRTUAL_ENV_VERSION := 21.7.16
+#
+# Security floors (eMudhra CA incident):
+# - pip >= 26.2.2: pip always trusts its own vendored certifi in addition to the system trust
+#   store, so any older pip keeps trusting the compromised CAs. The hash-pinned version lives in
+#   PIP_BOOTSTRAP_REQUIREMENTS_FILE.
+# - virtualenv >= 21.14.12: used on the build host only. It is not installed into the shipped env.
+# Keep tools/cloudera/build_hue_common.sh in sync with these versions.
+##############################
+PIP_BOOTSTRAP_REQUIREMENTS_FILE := $(ROOT)/desktop/core/pip-bootstrap-requirements.txt
+VIRTUAL_ENV_VERSION := 21.14.13
 VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
+# virtualenv >= 20.36 rejects --copies for Homebrew Python on macOS; Linux builds need copies.
+VIRTUAL_ENV_COPY_OPT := $(if $(filter Darwin,$(shell uname -s)),,--copies)
 
 RELOCATABLE := $(shell which virtualenv-make-relocatable)

Makefile.vars.priv

@@ -62,7 +62,6 @@ endif
 
 # Various commands
 APP_REG := $(ENV_PYTHON) $(TOOLS_DIR)/app_reg/app_reg.py
-VIRTUAL_BOOTSTRAP := $(TOOLS_DIR)/virtual-bootstrap/virtual-bootstrap.py
 
 
 # <<<< DEV ONLY

desktop/core/ext-py3/boto-2.49.0/MANIFEST.in

@@ -1,4 +1,3 @@
-include boto/cacerts/cacerts.txt
 include README.rst
 include boto/file/README
 include .gitignore