From 4a60b09482ef883089c8b98fae636903cf6fb214 Mon Sep 17 00:00:00 2001 From: Harsh Gupta <42064744+Harshg999@users.noreply.github.com> Date: Fri, 30 Oct 2026 01:46:29 +0530 Subject: [PATCH 1/2] [core] Upgrade setuptools to 84.1.0 and stop using pkg_resources setuptools 80.9.0 is affected by a MANIFEST.in exclusion bypass (fixed in 83.0.0) and a Windows path escape during legacy wheel installation (fixed in 84.1.0). setuptools 82 removed pkg_resources, so Hue and the packages it installs have to stop importing it. - appmanager, supervisor, settings and gunicorn_server_utils read entry points and the desktop version through importlib.metadata. - pyformance 0.3.2 calls pkg_resources.declare_namespace() without a guard in its __init__.py. Vendor it under ext-py3 without that line (it is not a namespace package) and install it as a local requirement. - SQLAlchemy 1.3 finds third party dialects (trino://, phoenix://) only through pkg_resources. Wrap the dialect registry so that those dialects are looked up through the same "sqlalchemy.dialects" entry points with importlib.metadata, after the built in dialects. Add tests. - The Makefile passed the cx-Oracle build constraint setuptools<82 as PIP_CONSTRAINT, which also constrains the setuptools the virtual env installs and would conflict with 84.1.0. Pass it only as PIP_BUILD_CONSTRAINT, which applies to pip's isolated build environments, where cx-Oracle's setup.py still needs pkg_resources. - appmanager.get_bootstrap_file used the Python 2 file() builtin. The files touched here are also made ruff clean, as CI requires, which removes some unused imports, bindings and test variables. --- Makefile | 8 +- desktop/core/Makefile | 4 +- desktop/core/base_requirements.txt | 4 +- desktop/core/build-constraints.txt | 6 +- desktop/core/ext-py3/pyformance-0.3.2/LICENSE | 13 + .../pyformance-0.3.2/pyformance/__init__.py | 8 + .../pyformance/__version__.py | 1 + .../pyformance/meters/__init__.py | 5 + .../pyformance/meters/counter.py | 31 ++ .../pyformance/meters/gauge.py | 57 +++ .../pyformance/meters/histogram.py | 93 +++++ .../pyformance/meters/meter.py | 58 +++ .../pyformance/meters/timer.py | 119 ++++++ .../pyformance-0.3.2/pyformance/registry.py | 370 ++++++++++++++++++ .../pyformance/reporters/__init__.py | 28 ++ .../pyformance/reporters/carbon_reporter.py | 68 ++++ .../pyformance/reporters/console_reporter.py | 39 ++ .../pyformance/reporters/csv_reporter.py | 49 +++ .../reporters/hosted_graphite_reporter.py | 47 +++ .../pyformance/reporters/reporter.py | 45 +++ .../pyformance/stats/__init__.py | 3 + .../pyformance/stats/moving_average.py | 69 ++++ .../pyformance/stats/samples.py | 112 ++++++ .../pyformance/stats/snapshot.py | 59 +++ .../ext-py3/pyformance-0.3.2/pyproject.toml | 3 + .../core/ext-py3/pyformance-0.3.2/setup.py | 22 ++ desktop/core/generate_requirements.py | 7 +- desktop/core/generate_requirements_test.py | 2 +- desktop/core/src/desktop/appmanager.py | 14 +- .../src/desktop/lib/gunicorn_server_utils.py | 4 +- desktop/core/src/desktop/settings.py | 4 +- desktop/core/src/desktop/supervisor.py | 25 +- .../src/notebook/connectors/sql_alchemy.py | 22 ++ .../notebook/connectors/sql_alchemy_tests.py | 76 ++-- 34 files changed, 1412 insertions(+), 63 deletions(-) create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/LICENSE create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/__init__.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/__version__.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/__init__.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/counter.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/gauge.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/histogram.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/meter.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/timer.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/registry.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/__init__.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/carbon_reporter.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/console_reporter.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/csv_reporter.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/hosted_graphite_reporter.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/reporter.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/__init__.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/moving_average.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/samples.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/snapshot.py create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/pyproject.toml create mode 100644 desktop/core/ext-py3/pyformance-0.3.2/setup.py diff --git a/Makefile b/Makefile index 4a549a9e6d3..c6eed7fb5f4 100644 --- a/Makefile +++ b/Makefile @@ -111,7 +111,7 @@ include Makefile.tarball # Build docs (unused) ################################### test_prep: - @PIP_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt PIP_BUILD_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PYTHON) -m pip install -r $(REQUIREMENT_TEST_FILE) + @PIP_BUILD_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PYTHON) -m pip install -r $(REQUIREMENT_TEST_FILE) ################################### # Build docs (unused) @@ -166,10 +166,10 @@ $(BLD_DIR_ENV)/bin/python: @$(ENV_PYTHON) -m pip install virtualenv-make-relocatable==$(VIRTUAL_ENV_RELOCATABLE_VERSION) @$(eval RELOCATABLE := $(shell which virtualenv-make-relocatable)) @echo "REQUIREMENT_FILE is $(REQUIREMENT_FILE)" - @echo "--- Using build constraints to prevent setuptools 82+ (cx-Oracle compatibility) ---" - @PIP_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt PIP_BUILD_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PIP) install -r $(REQUIREMENT_FILE) + @echo "--- Building from source with setuptools < 82 in the build environment (cx-Oracle compatibility) ---" + @PIP_BUILD_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PIP) install -r $(REQUIREMENT_FILE) @echo "--- Virtual environment setup complete for $(PYTHON_VER) ---" - @PIP_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt PIP_BUILD_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PIP) install $(NAVOPTAPI_WHL) + @PIP_BUILD_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PIP) install $(NAVOPTAPI_WHL) @echo "--- Finished installing $(NAVOPTAPI_WHL) into virtual-env ---" ################################### # Build desktop diff --git a/desktop/core/Makefile b/desktop/core/Makefile index 2f4ed4a55b9..d8463224eed 100644 --- a/desktop/core/Makefile +++ b/desktop/core/Makefile @@ -38,5 +38,5 @@ default:: env-install: $(EXT_ENV_INSTALL) compile @echo '--- Installing $(APP_NAME) into virtual-env' @echo '--- env-install: EXT_ENV_INSTALL: $(EXT_ENV_INSTALL)' - @echo '--- Using build constraints to prevent setuptools 82+ (cx-Oracle compatibility)' - @PIP_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PYTHON) -m pip install -e . + @echo '--- Building with setuptools < 82 in the build environment' + @PIP_BUILD_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PYTHON) -m pip install -e . diff --git a/desktop/core/base_requirements.txt b/desktop/core/base_requirements.txt index df2d0cb5fc5..c7fa50587d9 100644 --- a/desktop/core/base_requirements.txt +++ b/desktop/core/base_requirements.txt @@ -48,7 +48,6 @@ protobuf==6.33.5 psutil==5.8.0 pyarrow==23.0.1 pydantic==2.10.6 -pyformance==0.3.2 PyJWT==2.15.1 pyOpenSSL==26.0.0 python-daemon==2.2.4 @@ -80,9 +79,10 @@ Markdown==3.8.1 numpy==1.24.4 pandas==2.0.3 pure-sasl==0.6.2 -setuptools==80.9.0 +setuptools==84.1.0 sqlparse==0.6.0 -e file://${ROOT}/desktop/core/ext-py3/boto-2.49.0 -e file://${ROOT}/desktop/core/ext-py3/djangomako-1.3.2 +-e file://${ROOT}/desktop/core/ext-py3/pyformance-0.3.2 -e file://${ROOT}/desktop/core/ext-py3/pysaml2-7.3.1 -e file://${ROOT}/desktop/core/ext-py3/python-sasl-0.3.1 diff --git a/desktop/core/build-constraints.txt b/desktop/core/build-constraints.txt index 9482f43800f..4e0585f81f8 100644 --- a/desktop/core/build-constraints.txt +++ b/desktop/core/build-constraints.txt @@ -4,6 +4,8 @@ # always builds it from source. Its setup.py imports pkg_resources, which # setuptools removed in 82.0.0. # -# This must be passed as PIP_BUILD_CONSTRAINT: PIP_CONSTRAINT does not apply to -# the isolated environment pip creates for build dependencies, and needs pip >= 25.3. +# This must be passed as PIP_BUILD_CONSTRAINT, which constrains only the isolated +# environment pip creates for build dependencies and needs pip >= 25.3. Do not pass +# it as PIP_CONSTRAINT: that would also constrain the setuptools the virtual env +# installs, which has to be 84.1.0 or later. setuptools<82 diff --git a/desktop/core/ext-py3/pyformance-0.3.2/LICENSE b/desktop/core/ext-py3/pyformance-0.3.2/LICENSE new file mode 100644 index 00000000000..5cd54bafe32 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/LICENSE @@ -0,0 +1,13 @@ +Copyright 2014 Omer Gertel + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/__init__.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/__init__.py new file mode 100644 index 00000000000..04184ab2e97 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/__init__.py @@ -0,0 +1,8 @@ +# Hue: upstream line 1 was `__import__('pkg_resources').declare_namespace(__name__)`. +# pyformance is not a namespace package (setup.py declares none), and setuptools 82 +# removed pkg_resources, so the line is dropped to allow setuptools >= 83 in the venv. + +from .registry import MetricsRegistry, global_registry, set_global_registry +from .registry import timer, counter, meter, histogram, gauge +from .registry import dump_metrics, clear, count_calls, meter_calls, hist_calls, time_calls +from .meters.timer import call_too_long diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/__version__.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/__version__.py new file mode 100644 index 00000000000..f9aa3e11093 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/__version__.py @@ -0,0 +1 @@ +__version__ = "0.3.2" diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/__init__.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/__init__.py new file mode 100644 index 00000000000..fd864657f41 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/__init__.py @@ -0,0 +1,5 @@ +from .counter import Counter +from .meter import Meter +from .histogram import Histogram +from .timer import Timer +from .gauge import Gauge, CallbackGauge, SimpleGauge diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/counter.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/counter.py new file mode 100644 index 00000000000..5c77742fbd3 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/counter.py @@ -0,0 +1,31 @@ +from threading import Lock + + +class Counter(object): + + """ + An incrementing and decrementing metric + """ + + def __init__(self): + super(Counter, self).__init__() + self.lock = Lock() + self.counter = 0 + + def inc(self, val=1): + "increment counter by val (default is 1)" + with self.lock: + self.counter = self.counter + val + + def dec(self, val=1): + "decrement counter by val (default is 1)" + self.inc(-val) + + def get_count(self): + "return current value of counter" + return self.counter + + def clear(self): + "reset counter to 0" + with self.lock: + self.counter = 0 diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/gauge.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/gauge.py new file mode 100644 index 00000000000..75c6f89ad8b --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/gauge.py @@ -0,0 +1,57 @@ +class Gauge(object): + + """ + A base class for reading of a particular. + + For example, to instrument a queue depth: + + class QueueLengthGaguge(Gauge): + def __init__(self, queue): + super(QueueGaguge, self).__init__() + self.queue = queue + + def get_value(self): + return len(self.queue) + + """ + + def get_value(self): + "A subclass of Gauge should implement this method" + raise NotImplementedError() + + +class CallbackGauge(Gauge): + + """ + A Gauge reading for a given callback + """ + + def __init__(self, callback): + "constructor expects a callable" + super(CallbackGauge, self).__init__() + self.callback = callback + + def get_value(self): + "returns the result of callback which is executed each time" + return self.callback() + + +class SimpleGauge(Gauge): + + """ + A gauge which holds values with simple getter- and setter-interface + """ + + def __init__(self, value=float("nan")): + "constructor accepts initial value" + super(SimpleGauge, self).__init__() + self._value = value + + def get_value(self): + "getter returns current value" + return self._value + + def set_value(self, value): + "setter changes current value" + # XXX: add locking? + self._value = value diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/histogram.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/histogram.py new file mode 100644 index 00000000000..474931a6432 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/histogram.py @@ -0,0 +1,93 @@ +import time +import math +from threading import Lock +from ..stats. samples import ExpDecayingSample, DEFAULT_SIZE, DEFAULT_ALPHA + + +class Histogram(object): + + """ + A metric which calculates the distribution of a value. + """ + + def __init__(self, size=DEFAULT_SIZE, alpha=DEFAULT_ALPHA, clock=time): + """ + Creates a new instance of a L{Histogram}. + """ + super(Histogram, self).__init__() + self.lock = Lock() + self.clock = clock + self.sample = ExpDecayingSample(size, alpha, clock) + self.clear() + + def add(self, value): + """ + Add value to histogram + + :type value: float + """ + with self.lock: + self.sample.update(value) + self.counter = self.counter + 1 + self.max = value if value > self.max else self.max + self.min = value if value < self.min else self.min + self.sum = self.sum + value + self._update_var(value) + + def clear(self): + "reset histogram to initial state" + with self.lock: + self.sample.clear() + self.counter = 0.0 + self.max = -2147483647.0 + self.min = 2147483647.0 + self.sum = 0.0 + self.var = [-1.0, 0.0] + + def get_count(self): + "get current value of counter" + return self.counter + + def get_sum(self): + "get current sum" + return self.sum + + def get_max(self): + "get current maximum" + return self.max + + def get_min(self): + "get current minimum" + return self.min + + def get_mean(self): + "get current mean" + if self.counter > 0: + return self.sum / self.counter + return 0 + + def get_stddev(self): + "get current standard deviation" + if self.counter > 0: + return math.sqrt(self.get_var()) + return 0 + + def get_var(self): + "get current variance" + if self.counter > 1: + return self.var[1] / (self.counter - 1) + return 0 + + def get_snapshot(self): + "get snapshot instance which holds the percentiles" + return self.sample.get_snapshot() + + def _update_var(self, value): + old_m, old_s = self.var + new_m, new_s = [0.0, 0.0] + if old_m == -1: + new_m = value + else: + new_m = old_m + ((value - old_m) / self.counter) + new_s = old_s + ((value - old_m) * (value - new_m)) + self.var = [new_m, new_s] diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/meter.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/meter.py new file mode 100644 index 00000000000..3a6b9bcfaf1 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/meter.py @@ -0,0 +1,58 @@ +import time +from threading import Lock +from ..stats.moving_average import ExpWeightedMovingAvg + + +class Meter(object): + + """ + A meter metric which measures mean throughput and one-, five-, and fifteen-minute + exponentially-weighted moving average throughputs. + """ + + def __init__(self, clock=time): + super(Meter, self).__init__() + self.lock = Lock() + self.clock = clock + self.clear() + + def clear(self): + with self.lock: + self.start_time = self.clock.time() + self.counter = 0.0 + self.m1rate = ExpWeightedMovingAvg(period=1, clock=self.clock) + self.m5rate = ExpWeightedMovingAvg(period=5, clock=self.clock) + self.m15rate = ExpWeightedMovingAvg(period=15, clock=self.clock) + + def get_one_minute_rate(self): + return self.m1rate.get_rate() + + def get_five_minute_rate(self): + return self.m5rate.get_rate() + + def get_fifteen_minute_rate(self): + return self.m15rate.get_rate() + + def tick(self): + self.m1rate.tick() + self.m5rate.tick() + self.m15rate.tick() + + def mark(self, value=1): + with self.lock: + self.counter += value + self.m1rate.add(value) + self.m5rate.add(value) + self.m15rate.add(value) + + def get_count(self): + return self.counter + + def get_mean_rate(self): + if self.counter == 0: + return 0 + elapsed = self.clock.time() - self.start_time + return self.counter / elapsed + + def _convertNsRate(self, ratePerNs): + return ratePerNs diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/timer.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/timer.py new file mode 100644 index 00000000000..352a46ce62f --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/timer.py @@ -0,0 +1,119 @@ +import time +try: + from blinker import Namespace +except ImportError: + Namespace = None +from .histogram import Histogram, DEFAULT_SIZE, DEFAULT_ALPHA +from .meter import Meter + +if Namespace is not None: + timer_signals = Namespace() + call_too_long = timer_signals.signal("call_too_long") +else: + call_too_long = None + + +class Timer(object): + + """ + A timer metric which aggregates timing durations and provides duration statistics, plus + throughput statistics via Meter and Histogram. + + """ + + def __init__(self, threshold=None, size=DEFAULT_SIZE, alpha=DEFAULT_ALPHA, clock=time): + super(Timer, self).__init__() + self.meter = Meter(clock=clock) + self.hist = Histogram(clock=clock) + self.threshold = threshold + + def get_count(self): + "get count from internal histogram" + return self.hist.get_count() + + def get_sum(self): + "get sum from internal histogram" + return self.hist.get_sum() + + def get_max(self): + "get max from internal histogram" + return self.hist.get_max() + + def get_min(self): + "get min from internal histogram" + return self.hist.get_min() + + def get_mean(self): + "get mean from internal histogram" + return self.hist.get_mean() + + def get_stddev(self): + "get stddev from internal histogram" + return self.hist.get_stddev() + + def get_var(self): + "get var from internal histogram" + return self.hist.get_var() + + def get_snapshot(self): + "get snapshot from internal histogram" + return self.hist.get_snapshot() + + def get_mean_rate(self): + "get mean rate from internal meter" + return self.meter.get_mean_rate() + + def get_one_minute_rate(self): + "get 1 minut rate from internal meter" + return self.meter.get_one_minute_rate() + + def get_five_minute_rate(self): + "get 5 minute rate from internal meter" + return self.meter.get_five_minute_rate() + + def get_fifteen_minute_rate(self): + "get 15 rate from internal meter" + return self.meter.get_fifteen_minute_rate() + + def _update(self, seconds): + if seconds >= 0: + self.hist.add(seconds) + self.meter.mark() + + def time(self, *args, **kwargs): + """ + Parameters will be sent to signal, if fired. + Returns a timer context instance which can be used from a with-statement. + Without with-statement you have to call the stop method on the context + """ + return TimerContext(self, self.meter.clock, *args, **kwargs) + + def clear(self): + "clear internal histogram and meter" + self.hist.clear() + self.meter.clear() + + +class TimerContext(object): + + def __init__(self, timer, clock, *args, **kwargs): + super(TimerContext, self).__init__() + self.clock = clock + self.timer = timer + self.start_time = self.clock.time() + self.kwargs = kwargs + self.args = args + + def stop(self): + elapsed = self.clock.time() - self.start_time + self.timer._update(elapsed) + if self.timer.threshold and self.timer.threshold < elapsed and call_too_long is not None: + call_too_long.send( + self.timer, elapsed=elapsed, *self.args, **self.kwargs) + return elapsed + + def __enter__(self): + pass + + def __exit__(self, t, v, tb): + self.stop() diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/registry.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/registry.py new file mode 100644 index 00000000000..0155385fcfc --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/registry.py @@ -0,0 +1,370 @@ +import functools +import re +import time +import sys +from .meters import Counter, Histogram, Meter, Timer, Gauge, CallbackGauge, SimpleGauge + + +class MetricsRegistry(object): + + """ + A single interface used to gather metrics on a service. It keeps track of + all the relevant Counters, Meters, Histograms, and Timers. It does not have + a reference back to its service. The service would create a + L{MetricsRegistry} to manage all of its metrics tools. + """ + def __init__(self, clock = time): + """ + Creates a new L{MetricsRegistry} instance. + """ + self._timers = {} + self._meters = {} + self._counters = {} + self._histograms = {} + self._gauges = {} + self._clock = clock + + def add(self, key, metric): + """ + Use this method to manually add custom metric instances to the registry + which are not created with their constructor's default arguments, + e.g. Histograms with a different size. + + :param key: name of the metric + :type key: C{str} + :param metric: instance of Histogram, Meter, Gauge, Timer or Counter + """ + class_map = { + Histogram: self._histograms, + Meter: self._meters, + Gauge: self.gauges, + Timer: self._timers, + Counter: self._counters, + } + for cls, registry in class_map.iteritems(): + if isinstance(metric, cls): + if key in registry: + raise LookupError("Metric %r already registered" % key) + registry[key] = registry + return + raise TypeError("Invalid class. Could not register metric %r" % key) + + def counter(self, key): + """ + Gets a counter based on a key, creates a new one if it does not exist. + + :param key: name of the metric + :type key: C{str} + + :return: L{Counter} + """ + if key not in self._counters: + self._counters[key] = Counter() + return self._counters[key] + + def histogram(self, key): + """ + Gets a histogram based on a key, creates a new one if it does not exist. + + :param key: name of the metric + :type key: C{str} + + :return: L{Histogram} + """ + if key not in self._histograms: + self._histograms[key] = Histogram(clock=self._clock) + return self._histograms[key] + + def gauge(self, key, gauge=None, default=float("nan")): + if key not in self._gauges: + if gauge is None: + gauge = SimpleGauge( + default) # raise TypeError("gauge required for registering") + elif not isinstance(gauge, Gauge): + if not callable(gauge): + raise TypeError("gauge getter not callable") + gauge = CallbackGauge(gauge) + self._gauges[key] = gauge + return self._gauges[key] + + def meter(self, key): + """ + Gets a meter based on a key, creates a new one if it does not exist. + + :param key: name of the metric + :type key: C{str} + + :return: L{Meter} + """ + if key not in self._meters: + self._meters[key] = Meter(clock=self._clock) + return self._meters[key] + + def timer(self, key): + """ + Gets a timer based on a key, creates a new one if it does not exist. + + :param key: name of the metric + :type key: C{str} + + :return: L{Timer} + """ + if key not in self._timers: + self._timers[key] = Timer(clock=self._clock) + return self._timers[key] + + def clear(self): + self._meters.clear() + self._counters.clear() + self._gauges.clear() + self._timers.clear() + self._histograms.clear() + + def _get_counter_metrics(self, key): + if key in self._counters: + counter = self._counters[key] + return {"count": counter.get_count()} + return {} + + def _get_gauge_metrics(self, key): + if key in self._gauges: + gauge = self._gauges[key] + return {"value": gauge.get_value()} + return {} + + def _get_histogram_metrics(self, key): + if key in self._histograms: + histogram = self._histograms[key] + snapshot = histogram.get_snapshot() + res = {"avg": histogram.get_mean(), + "count": histogram.get_count(), + "max": histogram.get_max(), + "min": histogram.get_min(), + "std_dev": histogram.get_stddev(), + "75_percentile": snapshot.get_75th_percentile(), + "95_percentile": snapshot.get_95th_percentile(), + "99_percentile": snapshot.get_99th_percentile(), + "999_percentile": snapshot.get_999th_percentile()} + return res + return {} + + def _get_meter_metrics(self, key): + if key in self._meters: + meter = self._meters[key] + res = {"count": meter.get_count(), + "15m_rate": meter.get_fifteen_minute_rate(), + "5m_rate": meter.get_five_minute_rate(), + "1m_rate": meter.get_one_minute_rate(), + "mean_rate": meter.get_mean_rate()} + return res + return {} + + def _get_timer_metrics(self, key): + if key in self._timers: + timer = self._timers[key] + snapshot = timer.get_snapshot() + res = {"avg": timer.get_mean(), + "sum": timer.get_sum(), + "count": timer.get_count(), + "max": timer.get_max(), + "min": timer.get_min(), + "std_dev": timer.get_stddev(), + "15m_rate": timer.get_fifteen_minute_rate(), + "5m_rate": timer.get_five_minute_rate(), + "1m_rate": timer.get_one_minute_rate(), + "mean_rate": timer.get_mean_rate(), + "75_percentile": snapshot.get_75th_percentile(), + "95_percentile": snapshot.get_95th_percentile(), + "99_percentile": snapshot.get_99th_percentile(), + "999_percentile": snapshot.get_999th_percentile()} + return res + return {} + + def get_metrics(self, key): + """ + Gets all the metrics for a specified key. + + :param key: name of the metric + :type key: C{str} + + :return: C{dict} + """ + metrics = {} + for getter in (self._get_counter_metrics, self._get_histogram_metrics, + self._get_meter_metrics, self._get_timer_metrics, + self._get_gauge_metrics): + metrics.update(getter(key)) + return metrics + + def dump_metrics(self): + """ + Formats all of the metrics and returns them as a dict. + + :return: C{list} of C{dict} of metrics + """ + metrics = {} + for metric_type in (self._counters, + self._histograms, + self._meters, + self._timers, + self._gauges): + for key in metric_type.keys(): + metrics[key] = self.get_metrics(key) + + return metrics + + +class RegexRegistry(MetricsRegistry): + + """ + A single interface used to gather metrics on a service. This class uses a regex to combine + measures that match a pattern. For example, if you have a REST API, instead of defining + a timer for each method, you can use a regex to capture all API calls and group them. + A pattern like '^/api/(?P)/\d+/(?P)?$' will group and measure the following: + /api/users/1 -> users + /api/users/1/edit -> users/edit + /api/users/2/edit -> users/edit + """ + def __init__(self, pattern = None, clock = time): + super(RegexRegistry, self).__init__(clock) + if pattern is not None: + self.pattern = re.compile(pattern) + else: + self.pattern = re.compile('^$') + + def _get_key(self, key): + matches = self.pattern.finditer(key) + key = '/'.join((v for match in matches for v in match.groups() if v)) + return key + + def timer(self, key): + return super(RegexRegistry, self).timer(self._get_key(key)) + + def histogram(self, key): + return super(RegexRegistry, self).histogram(self._get_key(key)) + + def counter(self, key): + return super(RegexRegistry, self).counter(self._get_key(key)) + + def gauge(self, key, gauge=None): + return super(RegexRegistry, self).gauge(self._get_key(key), gauge) + + def meter(self, key): + return super(RegexRegistry, self).meter(self._get_key(key)) + + +_global_registry = MetricsRegistry() + + +def global_registry(): + return _global_registry + + +def set_global_registry(registry): + global _global_registry + _global_registry = registry + + +def counter(key): + return _global_registry.counter(key) + + +def histogram(key): + return _global_registry.histogram(key) + + +def meter(key): + return _global_registry.meter(key) + + +def timer(key): + return _global_registry.timer(key) + + +def gauge(key, gauge=None): + return _global_registry.gauge(key, gauge) + + +def dump_metrics(): + return _global_registry.dump_metrics() + + +def clear(): + return _global_registry.clear() + +def get_qualname(obj): + if sys.version_info[0] > 2: + return obj.__qualname__ + return obj.__name__ + +def count_calls(fn): + """ + Decorator to track the number of times a function is called. + + :param fn: the function to be decorated + :type fn: C{func} + + :return: the decorated function + :rtype: C{func} + """ + @functools.wraps(fn) + def wrapper(*args, **kwargs): + counter("%s_calls" % get_qualname(fn)).inc() + return fn(*args, **kwargs) + return wrapper + + +def meter_calls(fn): + """ + Decorator to the rate at which a function is called. + + :param fn: the function to be decorated + :type fn: C{func} + + :return: the decorated function + :rtype: C{func} + """ + @functools.wraps(fn) + def wrapper(*args, **kwargs): + meter("%s_calls" % get_qualname(fn)).mark() + return fn(*args, **kwargs) + return wrapper + + +def hist_calls(fn): + """ + Decorator to check the distribution of return values of a function. + + :param fn: the function to be decorated + :type fn: C{func} + + :return: the decorated function + :rtype: C{func} + """ + @functools.wraps(fn) + def wrapper(*args, **kwargs): + _histogram = histogram("%s_calls" % get_qualname(fn)) + rtn = fn(*args, **kwargs) + if type(rtn) in (int, float): + _histogram.update(rtn) + return rtn + return wrapper + + +def time_calls(fn): + """ + Decorator to time the execution of the function. + + :param fn: the function to be decorated + :type fn: C{func} + + :return: the decorated function + :rtype: C{func} + """ + @functools.wraps(fn) + def wrapper(*args, **kwargs): + _timer = timer("%s_calls" % get_qualname(fn)) + with _timer.time(fn = get_qualname(fn)): + return fn(*args, **kwargs) + return wrapper + diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/__init__.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/__init__.py new file mode 100644 index 00000000000..ce44e5f2933 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/__init__.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# lazy import reporters to minimize startup time + + +def HostedGraphiteReporter(*args, **kwargs): + from .hosted_graphite_reporter import HostedGraphiteReporter as cls + return cls(*args, **kwargs) + + +def CarbonReporter(*args, **kwargs): + from .carbon_reporter import CarbonReporter as cls + return cls(*args, **kwargs) + + +def UdpCarbonReporter(*args, **kwargs): + from .carbon_reporter import UdpCarbonReporter as cls + return cls(*args, **kwargs) + + +def ConsoleReporter(*args, **kwargs): + from .console_reporter import ConsoleReporter as cls + return cls(*args, **kwargs) + + +def CsvReporter(*args, **kwargs): + from .csv_reporter import CsvReporter as cls + return cls(*args, **kwargs) diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/carbon_reporter.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/carbon_reporter.py new file mode 100644 index 00000000000..e0caef6bd3d --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/carbon_reporter.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +import socket +import sys + +from .reporter import Reporter + +DEFAULT_CARBON_SERVER = '0.0.0.0' +DEFAULT_CARBON_PORT = 2003 + + +class CarbonReporter(Reporter): + + """ + Carbon is the network daemon to collect metrics for Graphite + """ + + def __init__(self, registry=None, reporting_interval=5, prefix="", + server=DEFAULT_CARBON_SERVER, port=DEFAULT_CARBON_PORT, socket_factory=socket.socket, + clock=None): + super(CarbonReporter, self).__init__( + registry, reporting_interval, clock) + self.prefix = prefix + self.server = server + self.port = port + self.socket_factory = socket_factory + + def report_now(self, registry=None, timestamp=None): + metrics = self._collect_metrics(registry or self.registry, timestamp) + if metrics: + # XXX: keep connection open + sock = self.socket_factory() + sock.connect((self.server, self.port)) + + if sys.version_info[0] > 2: + sock.sendall(metrics.encode()) + else: + sock.sendall(metrics) + + sock.close() + + def _collect_metrics(self, registry, timestamp=None): + timestamp = timestamp or int(round(self.clock.time())) + metrics = registry.dump_metrics() + metrics_data = [] + for key in metrics.keys(): + for value_key in metrics[key].keys(): + metricLine = "%s%s.%s %s %s\n" % ( + self.prefix, key, value_key, metrics[key][value_key], timestamp) + metrics_data.append(metricLine) + return ''.join(metrics_data) + + +class UdpCarbonReporter(CarbonReporter): + + """ + The default CarbonReporter uses TCP. + This sub-class uses UDP instead which might be unreliable but it is faster + """ + + def report_now(self, registry=None, timestamp=None): + metrics = self._collect_metrics(registry or self.registry, timestamp) + if metrics: + sock = self.socket_factory(socket.AF_INET, socket.SOCK_DGRAM) + + if sys.version_info[0] > 2: + sock.sendto(metrics.encode(), (self.server, self.port)) + else: + sock.sendto(metrics, (self.server, self.port)) diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/console_reporter.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/console_reporter.py new file mode 100644 index 00000000000..2d18727372d --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/console_reporter.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +import sys +import datetime +from .reporter import Reporter + + +class ConsoleReporter(Reporter): + + """ + Show metrics in a human readable form. + This is useful for debugging if you want to read the current state on the console. + """ + + def __init__(self, registry=None, reporting_interval=30, stream=sys.stderr, clock=None): + super(ConsoleReporter, self).__init__( + registry, reporting_interval, clock) + self.stream = stream + + def report_now(self, registry=None, timestamp=None): + metrics = self._collect_metrics(registry or self.registry, timestamp) + for line in metrics: + print(line, file=self.stream) + + def _collect_metrics(self, registry, timestamp=None): + timestamp = timestamp or int(round(self.clock.time())) + dt = datetime.datetime(1970, 1, 1) + \ + datetime.timedelta(seconds=timestamp) + metrics = registry.dump_metrics() + metrics_data = ["== %s ===================================" % + dt.strftime("%Y-%m-%d %H:%M:%S")] + for key in metrics.keys(): + values = metrics[key] + metrics_data.append("%s:" % key) + for value_key in values.keys(): + metrics_data.append( + "%20s = %s" % (value_key, values[value_key])) + metrics_data.append("") + return metrics_data diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/csv_reporter.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/csv_reporter.py new file mode 100644 index 00000000000..e7807e72ba5 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/csv_reporter.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +import sys +import os +import datetime +from .reporter import Reporter + + +class CsvReporter(Reporter): + + """ + Show metrics in comma-separated-files. + Each metrics gets its own file + """ + + def __init__(self, registry=None, reporting_interval=30, path=None, separator="\t", clock=None): + super(CsvReporter, self).__init__( + registry, reporting_interval, clock) + self.path = path or os.getcwd() + if not os.path.exists(self.path): + os.makedirs(self.path) + self.separator = separator + self.files = {} + + def report_now(self, registry=None, timestamp=None): + self._save_metrics(registry or self.registry, timestamp) + + def _save_metrics(self, registry, timestamp=None): + timestamp = timestamp or int(round(self.clock.time())) + dt = datetime.datetime(1970, 1, 1) + \ + datetime.timedelta(seconds=timestamp) + date = dt.strftime("%Y-%m-%d %H:%M:%S") + metrics = registry.dump_metrics() + for key in metrics.keys(): + values = metrics[key] + value_keys = list(sorted(values.keys())) + target = os.path.join(self.path, "%s.csv" % key) + f = self.files.get(target, None) + if f is None: + if not os.path.exists(target): + f = open(target, "w") + f.write("%s\n" % self.separator.join(["timestamp"] + value_keys)) + else: + f = open(target, "a") + self.files[target] = f + cols = [date] + for vk in value_keys: + cols.append(values[vk]) + f.write("%s\n" % self.separator.join(map(str, cols))) + f.flush() diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/hosted_graphite_reporter.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/hosted_graphite_reporter.py new file mode 100644 index 00000000000..b72a4d284ea --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/hosted_graphite_reporter.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +import sys +import urllib2 +import base64 + +from .meters import Counter, Histogram, Meter, Timer +from .registry import MetricsRegistry +from .reporter import Reporter + + +class HostedGraphiteReporter(Reporter): + + """ + This reporter requires an api key from http://www.hostedgraphite.com + """ + + def __init__( + self, hosted_graphite_api_key, registry=None, reporting_interval=10, url="https://hostedgraphite.com/api/v1/sink", + clock=None): + super(HostedGraphiteReporter, self).__init__( + registry, reporting_interval, clock) + self.url = url + self.api_key = hosted_graphite_api_key + + def report_now(self, registry=None, timestamp=None): + metrics = self._collect_metrics(registry or self._registry, timestamp) + if metrics: + try: + # XXX: better use http-keepalive/pipelining somehow? + request = urllib2.Request(self.url, metrics) + request.add_header("Authorization", "Basic %s" % + base64.encodestring(self.api_key).strip()) + result = urllib2.urlopen(request) + except Exception as e: + print(e, file=sys.stderr) + + def _collect_metrics(self, registry, timestamp=None): + timestamp = timestamp or int(round(self.clock.time())) + metrics = registry.dump_metrics() + metrics_data = [] + for key in metrics.keys(): + for value_key in metrics[key].keys(): + metric_line = "%s.%s %s %s\n" % ( + key, value_key, metrics[key][value_key], timestamp) + metrics_data.append(metric_line) + return ''.join(metrics_data) diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/reporter.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/reporter.py new file mode 100644 index 00000000000..5c2fc03cc5f --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/reporter.py @@ -0,0 +1,45 @@ +import time +from threading import Thread, Event +from ..registry import global_registry, get_qualname + + +class Reporter(object): + + def create_thread(self): + # noinspection PyAttributeOutsideInit + self._loop_thread = Thread(target=self._loop, name="pyformance reporter {0}".format(get_qualname(type(self)))) + self._loop_thread.setDaemon(True) + + def __init__(self, registry=None, reporting_interval=30, clock=None): + self.registry = registry or global_registry() + self.reporting_interval = reporting_interval + self.clock = clock or time + self._stopped = Event() + self.create_thread() + + def start(self): + if self._stopped.is_set(): + return False + + r = str(self._loop_thread) + if "stopped" in r: + # has to be recreated in a celery worker + self.create_thread() + elif "started" in r: + # already started + return False + + self._loop_thread.start() + return True + + def stop(self): + self._stopped.set() + + def _loop(self): + while not self._stopped.is_set(): + self.report_now(self.registry) + time.sleep(self.reporting_interval) + # self._stopped.clear() + + def report_now(self, registry=None, timestamp=None): + raise NotImplementedError(self.report_now) diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/__init__.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/__init__.py new file mode 100644 index 00000000000..486f658147c --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/__init__.py @@ -0,0 +1,3 @@ +from .samples import ExpDecayingSample +from .moving_average import ExpWeightedMovingAvg +from .snapshot import Snapshot diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/moving_average.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/moving_average.py new file mode 100644 index 00000000000..0fa57076e95 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/moving_average.py @@ -0,0 +1,69 @@ +import math +import time + + +class ExpWeightedMovingAvg(object): + + """ + An exponentially-weighted moving average. + """ + INTERVAL = 5.0 # seconds + SECONDS_PER_MINUTE = 60.0 + + def __init__(self, period, interval=INTERVAL, clock=time): + """ + Create a new EWMA with a specific smoothing constant. + + :type period: C{int} + :param period: the time in minutes it takes to reach a given significance level + :type interval: C{int} + :param interval: the expected tick interval, defaults to 5s + """ + super(ExpWeightedMovingAvg, self).__init__() + self.clock = clock + self.uncounted = 0.0 + self.interval = interval + self.rate = -1 + self.period = period * ExpWeightedMovingAvg.SECONDS_PER_MINUTE + self.last_tick = self.clock.time() + + def get_rate(self): + if self.clock.time() - self.last_tick >= self.interval: + self.tick() + if self.rate >= 0: + return self.rate + return 0 + + def add(self, value): + self.uncounted += value + + def tick(self): + """ + Mark the passage of time and decay the current rate accordingly. + """ + prev = self.last_tick + now = self.clock.time() + interval = now - prev + if interval <= 0: + return + + instant_rate = self.uncounted / interval + self.uncounted = 0.0 + + if self.rate >= 0: + self.rate += (self._alpha(interval) * (instant_rate - self.rate)) + else: + self.rate = instant_rate + + self.last_tick = now + + def _alpha(self, interval): + """ + Calculate the alpha based on the time since the last tick. This is + necessary because a single threaded Python program loses precision + under high load, so we can't assume a consistant I{EWMA._interval}. + + :type interval: C{float} + :param interval: the interval we use to calculate the alpha + """ + return 1 - math.exp(-interval / self.period) diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/samples.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/samples.py new file mode 100644 index 00000000000..618b4c3ca45 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/samples.py @@ -0,0 +1,112 @@ +import time +import random +import math +import heapq +from .snapshot import Snapshot + +DEFAULT_SIZE = 1028 +DEFAULT_ALPHA = 0.015 + + +class ExpDecayingSample(object): + + """ + An exponentially-decaying random sample of longs. Uses Cormode et al's + forward-decaying priority reservoir sampling method to produce a + statistically representative sample, exponentially biased towards newer + entries. + + @see: + Cormode et al. Forward Decay: A Practical Time Decay Model for + Streaming Systems. ICDE '09: Proceedings of the 2009 IEEE + International Conference on Data Engineering (2009) + """ + + RESCALE_THREASHOLD = 3600.0 # 1 hour + + def __init__(self, size=DEFAULT_SIZE, alpha=DEFAULT_ALPHA, clock=time): + """ + Creates a new L{ExponentiallyDecayingSample}. + + :type size: C{int} + :param size: the number of samples to keep in the sampling reservoir + :type alpha: C{float} + :param alpha: the exponential decay factor; the higher this is, the more + biased the sample will be towards newer values + :type clock: C{function} + :param clock: the function used to return the current time, default to + seconds since the epoch; to be used with other time + units, or with the twisted clock for our testing purposes + """ + super(ExpDecayingSample, self).__init__() + self.clock = clock + self.size = size + self.alpha = alpha + self.clear() + + def clear(self): + self.values = {} + self.priorities = [] + self.counter = 0 + self.start_time = self.clock.time() + self.next_time = self.clock.time() + \ + ExpDecayingSample.RESCALE_THREASHOLD + + def get_size(self): + return self.counter if self.counter < self.size else self.size + + def update(self, value): + """ + Adds a value to the sample. + + :type value: C{int} or C{float} + :param value: the value to be added + """ + if self.size == 0: + return + self._rescale_if_necessary() + priority = self._weight( + self.clock.time() - self.start_time) / random.random() + new_counter = self.counter + 1 + self.counter = new_counter + + if new_counter <= self.size: + self.values[priority] = value + heapq.heappush(self.priorities, priority) + else: + first = heapq.heappop(self.priorities) + if first < priority: + if priority not in self.values: + self.values[priority] = value + heapq.heappush(self.priorities, priority) + while first not in self.values: + first = heapq.heappop(self.priorities) + del self.values[first] + else: + heapq.heappush(self.priorities, first) + + def _rescale_if_necessary(self): + if self.clock.time() >= self.next_time: + self._rescale() + + def _rescale(self): + self.next_time = self.clock.time() + \ + ExpDecayingSample.RESCALE_THREASHOLD + old_start_time = self.start_time + self.start_time = self.clock.time() + new_values = {} + new_priorities = [] + for key, val in self.values.items(): + priority = key * \ + math.exp(-self.alpha * (self.start_time - old_start_time)) + new_values[priority] = val + heapq.heappush(new_priorities, priority) + self.values = new_values + self.priorities = new_priorities + self.counter = len(self.values) + + def _weight(self, value): + return math.exp(self.alpha * value) + + def get_snapshot(self): + return Snapshot(self.values.values()) diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/snapshot.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/snapshot.py new file mode 100644 index 00000000000..ac9e693353f --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/snapshot.py @@ -0,0 +1,59 @@ +class Snapshot(object): + + """ + This class is used by the histogram meter + """ + + MEDIAN = 0.5 + P75_Q = 0.75 + P95_Q = 0.95 + P99_Q = 0.99 + P999_Q = 0.999 + + def __init__(self, values): + super(Snapshot, self).__init__() + self.values = sorted(values) + + def get_size(self): + "get current size" + return len(self.values) + + def get_median(self): + "get current median" + return self.get_percentile(Snapshot.MEDIAN) + + def get_75th_percentile(self): + "get current 75th percentile" + return self.get_percentile(Snapshot.P75_Q) + + def get_95th_percentile(self): + "get current 95th percentile" + return self.get_percentile(Snapshot.P95_Q) + + def get_99th_percentile(self): + "get current 99th percentile" + return self.get_percentile(Snapshot.P99_Q) + + def get_999th_percentile(self): + "get current 999th percentile" + return self.get_percentile(Snapshot.P999_Q) + + def get_percentile(self, percentile): + """ + get custom percentile + + :param percentile: float value between 0 and 1 + """ + if percentile < 0 or percentile > 1: + raise ValueError("{0} is not in [0..1]".format(percentile)) + length = len(self.values) + if length == 0: + return 0 + pos = percentile * (length + 1) + if pos < 1: + return self.values[0] + if pos >= length: + return self.values[-1] + lower = self.values[int(pos) - 1] + upper = self.values[int(pos)] + return lower + (pos - int(pos)) * (upper - lower) diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyproject.toml b/desktop/core/ext-py3/pyformance-0.3.2/pyproject.toml new file mode 100644 index 00000000000..9787c3bdf00 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/desktop/core/ext-py3/pyformance-0.3.2/setup.py b/desktop/core/ext-py3/pyformance-0.3.2/setup.py new file mode 100644 index 00000000000..7898dc38e10 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/setup.py @@ -0,0 +1,22 @@ +# +# pyformance 0.3.2 (https://github.com/omergertel/pyformance), vendored for Hue. +# +# Sources are the pyformance 0.3.2 release unchanged, except pyformance/__init__.py: +# its pkg_resources.declare_namespace() call is removed (see the comment there). +# +from setuptools import setup, find_packages + +setup( + name="pyformance", + version="0.3.2", + author="Omer Getrel", + author_email="omer.gertel@gmail.com", + description="Performance metrics, based on Coda Hale's Yammer metrics", + license="Apache 2.0", + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python :: 2.7", + ], + packages=find_packages(), +) diff --git a/desktop/core/generate_requirements.py b/desktop/core/generate_requirements.py index 680e78e4ba7..02ddecc2be0 100755 --- a/desktop/core/generate_requirements.py +++ b/desktop/core/generate_requirements.py @@ -39,6 +39,8 @@ def __init__(self): "boto-2.49.0", # Vendored to relax the hard Mako==1.2.3 pin of djangomako 1.3.2 (see its setup.py). "djangomako-1.3.2", + # Vendored without its pkg_resources.declare_namespace() call, which setuptools >= 82 can no longer serve. + "pyformance-0.3.2", "pysaml2-7.3.1", "python-sasl-0.3.1", ] @@ -54,7 +56,7 @@ def __init__(self): # also drops the remaining emSign roots. It is a transitive dependency of requests; pin it so the CA # bundle never depends on whatever is newest at build time. "certifi==2026.10.23", - # cryptography 46.x requires cffi >= 2.0.0. Held at 2.0.0 because 2.1.0 needs Python >= 3.10. + # cryptography requires cffi >= 2.0.0. Held at 2.0.0 because 2.1.0 needs Python >= 3.10. "cffi==2.0.0", "configobj==5.0.9", "cryptography==46.0.7", @@ -97,7 +99,6 @@ def __init__(self): "psutil==5.8.0", "pyarrow==23.0.1", "pydantic==2.10.6", - "pyformance==0.3.2", "PyJWT==2.15.1", # Held at 26.0.0: 26.4.0 drops X509Req, which pysaml2 still needs. Caps cryptography at <47. "pyOpenSSL==26.0.0", @@ -135,7 +136,7 @@ def __init__(self): "numpy==1.24.4", "pandas==2.0.3", "pure-sasl==0.6.2", - "setuptools==80.9.0", + "setuptools==84.1.0", "sqlparse==0.6.0", ] diff --git a/desktop/core/generate_requirements_test.py b/desktop/core/generate_requirements_test.py index 17dcad0aed6..951f365492e 100644 --- a/desktop/core/generate_requirements_test.py +++ b/desktop/core/generate_requirements_test.py @@ -49,7 +49,7 @@ def setup_method(self): os.makedirs(self.ext_py3_dir) # Create empty placeholder files that simulate local requirement packages - for req in ["boto-2.49.0", "djangomako-1.3.2", "pysaml2-7.3.1", "python-sasl-0.3.1"]: + for req in ["boto-2.49.0", "djangomako-1.3.2", "pyformance-0.3.2", "pysaml2-7.3.1", "python-sasl-0.3.1"]: with open(os.path.join(self.ext_py3_dir, req), "w") as f: f.write("") diff --git a/desktop/core/src/desktop/appmanager.py b/desktop/core/src/desktop/appmanager.py index 05464a36140..e754095abda 100644 --- a/desktop/core/src/desktop/appmanager.py +++ b/desktop/core/src/desktop/appmanager.py @@ -15,14 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os import re import sys -import logging import traceback from builtins import object +from importlib.metadata import entry_points -import pkg_resources from django.utils.translation import gettext as _ import desktop @@ -56,7 +56,7 @@ def _import_module_or_none(module): try: __import__(module) return sys.modules[module] - except ImportError as ie: + except ImportError: # If the exception came from us importing, we want to just # return None. We need to inspect the stack, though, so we properly # reraise in the case that the module we're importing triggered @@ -78,7 +78,7 @@ class DesktopModuleInfo(object): A Desktop app is encapsulated by a module. These modules are found using the "desktop.sdk.application" - entrypoint (from pkg_resources/setup.py). + entrypoint (declared in setup.py). Each desktop app may have a settings file, which lists DJANGO_APPS to be installed as @@ -184,7 +184,7 @@ def get_bootstrap_file(self): if self.static_dir: f = os.path.join(self.static_dir, "bootstrap.js") if os.path.exists(f): - return file(f) + return open(f) else: return None @@ -236,7 +236,7 @@ def load_libs(): raise Exception("load_apps already has been called.") DESKTOP_LIBS = [] - for lib in pkg_resources.iter_entry_points("desktop.sdk.lib"): + for lib in entry_points(group="desktop.sdk.lib"): m = lib.load() DESKTOP_LIBS.append(DesktopModuleInfo(m)) @@ -260,7 +260,7 @@ def load_apps(app_blacklist): raise Exception(_("load_apps has already been called.")) DESKTOP_APPS = [] - for sdk_app in pkg_resources.iter_entry_points("desktop.sdk.application"): + for sdk_app in entry_points(group="desktop.sdk.application"): if desktop.conf.CSP_NONCE.get(): # This Apps are not supported CSP Nonce is enabled. diff --git a/desktop/core/src/desktop/lib/gunicorn_server_utils.py b/desktop/core/src/desktop/lib/gunicorn_server_utils.py index 7dbca37bcdd..0d9bdde06f7 100644 --- a/desktop/core/src/desktop/lib/gunicorn_server_utils.py +++ b/desktop/core/src/desktop/lib/gunicorn_server_utils.py @@ -29,9 +29,9 @@ import sys import tempfile import time +from importlib.metadata import version import gunicorn.app.base -import pkg_resources import psutil import redis from django.core.wsgi import get_wsgi_application @@ -90,7 +90,7 @@ def worker_int(worker): def enable_logging(args, options): """Enable basic logging for the process""" - HUE_DESKTOP_VERSION = pkg_resources.get_distribution("desktop").version or "Unknown" + HUE_DESKTOP_VERSION = version("desktop") or "Unknown" # Start basic logging as soon as possible. if "HUE_PROCESS_NAME" not in os.environ: _proc = os.path.basename(len(sys.argv) > 1 and sys.argv[1] or sys.argv[0]) diff --git a/desktop/core/src/desktop/settings.py b/desktop/core/src/desktop/settings.py index 356041d8df2..2d071439181 100644 --- a/desktop/core/src/desktop/settings.py +++ b/desktop/core/src/desktop/settings.py @@ -28,8 +28,8 @@ import sys import uuid from builtins import map, zip +from importlib.metadata import version -import pkg_resources from django.utils.translation import gettext_lazy as _ import desktop.redaction @@ -44,7 +44,7 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__)), '..', '..', '..')) -HUE_DESKTOP_VERSION = pkg_resources.get_distribution("desktop").version or "Unknown" +HUE_DESKTOP_VERSION = version("desktop") or "Unknown" NICE_NAME = "Hue" ENV_HUE_PROCESS_NAME = "HUE_PROCESS_NAME" diff --git a/desktop/core/src/desktop/supervisor.py b/desktop/core/src/desktop/supervisor.py index b77b26662ab..e59a0677496 100644 --- a/desktop/core/src/desktop/supervisor.py +++ b/desktop/core/src/desktop/supervisor.py @@ -32,26 +32,25 @@ from __future__ import print_function -import os import grp -import pwd -import sys -import time -import signal import logging import optparse -import threading +import os +import pwd +import signal import subprocess +import sys +import threading +import time from builtins import object, range +from importlib.metadata import entry_points import daemon -import pkg_resources -from daemon.daemon import DaemonContext from daemon.pidfile import TimeoutPIDLockFile -import desktop.log -import desktop.lib.paths import desktop.lib.daemon_utils +import desktop.lib.paths +import desktop.log PROC_NAME = 'supervisor' LOG = logging.getLogger() @@ -164,7 +163,7 @@ def run(self): return LOG.error("Process %s exited abnormally. Restarting it." % (proc_str,)) - except BaseException as ex: + except BaseException: LOG.exception("Uncaught exception. Supervisor exiting.") self.state = Supervisor.ERROR @@ -229,7 +228,7 @@ def get_pid_cmdline(pid): def get_supervisees(): """Pull the supervisor specifications out of the entry point.""" - eps = list(pkg_resources.iter_entry_points(ENTRY_POINT_GROUP)) + eps = list(entry_points(group=ENTRY_POINT_GROUP)) return dict((ep.name, ep.load()) for ep in eps) @@ -366,7 +365,7 @@ def main(): sups.append(sup) wait_loop(sups, options) - except BaseException as ex: + except BaseException: LOG.exception("Exception in supervisor main loop") shutdown(sups) # shutdown() exits the process diff --git a/desktop/libs/notebook/src/notebook/connectors/sql_alchemy.py b/desktop/libs/notebook/src/notebook/connectors/sql_alchemy.py index db95cadf907..ba2900269fe 100644 --- a/desktop/libs/notebook/src/notebook/connectors/sql_alchemy.py +++ b/desktop/libs/notebook/src/notebook/connectors/sql_alchemy.py @@ -56,11 +56,13 @@ import re import textwrap import uuid +from importlib.metadata import entry_points from string import Template from urllib.parse import parse_qs as urllib_parse_qs, quote_plus as urllib_quote_plus, urlparse as urllib_urlparse from past.builtins import long from sqlalchemy import create_engine, inspect, MetaData, Table +from sqlalchemy.dialects import registry from sqlalchemy.exc import CompileError, NoSuchTableError, OperationalError, ProgrammingError, UnsupportedCompilationError from desktop.lib.i18n import force_unicode @@ -76,6 +78,26 @@ LOG = logging.getLogger() +_BUILTIN_DIALECT_LOADER = registry.auto_fn + + +def _load_dialect(name): + ''' + SQLAlchemy 1.3 finds third-party dialects (trino://, phoenix://...) via pkg_resources, which setuptools >= 82 + no longer ships. Look up the same "sqlalchemy.dialects" entry points with importlib.metadata instead, after + the built-in dialects, in the order SQLAlchemy uses. + ''' + loader = _BUILTIN_DIALECT_LOADER(name) + if loader: + return loader + + entry_point = next(iter(entry_points(group='sqlalchemy.dialects', name=name)), None) + return entry_point.load if entry_point else None + + +registry.auto_fn = _load_dialect + + def query_error_handler(func): def decorator(*args, **kwargs): try: diff --git a/desktop/libs/notebook/src/notebook/connectors/sql_alchemy_tests.py b/desktop/libs/notebook/src/notebook/connectors/sql_alchemy_tests.py index 20cab695a22..08c70d3427c 100644 --- a/desktop/libs/notebook/src/notebook/connectors/sql_alchemy_tests.py +++ b/desktop/libs/notebook/src/notebook/connectors/sql_alchemy_tests.py @@ -16,19 +16,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import logging from builtins import object from unittest.mock import MagicMock, Mock, patch import pytest +from sqlalchemy.dialects import registry from sqlalchemy.exc import UnsupportedCompilationError -from sqlalchemy.types import ARRAY, JSON, VARCHAR, NullType +from sqlalchemy.types import ARRAY, JSON, NullType, VARCHAR from desktop.auth.backend import rewrite_user from desktop.lib.django_test_util import make_logged_in_client from notebook.connectors.base import AuthenticationRequired -from notebook.connectors.sql_alchemy import Assist, SqlAlchemyApi +from notebook.connectors.sql_alchemy import _load_dialect, Assist, SqlAlchemyApi from useradmin.models import User LOG = logging.getLogger() @@ -74,7 +74,7 @@ def test_create_athena_engine(self): } } - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: + with patch('notebook.connectors.sql_alchemy.create_engine'): SqlAlchemyApi(self.user, interpreter)._create_engine() def test_fetch_result_empty(self): @@ -141,7 +141,7 @@ def test_create_engine_auth_error(self): } } - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: + with patch('notebook.connectors.sql_alchemy.create_engine'): with pytest.raises(AuthenticationRequired): SqlAlchemyApi(self.user, interpreter)._create_engine() @@ -165,7 +165,7 @@ def test_create_engine_auth(self): } } - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: + with patch('notebook.connectors.sql_alchemy.create_engine'): SqlAlchemyApi(self.user, interpreter)._create_engine() def test_create_connection_error(self): @@ -176,7 +176,7 @@ def test_create_connection_error(self): } } - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: + with patch('notebook.connectors.sql_alchemy.create_engine'): with pytest.raises(AuthenticationRequired): engine = SqlAlchemyApi(self.user, interpreter)._create_engine() SqlAlchemyApi(self.user, interpreter)._create_connection(engine) @@ -201,7 +201,7 @@ def test_create_connection(self): } } - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: + with patch('notebook.connectors.sql_alchemy.create_engine'): engine = SqlAlchemyApi(self.user, interpreter)._create_engine() SqlAlchemyApi(self.user, interpreter)._create_connection(engine) @@ -216,14 +216,14 @@ def test_create_engine_with_impersonation(self): } with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: - engine = SqlAlchemyApi(self.user, interpreter)._create_engine() + SqlAlchemyApi(self.user, interpreter)._create_engine() create_engine.assert_called_with('presto://hue:8080/hue', pool_pre_ping=True) interpreter['options']['has_impersonation'] = True # On with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: - engine = SqlAlchemyApi(self.user, interpreter)._create_engine() + SqlAlchemyApi(self.user, interpreter)._create_engine() create_engine.assert_called_with('presto://hue:8080/hue', connect_args={'principal_username': 'test'}, @@ -240,14 +240,14 @@ def test_create_engine_with_impersonation_phoenix(self): } with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: - engine = SqlAlchemyApi(self.user, interpreter)._create_engine() + SqlAlchemyApi(self.user, interpreter)._create_engine() create_engine.assert_called_with('phoenix://hue:8080/hue', pool_pre_ping=False) interpreter['options']['has_impersonation'] = True # On with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: - engine = SqlAlchemyApi(self.user, interpreter)._create_engine() + SqlAlchemyApi(self.user, interpreter)._create_engine() create_engine.assert_called_with('phoenix://test@hue:8080/hue', pool_pre_ping=False) @@ -275,7 +275,7 @@ def test_explain(self): def test_check_status(self): notebook = Mock() - with patch('notebook.connectors.sql_alchemy.CONNECTIONS') as CONNECTIONS: + with patch('notebook.connectors.sql_alchemy.CONNECTIONS'): snippet = {'result': {'handle': {'guid': 'guid-1', 'has_result_set': False}}} response = SqlAlchemyApi(self.user, self.interpreter).check_status(notebook, snippet) @@ -288,9 +288,9 @@ def test_check_status(self): def test_get_sample_data(self): snippet = Mock() - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: + with patch('notebook.connectors.sql_alchemy.create_engine'): with patch('notebook.connectors.sql_alchemy.Assist.get_sample_data') as get_sample_data: - with patch('notebook.connectors.sql_alchemy.inspect') as inspect: + with patch('notebook.connectors.sql_alchemy.inspect'): get_sample_data.return_value = (['col1'], [[1], [2]]) response = SqlAlchemyApi(self.user, self.interpreter).get_sample_data(snippet) @@ -303,8 +303,8 @@ def test_get_sample_data(self): def test_get_tables(self): snippet = MagicMock() - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: - with patch('notebook.connectors.sql_alchemy.inspect') as inspect: + with patch('notebook.connectors.sql_alchemy.create_engine'): + with patch('notebook.connectors.sql_alchemy.inspect'): with patch('notebook.connectors.sql_alchemy.Assist.get_table_names') as get_table_names: with patch('notebook.connectors.sql_alchemy.Assist.get_view_names') as get_view_names: get_table_names.return_value = ['table1'] @@ -319,9 +319,9 @@ def test_get_tables(self): def test_get_sample_data_table(self): snippet = Mock() - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: + with patch('notebook.connectors.sql_alchemy.create_engine'): with patch('notebook.connectors.sql_alchemy.Assist.get_sample_data') as get_sample_data: - with patch('notebook.connectors.sql_alchemy.inspect') as inspect: + with patch('notebook.connectors.sql_alchemy.inspect'): get_sample_data.return_value = (['col1'], [[1], [2]]) response = SqlAlchemyApi(self.user, self.interpreter).get_sample_data(snippet, database='database1', table='table1') @@ -349,7 +349,7 @@ def test_dialect_trim_statement_semicolon(self): snippet = {'statement': 'SELECT 1;', 'result': {}} # Trim - engine = SqlAlchemyApi(self.user, interpreter).execute(notebook, snippet) + SqlAlchemyApi(self.user, interpreter).execute(notebook, snippet) execute.assert_called_with('SELECT 1') @@ -357,7 +357,7 @@ def test_dialect_trim_statement_semicolon(self): interpreter['options']['url'] = 'mysql://hue:3306/hue' interpreter['dialect_properties']['sql_identifier_quote'] = '`' - engine = SqlAlchemyApi(self.user, interpreter).execute(notebook, snippet) + SqlAlchemyApi(self.user, interpreter).execute(notebook, snippet) execute.assert_called_with('SELECT 1') @@ -423,8 +423,8 @@ def test_empty_database_names(self): } snippet = MagicMock() - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: - with patch('notebook.connectors.sql_alchemy.inspect') as inspect: + with patch('notebook.connectors.sql_alchemy.create_engine'): + with patch('notebook.connectors.sql_alchemy.inspect'): with patch('notebook.connectors.sql_alchemy.Assist') as Assist: Assist.return_value = Mock(get_databases=Mock(return_value=['SYSTEM', None])) @@ -439,8 +439,8 @@ def test_columns_with_null_type(self): } snippet = MagicMock() - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: - with patch('notebook.connectors.sql_alchemy.inspect') as inspect: + with patch('notebook.connectors.sql_alchemy.create_engine'): + with patch('notebook.connectors.sql_alchemy.inspect'): with patch('notebook.connectors.sql_alchemy.Assist') as Assist: def col1_dict(key): return { @@ -525,3 +525,29 @@ def test_fix_bigquery_db_prefixes(self): assert api._fix_bigquery_db_prefixes('table') == 'table' assert api._fix_bigquery_db_prefixes('db.table') == 'table' + + +class TestLoadDialect(object): + + def test_registry_uses_load_dialect(self): + assert registry.auto_fn is _load_dialect + + def test_builtin_dialect_before_entry_points(self): + with patch('notebook.connectors.sql_alchemy.entry_points') as entry_points: + loader = _load_dialect('sqlite') + + assert loader().name == 'sqlite' + entry_points.assert_not_called() + + def test_entry_point_dialect(self): + entry_point = Mock() + + with patch('notebook.connectors.sql_alchemy.entry_points', return_value=[entry_point]) as entry_points: + loader = _load_dialect('trino') + + entry_points.assert_called_once_with(group='sqlalchemy.dialects', name='trino') + assert loader is entry_point.load + + def test_unknown_dialect(self): + with patch('notebook.connectors.sql_alchemy.entry_points', return_value=[]): + assert _load_dialect('nosuchdialect') is None From 78656ce456642526bb7c5c0d84909b5373ec00d0 Mon Sep 17 00:00:00 2001 From: Harsh Gupta <42064744+Harshg999@users.noreply.github.com> Date: Fri, 30 Oct 2026 01:47:46 +0530 Subject: [PATCH 2/2] [core] Upgrade cryptography to 50.0.2 and pyOpenSSL to 26.4.0 cryptography 46.0.7 is affected by four advisories (a Bleichenbacher oracle in PKCS#7 EnvelopedData decryption, exponential path building with duplicate self-signed intermediates, wildcard DNS names escaping permittedSubtrees, and a vulnerable OpenSSL in the wheels). The fixes are in 48.0.1, 49.0.0 and 50.0.0. pyOpenSSL was held at 26.0.0 because it caps cryptography below 47. 26.3.0 is the first release that accepts cryptography 49 and 50, so move to 26.4.0. It no longer has X509Req, load_certificate_request or crypto.verify(), which the vendored pysaml2 certificate code used: - OpenSSLWrapper.create_certificate(request=True), which the SAML signing code calls by default, failed with an AttributeError. Build the certificate signing request with cryptography. - create_cert_signed_certificate loaded the request with pyOpenSSL. Load it and issue the certificate with cryptography. - verify() ignored the failure of crypto.verify() and reported every certificate as invalid. Check the signature with cryptography. Add libsaml/cert_tests.py, which creates, signs and verifies certificates through that code. 8 of its 11 tests fail without the change. --- desktop/core/base_requirements.txt | 4 +- .../ext-py3/pysaml2-7.3.1/src/saml2/cert.py | 117 ++++++++++------- desktop/core/generate_requirements.py | 7 +- .../libs/libsaml/src/libsaml/cert_tests.py | 124 ++++++++++++++++++ 4 files changed, 203 insertions(+), 49 deletions(-) create mode 100644 desktop/libs/libsaml/src/libsaml/cert_tests.py diff --git a/desktop/core/base_requirements.txt b/desktop/core/base_requirements.txt index c7fa50587d9..0185fc9f41a 100644 --- a/desktop/core/base_requirements.txt +++ b/desktop/core/base_requirements.txt @@ -9,7 +9,7 @@ celery[redis]==5.4.0 certifi==2026.10.23 cffi==2.0.0 configobj==5.0.9 -cryptography==46.0.7 +cryptography==50.0.2 cx-Oracle==8.3.0 Django==5.2.18 django-auth-ldap==5.3.0 @@ -49,7 +49,7 @@ psutil==5.8.0 pyarrow==23.0.1 pydantic==2.10.6 PyJWT==2.15.1 -pyOpenSSL==26.0.0 +pyOpenSSL==26.4.0 python-daemon==2.2.4 python-dateutil==2.8.2 python-ldap==3.4.5 diff --git a/desktop/core/ext-py3/pysaml2-7.3.1/src/saml2/cert.py b/desktop/core/ext-py3/pysaml2-7.3.1/src/saml2/cert.py index c5f626601ac..45f97b6e020 100644 --- a/desktop/core/ext-py3/pysaml2-7.3.1/src/saml2/cert.py +++ b/desktop/core/ext-py3/pysaml2-7.3.1/src/saml2/cert.py @@ -5,6 +5,10 @@ from os import remove from os.path import join +from cryptography import x509 +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.x509.oid import NameOID from OpenSSL import crypto import dateutil.parser import pytz @@ -24,6 +28,34 @@ class PayloadError(Exception): pass +def _to_bytes(value): + return value if isinstance(value, bytes) else value.encode("utf-8") + + +def _hash_algorithm(name): + """ + Hue change: pyOpenSSL 26 removed X509Req, so certificate requests are built with cryptography, which wants a + hash object instead of a name like "sha256". + """ + try: + return getattr(hashes, name.upper())() + except AttributeError: + raise WrongInput(f"Unsupported hash algorithm: {name}") + + +def _subject_name(cert_info): + return x509.Name( + [ + x509.NameAttribute(NameOID.COUNTRY_NAME, cert_info["country_code"]), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, cert_info["state"]), + x509.NameAttribute(NameOID.LOCALITY_NAME, cert_info["city"]), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, cert_info["organization"]), + x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, cert_info["organization_unit"]), + x509.NameAttribute(NameOID.COMMON_NAME, cert_info["cn"]), + ] + ) + + class OpenSSLWrapper: def __init__(self): pass @@ -134,31 +166,34 @@ def create_certificate( k = crypto.PKey() k.generate_key(crypto.TYPE_RSA, key_length) - # create a self-signed cert - cert = crypto.X509() - - if request: - cert = crypto.X509Req() - if len(cert_info["country_code"]) != 2: raise WrongInput("Country code must be two letters!") - cert.get_subject().C = cert_info["country_code"] - cert.get_subject().ST = cert_info["state"] - cert.get_subject().L = cert_info["city"] - cert.get_subject().O = cert_info["organization"] # noqa: E741 - cert.get_subject().OU = cert_info["organization_unit"] - cert.get_subject().CN = cn - if not request: + + if request: + csr = ( + x509.CertificateSigningRequestBuilder() + .subject_name(_subject_name(cert_info)) + .sign(k.to_cryptography_key(), _hash_algorithm(hash_alg)) + ) + else: + # create a self-signed cert + cert = crypto.X509() + cert.get_subject().C = cert_info["country_code"] + cert.get_subject().ST = cert_info["state"] + cert.get_subject().L = cert_info["city"] + cert.get_subject().O = cert_info["organization"] # noqa: E741 + cert.get_subject().OU = cert_info["organization_unit"] + cert.get_subject().CN = cn cert.set_serial_number(sn) cert.gmtime_adj_notBefore(valid_from) # Valid before present time cert.gmtime_adj_notAfter(valid_to) # 3 650 days cert.set_issuer(cert.get_subject()) - cert.set_pubkey(k) - cert.sign(k, hash_alg) + cert.set_pubkey(k) + cert.sign(k, hash_alg) try: if request: - tmp_cert = crypto.dump_certificate_request(crypto.FILETYPE_PEM, cert) + tmp_cert = csr.public_bytes(serialization.Encoding.PEM) else: tmp_cert = crypto.dump_certificate(crypto.FILETYPE_PEM, cert) tmp_key = None @@ -237,27 +272,24 @@ def create_cert_signed_certificate( :return: String representation of the signed certificate. """ - ca_cert = crypto.load_certificate(crypto.FILETYPE_PEM, sign_cert_str) - ca_key = None - if passphrase is not None: - ca_key = crypto.load_privatekey(crypto.FILETYPE_PEM, sign_key_str, passphrase) - else: - ca_key = crypto.load_privatekey(crypto.FILETYPE_PEM, sign_key_str) - req_cert = crypto.load_certificate_request(crypto.FILETYPE_PEM, request_cert_str) - - cert = crypto.X509() - cert.set_subject(req_cert.get_subject()) - cert.set_serial_number(sn) - cert.gmtime_adj_notBefore(valid_from) - cert.gmtime_adj_notAfter(valid_to) - cert.set_issuer(ca_cert.get_subject()) - cert.set_pubkey(req_cert.get_pubkey()) - cert.sign(ca_key, hash_alg) - - cert_dump = crypto.dump_certificate(crypto.FILETYPE_PEM, cert) - if isinstance(cert_dump, str): - return cert_dump - return cert_dump.decode("utf-8") + ca_cert = x509.load_pem_x509_certificate(_to_bytes(sign_cert_str)) + ca_key = serialization.load_pem_private_key( + _to_bytes(sign_key_str), password=_to_bytes(passphrase) if passphrase is not None else None + ) + req_cert = x509.load_pem_x509_csr(_to_bytes(request_cert_str)) + + now = datetime.datetime.now(datetime.timezone.utc) + cert = ( + x509.CertificateBuilder() + .subject_name(req_cert.subject) + .issuer_name(ca_cert.subject) + .public_key(req_cert.public_key()) + .serial_number(sn) + .not_valid_before(now + datetime.timedelta(seconds=valid_from)) + .not_valid_after(now + datetime.timedelta(seconds=valid_to)) + .sign(ca_key, _hash_algorithm(hash_alg)) + ) + return cert.public_bytes(serialization.Encoding.PEM).decode("utf-8") def verify_chain(self, cert_chain_str_list, cert_str): """ @@ -321,16 +353,13 @@ def verify(self, signing_cert_str, cert_str): if ca_cert.get_subject().CN == cert.get_subject().CN: return False, ("CN may not be equal for CA certificate and the " "signed certificate.") - cert_algorithm = cert.get_signature_algorithm() - cert_algorithm = cert_algorithm.decode("ascii") - cert_str = cert_str.encode("ascii") - - cert_crypto = saml2.cryptography.pki.load_pem_x509_certificate(cert_str) + cert_crypto = saml2.cryptography.pki.load_pem_x509_certificate(cert_str.encode("ascii")) + # Hue change: pyOpenSSL 26 removed crypto.verify(), so check the signature with cryptography. try: - crypto.verify(ca_cert, cert_crypto.signature, cert_crypto.tbs_certificate_bytes, cert_algorithm) + cert_crypto.verify_directly_issued_by(ca_cert.to_cryptography()) return True, "Signed certificate is valid and correctly signed by CA certificate." - except crypto.Error as e: + except (InvalidSignature, ValueError, TypeError) as e: return False, f"Certificate is incorrectly signed: {str(e)}" except Exception as e: return False, f"Certificate is not valid for an unknown reason. {str(e)}" diff --git a/desktop/core/generate_requirements.py b/desktop/core/generate_requirements.py index 02ddecc2be0..d3dbd2c5a4f 100755 --- a/desktop/core/generate_requirements.py +++ b/desktop/core/generate_requirements.py @@ -59,7 +59,7 @@ def __init__(self): # cryptography requires cffi >= 2.0.0. Held at 2.0.0 because 2.1.0 needs Python >= 3.10. "cffi==2.0.0", "configobj==5.0.9", - "cryptography==46.0.7", + "cryptography==50.0.2", "cx-Oracle==8.3.0", "Django==5.2.18", "django-auth-ldap==5.3.0", @@ -100,8 +100,9 @@ def __init__(self): "pyarrow==23.0.1", "pydantic==2.10.6", "PyJWT==2.15.1", - # Held at 26.0.0: 26.4.0 drops X509Req, which pysaml2 still needs. Caps cryptography at <47. - "pyOpenSSL==26.0.0", + # 26.3.0 is the first release that accepts cryptography 49 and 50. It no longer has X509Req, so the certificate + # request code of the vendored pysaml2 uses cryptography directly. + "pyOpenSSL==26.4.0", "python-daemon==2.2.4", "python-dateutil==2.8.2", "python-ldap==3.4.5", diff --git a/desktop/libs/libsaml/src/libsaml/cert_tests.py b/desktop/libs/libsaml/src/libsaml/cert_tests.py new file mode 100644 index 00000000000..5e02d69600a --- /dev/null +++ b/desktop/libs/libsaml/src/libsaml/cert_tests.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# Licensed to Cloudera, Inc. under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. Cloudera, Inc. licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Tests for the certificate handling of the vendored pysaml2 (saml2.cert). + +pyOpenSSL 26 removed X509Req, load_certificate_request and verify(), which the SAML signing code uses to create a +temporary signing certificate. These tests run that flow against the installed pyOpenSSL and cryptography. +""" + +import pytest +from cryptography import x509 +from cryptography.x509.oid import NameOID +from saml2.cert import CertificateError, OpenSSLWrapper, WrongInput + +CA_INFO = { + "cn": "ca.example.com", + "country_code": "se", + "state": "AC", + "city": "Dorotea", + "organization": "Company", + "organization_unit": "Sales", +} +SERVICE_INFO = dict(CA_INFO, cn="service.example.com") + + +def common_name(pem): + certificate = x509.load_pem_x509_certificate(pem if isinstance(pem, bytes) else pem.encode()) + return certificate.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value + + +class TestOpenSSLWrapper: + def setup_method(self): + self.wrapper = OpenSSLWrapper() + self.ca_cert, self.ca_key = self.wrapper.create_certificate(CA_INFO, request=False, key_length=2048) + + def create_signed_certificate(self, **kwargs): + request, _key = self.wrapper.create_certificate(SERVICE_INFO, request=True, key_length=2048) + return self.wrapper.create_cert_signed_certificate(self.ca_cert.decode(), self.ca_key.decode(), request.decode(), **kwargs) + + def test_create_self_signed_certificate(self): + assert common_name(self.ca_cert) == "ca.example.com" + assert self.ca_key.startswith(b"-----BEGIN") + + def test_create_certificate_request(self): + request, key = self.wrapper.create_certificate(SERVICE_INFO, request=True, key_length=2048) + + csr = x509.load_pem_x509_csr(request) + assert csr.is_signature_valid + assert csr.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value == "service.example.com" + assert csr.subject.get_attributes_for_oid(NameOID.COUNTRY_NAME)[0].value == "se" + assert key.startswith(b"-----BEGIN") + + def test_certificate_request_requires_a_two_letter_country_code(self): + with pytest.raises(WrongInput): + self.wrapper.create_certificate(dict(SERVICE_INFO, country_code="swe"), request=True) + + def test_unsupported_hash_algorithm_is_rejected(self): + with pytest.raises(WrongInput): + self.wrapper.create_certificate(SERVICE_INFO, request=True, hash_alg="nosuchhash") + + def test_encrypted_private_key(self): + _cert, key = self.wrapper.create_certificate( + CA_INFO, key_length=2048, cipher_passphrase={"cipher": "aes256", "passphrase": "secret"} + ) + + assert b"ENCRYPTED" in key + + def test_sign_certificate_request_with_ca(self): + signed = self.create_signed_certificate() + + certificate = x509.load_pem_x509_certificate(signed.encode()) + assert common_name(signed) == "service.example.com" + assert certificate.issuer.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value == "ca.example.com" + + def test_sign_certificate_request_with_encrypted_ca_key(self): + ca_cert, ca_key = self.wrapper.create_certificate( + CA_INFO, key_length=2048, cipher_passphrase={"cipher": "aes256", "passphrase": "secret"} + ) + request, _key = self.wrapper.create_certificate(SERVICE_INFO, request=True, key_length=2048) + + signed = self.wrapper.create_cert_signed_certificate(ca_cert.decode(), ca_key.decode(), request.decode(), passphrase="secret") + + assert common_name(signed) == "service.example.com" + + def test_verify_accepts_certificate_signed_by_ca(self): + valid, _message = self.wrapper.verify(self.ca_cert.decode(), self.create_signed_certificate()) + + assert valid + + def test_verify_rejects_certificate_signed_by_another_ca(self): + other_ca_cert, _other_key = self.wrapper.create_certificate(dict(CA_INFO, cn="other.example.com"), key_length=2048) + + valid, message = self.wrapper.verify(other_ca_cert.decode(), self.create_signed_certificate()) + + assert not valid + assert "incorrectly signed" in message + + def test_verify_rejects_certificate_with_same_common_name_as_the_ca(self): + request, _key = self.wrapper.create_certificate(CA_INFO, request=True, key_length=2048) + signed = self.wrapper.create_cert_signed_certificate(self.ca_cert.decode(), self.ca_key.decode(), request.decode()) + + valid, message = self.wrapper.verify(self.ca_cert.decode(), signed) + + assert not valid + assert "CN may not be equal" in message + + def test_certificate_error_is_raised_when_the_key_cannot_be_dumped(self): + with pytest.raises(CertificateError): + self.wrapper.create_certificate(CA_INFO, key_length=2048, cipher_passphrase={"cipher": "nosuchcipher", "passphrase": "secret"})