diff --git a/Makefile b/Makefile index 4a549a9e6d3..c6eed7fb5f4 100644 --- a/Makefile +++ b/Makefile @@ -111,7 +111,7 @@ include Makefile.tarball # Build docs (unused) ################################### test_prep: - @PIP_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt PIP_BUILD_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PYTHON) -m pip install -r $(REQUIREMENT_TEST_FILE) + @PIP_BUILD_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PYTHON) -m pip install -r $(REQUIREMENT_TEST_FILE) ################################### # Build docs (unused) @@ -166,10 +166,10 @@ $(BLD_DIR_ENV)/bin/python: @$(ENV_PYTHON) -m pip install virtualenv-make-relocatable==$(VIRTUAL_ENV_RELOCATABLE_VERSION) @$(eval RELOCATABLE := $(shell which virtualenv-make-relocatable)) @echo "REQUIREMENT_FILE is $(REQUIREMENT_FILE)" - @echo "--- Using build constraints to prevent setuptools 82+ (cx-Oracle compatibility) ---" - @PIP_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt PIP_BUILD_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PIP) install -r $(REQUIREMENT_FILE) + @echo "--- Building from source with setuptools < 82 in the build environment (cx-Oracle compatibility) ---" + @PIP_BUILD_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PIP) install -r $(REQUIREMENT_FILE) @echo "--- Virtual environment setup complete for $(PYTHON_VER) ---" - @PIP_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt PIP_BUILD_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PIP) install $(NAVOPTAPI_WHL) + @PIP_BUILD_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PIP) install $(NAVOPTAPI_WHL) @echo "--- Finished installing $(NAVOPTAPI_WHL) into virtual-env ---" ################################### # Build desktop diff --git a/desktop/core/Makefile b/desktop/core/Makefile index 2f4ed4a55b9..d8463224eed 100644 --- a/desktop/core/Makefile +++ b/desktop/core/Makefile @@ -38,5 +38,5 @@ default:: env-install: $(EXT_ENV_INSTALL) compile @echo '--- Installing $(APP_NAME) into virtual-env' @echo '--- env-install: EXT_ENV_INSTALL: $(EXT_ENV_INSTALL)' - @echo '--- Using build constraints to prevent setuptools 82+ (cx-Oracle compatibility)' - @PIP_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PYTHON) -m pip install -e . + @echo '--- Building with setuptools < 82 in the build environment' + @PIP_BUILD_CONSTRAINT=$(ROOT)/desktop/core/build-constraints.txt $(ENV_PYTHON) -m pip install -e . diff --git a/desktop/core/base_requirements.txt b/desktop/core/base_requirements.txt index df2d0cb5fc5..0185fc9f41a 100644 --- a/desktop/core/base_requirements.txt +++ b/desktop/core/base_requirements.txt @@ -9,7 +9,7 @@ celery[redis]==5.4.0 certifi==2026.10.23 cffi==2.0.0 configobj==5.0.9 -cryptography==46.0.7 +cryptography==50.0.2 cx-Oracle==8.3.0 Django==5.2.18 django-auth-ldap==5.3.0 @@ -48,9 +48,8 @@ protobuf==6.33.5 psutil==5.8.0 pyarrow==23.0.1 pydantic==2.10.6 -pyformance==0.3.2 PyJWT==2.15.1 -pyOpenSSL==26.0.0 +pyOpenSSL==26.4.0 python-daemon==2.2.4 python-dateutil==2.8.2 python-ldap==3.4.5 @@ -80,9 +79,10 @@ Markdown==3.8.1 numpy==1.24.4 pandas==2.0.3 pure-sasl==0.6.2 -setuptools==80.9.0 +setuptools==84.1.0 sqlparse==0.6.0 -e file://${ROOT}/desktop/core/ext-py3/boto-2.49.0 -e file://${ROOT}/desktop/core/ext-py3/djangomako-1.3.2 +-e file://${ROOT}/desktop/core/ext-py3/pyformance-0.3.2 -e file://${ROOT}/desktop/core/ext-py3/pysaml2-7.3.1 -e file://${ROOT}/desktop/core/ext-py3/python-sasl-0.3.1 diff --git a/desktop/core/build-constraints.txt b/desktop/core/build-constraints.txt index 9482f43800f..4e0585f81f8 100644 --- a/desktop/core/build-constraints.txt +++ b/desktop/core/build-constraints.txt @@ -4,6 +4,8 @@ # always builds it from source. Its setup.py imports pkg_resources, which # setuptools removed in 82.0.0. # -# This must be passed as PIP_BUILD_CONSTRAINT: PIP_CONSTRAINT does not apply to -# the isolated environment pip creates for build dependencies, and needs pip >= 25.3. +# This must be passed as PIP_BUILD_CONSTRAINT, which constrains only the isolated +# environment pip creates for build dependencies and needs pip >= 25.3. Do not pass +# it as PIP_CONSTRAINT: that would also constrain the setuptools the virtual env +# installs, which has to be 84.1.0 or later. setuptools<82 diff --git a/desktop/core/ext-py3/pyformance-0.3.2/LICENSE b/desktop/core/ext-py3/pyformance-0.3.2/LICENSE new file mode 100644 index 00000000000..5cd54bafe32 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/LICENSE @@ -0,0 +1,13 @@ +Copyright 2014 Omer Gertel + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/__init__.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/__init__.py new file mode 100644 index 00000000000..04184ab2e97 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/__init__.py @@ -0,0 +1,8 @@ +# Hue: upstream line 1 was `__import__('pkg_resources').declare_namespace(__name__)`. +# pyformance is not a namespace package (setup.py declares none), and setuptools 82 +# removed pkg_resources, so the line is dropped to allow setuptools >= 83 in the venv. + +from .registry import MetricsRegistry, global_registry, set_global_registry +from .registry import timer, counter, meter, histogram, gauge +from .registry import dump_metrics, clear, count_calls, meter_calls, hist_calls, time_calls +from .meters.timer import call_too_long diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/__version__.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/__version__.py new file mode 100644 index 00000000000..f9aa3e11093 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/__version__.py @@ -0,0 +1 @@ +__version__ = "0.3.2" diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/__init__.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/__init__.py new file mode 100644 index 00000000000..fd864657f41 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/__init__.py @@ -0,0 +1,5 @@ +from .counter import Counter +from .meter import Meter +from .histogram import Histogram +from .timer import Timer +from .gauge import Gauge, CallbackGauge, SimpleGauge diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/counter.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/counter.py new file mode 100644 index 00000000000..5c77742fbd3 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/counter.py @@ -0,0 +1,31 @@ +from threading import Lock + + +class Counter(object): + + """ + An incrementing and decrementing metric + """ + + def __init__(self): + super(Counter, self).__init__() + self.lock = Lock() + self.counter = 0 + + def inc(self, val=1): + "increment counter by val (default is 1)" + with self.lock: + self.counter = self.counter + val + + def dec(self, val=1): + "decrement counter by val (default is 1)" + self.inc(-val) + + def get_count(self): + "return current value of counter" + return self.counter + + def clear(self): + "reset counter to 0" + with self.lock: + self.counter = 0 diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/gauge.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/gauge.py new file mode 100644 index 00000000000..75c6f89ad8b --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/gauge.py @@ -0,0 +1,57 @@ +class Gauge(object): + + """ + A base class for reading of a particular. + + For example, to instrument a queue depth: + + class QueueLengthGaguge(Gauge): + def __init__(self, queue): + super(QueueGaguge, self).__init__() + self.queue = queue + + def get_value(self): + return len(self.queue) + + """ + + def get_value(self): + "A subclass of Gauge should implement this method" + raise NotImplementedError() + + +class CallbackGauge(Gauge): + + """ + A Gauge reading for a given callback + """ + + def __init__(self, callback): + "constructor expects a callable" + super(CallbackGauge, self).__init__() + self.callback = callback + + def get_value(self): + "returns the result of callback which is executed each time" + return self.callback() + + +class SimpleGauge(Gauge): + + """ + A gauge which holds values with simple getter- and setter-interface + """ + + def __init__(self, value=float("nan")): + "constructor accepts initial value" + super(SimpleGauge, self).__init__() + self._value = value + + def get_value(self): + "getter returns current value" + return self._value + + def set_value(self, value): + "setter changes current value" + # XXX: add locking? + self._value = value diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/histogram.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/histogram.py new file mode 100644 index 00000000000..474931a6432 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/histogram.py @@ -0,0 +1,93 @@ +import time +import math +from threading import Lock +from ..stats. samples import ExpDecayingSample, DEFAULT_SIZE, DEFAULT_ALPHA + + +class Histogram(object): + + """ + A metric which calculates the distribution of a value. + """ + + def __init__(self, size=DEFAULT_SIZE, alpha=DEFAULT_ALPHA, clock=time): + """ + Creates a new instance of a L{Histogram}. + """ + super(Histogram, self).__init__() + self.lock = Lock() + self.clock = clock + self.sample = ExpDecayingSample(size, alpha, clock) + self.clear() + + def add(self, value): + """ + Add value to histogram + + :type value: float + """ + with self.lock: + self.sample.update(value) + self.counter = self.counter + 1 + self.max = value if value > self.max else self.max + self.min = value if value < self.min else self.min + self.sum = self.sum + value + self._update_var(value) + + def clear(self): + "reset histogram to initial state" + with self.lock: + self.sample.clear() + self.counter = 0.0 + self.max = -2147483647.0 + self.min = 2147483647.0 + self.sum = 0.0 + self.var = [-1.0, 0.0] + + def get_count(self): + "get current value of counter" + return self.counter + + def get_sum(self): + "get current sum" + return self.sum + + def get_max(self): + "get current maximum" + return self.max + + def get_min(self): + "get current minimum" + return self.min + + def get_mean(self): + "get current mean" + if self.counter > 0: + return self.sum / self.counter + return 0 + + def get_stddev(self): + "get current standard deviation" + if self.counter > 0: + return math.sqrt(self.get_var()) + return 0 + + def get_var(self): + "get current variance" + if self.counter > 1: + return self.var[1] / (self.counter - 1) + return 0 + + def get_snapshot(self): + "get snapshot instance which holds the percentiles" + return self.sample.get_snapshot() + + def _update_var(self, value): + old_m, old_s = self.var + new_m, new_s = [0.0, 0.0] + if old_m == -1: + new_m = value + else: + new_m = old_m + ((value - old_m) / self.counter) + new_s = old_s + ((value - old_m) * (value - new_m)) + self.var = [new_m, new_s] diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/meter.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/meter.py new file mode 100644 index 00000000000..3a6b9bcfaf1 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/meter.py @@ -0,0 +1,58 @@ +import time +from threading import Lock +from ..stats.moving_average import ExpWeightedMovingAvg + + +class Meter(object): + + """ + A meter metric which measures mean throughput and one-, five-, and fifteen-minute + exponentially-weighted moving average throughputs. + """ + + def __init__(self, clock=time): + super(Meter, self).__init__() + self.lock = Lock() + self.clock = clock + self.clear() + + def clear(self): + with self.lock: + self.start_time = self.clock.time() + self.counter = 0.0 + self.m1rate = ExpWeightedMovingAvg(period=1, clock=self.clock) + self.m5rate = ExpWeightedMovingAvg(period=5, clock=self.clock) + self.m15rate = ExpWeightedMovingAvg(period=15, clock=self.clock) + + def get_one_minute_rate(self): + return self.m1rate.get_rate() + + def get_five_minute_rate(self): + return self.m5rate.get_rate() + + def get_fifteen_minute_rate(self): + return self.m15rate.get_rate() + + def tick(self): + self.m1rate.tick() + self.m5rate.tick() + self.m15rate.tick() + + def mark(self, value=1): + with self.lock: + self.counter += value + self.m1rate.add(value) + self.m5rate.add(value) + self.m15rate.add(value) + + def get_count(self): + return self.counter + + def get_mean_rate(self): + if self.counter == 0: + return 0 + elapsed = self.clock.time() - self.start_time + return self.counter / elapsed + + def _convertNsRate(self, ratePerNs): + return ratePerNs diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/timer.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/timer.py new file mode 100644 index 00000000000..352a46ce62f --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/meters/timer.py @@ -0,0 +1,119 @@ +import time +try: + from blinker import Namespace +except ImportError: + Namespace = None +from .histogram import Histogram, DEFAULT_SIZE, DEFAULT_ALPHA +from .meter import Meter + +if Namespace is not None: + timer_signals = Namespace() + call_too_long = timer_signals.signal("call_too_long") +else: + call_too_long = None + + +class Timer(object): + + """ + A timer metric which aggregates timing durations and provides duration statistics, plus + throughput statistics via Meter and Histogram. + + """ + + def __init__(self, threshold=None, size=DEFAULT_SIZE, alpha=DEFAULT_ALPHA, clock=time): + super(Timer, self).__init__() + self.meter = Meter(clock=clock) + self.hist = Histogram(clock=clock) + self.threshold = threshold + + def get_count(self): + "get count from internal histogram" + return self.hist.get_count() + + def get_sum(self): + "get sum from internal histogram" + return self.hist.get_sum() + + def get_max(self): + "get max from internal histogram" + return self.hist.get_max() + + def get_min(self): + "get min from internal histogram" + return self.hist.get_min() + + def get_mean(self): + "get mean from internal histogram" + return self.hist.get_mean() + + def get_stddev(self): + "get stddev from internal histogram" + return self.hist.get_stddev() + + def get_var(self): + "get var from internal histogram" + return self.hist.get_var() + + def get_snapshot(self): + "get snapshot from internal histogram" + return self.hist.get_snapshot() + + def get_mean_rate(self): + "get mean rate from internal meter" + return self.meter.get_mean_rate() + + def get_one_minute_rate(self): + "get 1 minut rate from internal meter" + return self.meter.get_one_minute_rate() + + def get_five_minute_rate(self): + "get 5 minute rate from internal meter" + return self.meter.get_five_minute_rate() + + def get_fifteen_minute_rate(self): + "get 15 rate from internal meter" + return self.meter.get_fifteen_minute_rate() + + def _update(self, seconds): + if seconds >= 0: + self.hist.add(seconds) + self.meter.mark() + + def time(self, *args, **kwargs): + """ + Parameters will be sent to signal, if fired. + Returns a timer context instance which can be used from a with-statement. + Without with-statement you have to call the stop method on the context + """ + return TimerContext(self, self.meter.clock, *args, **kwargs) + + def clear(self): + "clear internal histogram and meter" + self.hist.clear() + self.meter.clear() + + +class TimerContext(object): + + def __init__(self, timer, clock, *args, **kwargs): + super(TimerContext, self).__init__() + self.clock = clock + self.timer = timer + self.start_time = self.clock.time() + self.kwargs = kwargs + self.args = args + + def stop(self): + elapsed = self.clock.time() - self.start_time + self.timer._update(elapsed) + if self.timer.threshold and self.timer.threshold < elapsed and call_too_long is not None: + call_too_long.send( + self.timer, elapsed=elapsed, *self.args, **self.kwargs) + return elapsed + + def __enter__(self): + pass + + def __exit__(self, t, v, tb): + self.stop() diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/registry.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/registry.py new file mode 100644 index 00000000000..0155385fcfc --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/registry.py @@ -0,0 +1,370 @@ +import functools +import re +import time +import sys +from .meters import Counter, Histogram, Meter, Timer, Gauge, CallbackGauge, SimpleGauge + + +class MetricsRegistry(object): + + """ + A single interface used to gather metrics on a service. It keeps track of + all the relevant Counters, Meters, Histograms, and Timers. It does not have + a reference back to its service. The service would create a + L{MetricsRegistry} to manage all of its metrics tools. + """ + def __init__(self, clock = time): + """ + Creates a new L{MetricsRegistry} instance. + """ + self._timers = {} + self._meters = {} + self._counters = {} + self._histograms = {} + self._gauges = {} + self._clock = clock + + def add(self, key, metric): + """ + Use this method to manually add custom metric instances to the registry + which are not created with their constructor's default arguments, + e.g. Histograms with a different size. + + :param key: name of the metric + :type key: C{str} + :param metric: instance of Histogram, Meter, Gauge, Timer or Counter + """ + class_map = { + Histogram: self._histograms, + Meter: self._meters, + Gauge: self.gauges, + Timer: self._timers, + Counter: self._counters, + } + for cls, registry in class_map.iteritems(): + if isinstance(metric, cls): + if key in registry: + raise LookupError("Metric %r already registered" % key) + registry[key] = registry + return + raise TypeError("Invalid class. Could not register metric %r" % key) + + def counter(self, key): + """ + Gets a counter based on a key, creates a new one if it does not exist. + + :param key: name of the metric + :type key: C{str} + + :return: L{Counter} + """ + if key not in self._counters: + self._counters[key] = Counter() + return self._counters[key] + + def histogram(self, key): + """ + Gets a histogram based on a key, creates a new one if it does not exist. + + :param key: name of the metric + :type key: C{str} + + :return: L{Histogram} + """ + if key not in self._histograms: + self._histograms[key] = Histogram(clock=self._clock) + return self._histograms[key] + + def gauge(self, key, gauge=None, default=float("nan")): + if key not in self._gauges: + if gauge is None: + gauge = SimpleGauge( + default) # raise TypeError("gauge required for registering") + elif not isinstance(gauge, Gauge): + if not callable(gauge): + raise TypeError("gauge getter not callable") + gauge = CallbackGauge(gauge) + self._gauges[key] = gauge + return self._gauges[key] + + def meter(self, key): + """ + Gets a meter based on a key, creates a new one if it does not exist. + + :param key: name of the metric + :type key: C{str} + + :return: L{Meter} + """ + if key not in self._meters: + self._meters[key] = Meter(clock=self._clock) + return self._meters[key] + + def timer(self, key): + """ + Gets a timer based on a key, creates a new one if it does not exist. + + :param key: name of the metric + :type key: C{str} + + :return: L{Timer} + """ + if key not in self._timers: + self._timers[key] = Timer(clock=self._clock) + return self._timers[key] + + def clear(self): + self._meters.clear() + self._counters.clear() + self._gauges.clear() + self._timers.clear() + self._histograms.clear() + + def _get_counter_metrics(self, key): + if key in self._counters: + counter = self._counters[key] + return {"count": counter.get_count()} + return {} + + def _get_gauge_metrics(self, key): + if key in self._gauges: + gauge = self._gauges[key] + return {"value": gauge.get_value()} + return {} + + def _get_histogram_metrics(self, key): + if key in self._histograms: + histogram = self._histograms[key] + snapshot = histogram.get_snapshot() + res = {"avg": histogram.get_mean(), + "count": histogram.get_count(), + "max": histogram.get_max(), + "min": histogram.get_min(), + "std_dev": histogram.get_stddev(), + "75_percentile": snapshot.get_75th_percentile(), + "95_percentile": snapshot.get_95th_percentile(), + "99_percentile": snapshot.get_99th_percentile(), + "999_percentile": snapshot.get_999th_percentile()} + return res + return {} + + def _get_meter_metrics(self, key): + if key in self._meters: + meter = self._meters[key] + res = {"count": meter.get_count(), + "15m_rate": meter.get_fifteen_minute_rate(), + "5m_rate": meter.get_five_minute_rate(), + "1m_rate": meter.get_one_minute_rate(), + "mean_rate": meter.get_mean_rate()} + return res + return {} + + def _get_timer_metrics(self, key): + if key in self._timers: + timer = self._timers[key] + snapshot = timer.get_snapshot() + res = {"avg": timer.get_mean(), + "sum": timer.get_sum(), + "count": timer.get_count(), + "max": timer.get_max(), + "min": timer.get_min(), + "std_dev": timer.get_stddev(), + "15m_rate": timer.get_fifteen_minute_rate(), + "5m_rate": timer.get_five_minute_rate(), + "1m_rate": timer.get_one_minute_rate(), + "mean_rate": timer.get_mean_rate(), + "75_percentile": snapshot.get_75th_percentile(), + "95_percentile": snapshot.get_95th_percentile(), + "99_percentile": snapshot.get_99th_percentile(), + "999_percentile": snapshot.get_999th_percentile()} + return res + return {} + + def get_metrics(self, key): + """ + Gets all the metrics for a specified key. + + :param key: name of the metric + :type key: C{str} + + :return: C{dict} + """ + metrics = {} + for getter in (self._get_counter_metrics, self._get_histogram_metrics, + self._get_meter_metrics, self._get_timer_metrics, + self._get_gauge_metrics): + metrics.update(getter(key)) + return metrics + + def dump_metrics(self): + """ + Formats all of the metrics and returns them as a dict. + + :return: C{list} of C{dict} of metrics + """ + metrics = {} + for metric_type in (self._counters, + self._histograms, + self._meters, + self._timers, + self._gauges): + for key in metric_type.keys(): + metrics[key] = self.get_metrics(key) + + return metrics + + +class RegexRegistry(MetricsRegistry): + + """ + A single interface used to gather metrics on a service. This class uses a regex to combine + measures that match a pattern. For example, if you have a REST API, instead of defining + a timer for each method, you can use a regex to capture all API calls and group them. + A pattern like '^/api/(?P)/\d+/(?P)?$' will group and measure the following: + /api/users/1 -> users + /api/users/1/edit -> users/edit + /api/users/2/edit -> users/edit + """ + def __init__(self, pattern = None, clock = time): + super(RegexRegistry, self).__init__(clock) + if pattern is not None: + self.pattern = re.compile(pattern) + else: + self.pattern = re.compile('^$') + + def _get_key(self, key): + matches = self.pattern.finditer(key) + key = '/'.join((v for match in matches for v in match.groups() if v)) + return key + + def timer(self, key): + return super(RegexRegistry, self).timer(self._get_key(key)) + + def histogram(self, key): + return super(RegexRegistry, self).histogram(self._get_key(key)) + + def counter(self, key): + return super(RegexRegistry, self).counter(self._get_key(key)) + + def gauge(self, key, gauge=None): + return super(RegexRegistry, self).gauge(self._get_key(key), gauge) + + def meter(self, key): + return super(RegexRegistry, self).meter(self._get_key(key)) + + +_global_registry = MetricsRegistry() + + +def global_registry(): + return _global_registry + + +def set_global_registry(registry): + global _global_registry + _global_registry = registry + + +def counter(key): + return _global_registry.counter(key) + + +def histogram(key): + return _global_registry.histogram(key) + + +def meter(key): + return _global_registry.meter(key) + + +def timer(key): + return _global_registry.timer(key) + + +def gauge(key, gauge=None): + return _global_registry.gauge(key, gauge) + + +def dump_metrics(): + return _global_registry.dump_metrics() + + +def clear(): + return _global_registry.clear() + +def get_qualname(obj): + if sys.version_info[0] > 2: + return obj.__qualname__ + return obj.__name__ + +def count_calls(fn): + """ + Decorator to track the number of times a function is called. + + :param fn: the function to be decorated + :type fn: C{func} + + :return: the decorated function + :rtype: C{func} + """ + @functools.wraps(fn) + def wrapper(*args, **kwargs): + counter("%s_calls" % get_qualname(fn)).inc() + return fn(*args, **kwargs) + return wrapper + + +def meter_calls(fn): + """ + Decorator to the rate at which a function is called. + + :param fn: the function to be decorated + :type fn: C{func} + + :return: the decorated function + :rtype: C{func} + """ + @functools.wraps(fn) + def wrapper(*args, **kwargs): + meter("%s_calls" % get_qualname(fn)).mark() + return fn(*args, **kwargs) + return wrapper + + +def hist_calls(fn): + """ + Decorator to check the distribution of return values of a function. + + :param fn: the function to be decorated + :type fn: C{func} + + :return: the decorated function + :rtype: C{func} + """ + @functools.wraps(fn) + def wrapper(*args, **kwargs): + _histogram = histogram("%s_calls" % get_qualname(fn)) + rtn = fn(*args, **kwargs) + if type(rtn) in (int, float): + _histogram.update(rtn) + return rtn + return wrapper + + +def time_calls(fn): + """ + Decorator to time the execution of the function. + + :param fn: the function to be decorated + :type fn: C{func} + + :return: the decorated function + :rtype: C{func} + """ + @functools.wraps(fn) + def wrapper(*args, **kwargs): + _timer = timer("%s_calls" % get_qualname(fn)) + with _timer.time(fn = get_qualname(fn)): + return fn(*args, **kwargs) + return wrapper + diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/__init__.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/__init__.py new file mode 100644 index 00000000000..ce44e5f2933 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/__init__.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# lazy import reporters to minimize startup time + + +def HostedGraphiteReporter(*args, **kwargs): + from .hosted_graphite_reporter import HostedGraphiteReporter as cls + return cls(*args, **kwargs) + + +def CarbonReporter(*args, **kwargs): + from .carbon_reporter import CarbonReporter as cls + return cls(*args, **kwargs) + + +def UdpCarbonReporter(*args, **kwargs): + from .carbon_reporter import UdpCarbonReporter as cls + return cls(*args, **kwargs) + + +def ConsoleReporter(*args, **kwargs): + from .console_reporter import ConsoleReporter as cls + return cls(*args, **kwargs) + + +def CsvReporter(*args, **kwargs): + from .csv_reporter import CsvReporter as cls + return cls(*args, **kwargs) diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/carbon_reporter.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/carbon_reporter.py new file mode 100644 index 00000000000..e0caef6bd3d --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/carbon_reporter.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +import socket +import sys + +from .reporter import Reporter + +DEFAULT_CARBON_SERVER = '0.0.0.0' +DEFAULT_CARBON_PORT = 2003 + + +class CarbonReporter(Reporter): + + """ + Carbon is the network daemon to collect metrics for Graphite + """ + + def __init__(self, registry=None, reporting_interval=5, prefix="", + server=DEFAULT_CARBON_SERVER, port=DEFAULT_CARBON_PORT, socket_factory=socket.socket, + clock=None): + super(CarbonReporter, self).__init__( + registry, reporting_interval, clock) + self.prefix = prefix + self.server = server + self.port = port + self.socket_factory = socket_factory + + def report_now(self, registry=None, timestamp=None): + metrics = self._collect_metrics(registry or self.registry, timestamp) + if metrics: + # XXX: keep connection open + sock = self.socket_factory() + sock.connect((self.server, self.port)) + + if sys.version_info[0] > 2: + sock.sendall(metrics.encode()) + else: + sock.sendall(metrics) + + sock.close() + + def _collect_metrics(self, registry, timestamp=None): + timestamp = timestamp or int(round(self.clock.time())) + metrics = registry.dump_metrics() + metrics_data = [] + for key in metrics.keys(): + for value_key in metrics[key].keys(): + metricLine = "%s%s.%s %s %s\n" % ( + self.prefix, key, value_key, metrics[key][value_key], timestamp) + metrics_data.append(metricLine) + return ''.join(metrics_data) + + +class UdpCarbonReporter(CarbonReporter): + + """ + The default CarbonReporter uses TCP. + This sub-class uses UDP instead which might be unreliable but it is faster + """ + + def report_now(self, registry=None, timestamp=None): + metrics = self._collect_metrics(registry or self.registry, timestamp) + if metrics: + sock = self.socket_factory(socket.AF_INET, socket.SOCK_DGRAM) + + if sys.version_info[0] > 2: + sock.sendto(metrics.encode(), (self.server, self.port)) + else: + sock.sendto(metrics, (self.server, self.port)) diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/console_reporter.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/console_reporter.py new file mode 100644 index 00000000000..2d18727372d --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/console_reporter.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +import sys +import datetime +from .reporter import Reporter + + +class ConsoleReporter(Reporter): + + """ + Show metrics in a human readable form. + This is useful for debugging if you want to read the current state on the console. + """ + + def __init__(self, registry=None, reporting_interval=30, stream=sys.stderr, clock=None): + super(ConsoleReporter, self).__init__( + registry, reporting_interval, clock) + self.stream = stream + + def report_now(self, registry=None, timestamp=None): + metrics = self._collect_metrics(registry or self.registry, timestamp) + for line in metrics: + print(line, file=self.stream) + + def _collect_metrics(self, registry, timestamp=None): + timestamp = timestamp or int(round(self.clock.time())) + dt = datetime.datetime(1970, 1, 1) + \ + datetime.timedelta(seconds=timestamp) + metrics = registry.dump_metrics() + metrics_data = ["== %s ===================================" % + dt.strftime("%Y-%m-%d %H:%M:%S")] + for key in metrics.keys(): + values = metrics[key] + metrics_data.append("%s:" % key) + for value_key in values.keys(): + metrics_data.append( + "%20s = %s" % (value_key, values[value_key])) + metrics_data.append("") + return metrics_data diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/csv_reporter.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/csv_reporter.py new file mode 100644 index 00000000000..e7807e72ba5 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/csv_reporter.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +import sys +import os +import datetime +from .reporter import Reporter + + +class CsvReporter(Reporter): + + """ + Show metrics in comma-separated-files. + Each metrics gets its own file + """ + + def __init__(self, registry=None, reporting_interval=30, path=None, separator="\t", clock=None): + super(CsvReporter, self).__init__( + registry, reporting_interval, clock) + self.path = path or os.getcwd() + if not os.path.exists(self.path): + os.makedirs(self.path) + self.separator = separator + self.files = {} + + def report_now(self, registry=None, timestamp=None): + self._save_metrics(registry or self.registry, timestamp) + + def _save_metrics(self, registry, timestamp=None): + timestamp = timestamp or int(round(self.clock.time())) + dt = datetime.datetime(1970, 1, 1) + \ + datetime.timedelta(seconds=timestamp) + date = dt.strftime("%Y-%m-%d %H:%M:%S") + metrics = registry.dump_metrics() + for key in metrics.keys(): + values = metrics[key] + value_keys = list(sorted(values.keys())) + target = os.path.join(self.path, "%s.csv" % key) + f = self.files.get(target, None) + if f is None: + if not os.path.exists(target): + f = open(target, "w") + f.write("%s\n" % self.separator.join(["timestamp"] + value_keys)) + else: + f = open(target, "a") + self.files[target] = f + cols = [date] + for vk in value_keys: + cols.append(values[vk]) + f.write("%s\n" % self.separator.join(map(str, cols))) + f.flush() diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/hosted_graphite_reporter.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/hosted_graphite_reporter.py new file mode 100644 index 00000000000..b72a4d284ea --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/hosted_graphite_reporter.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +import sys +import urllib2 +import base64 + +from .meters import Counter, Histogram, Meter, Timer +from .registry import MetricsRegistry +from .reporter import Reporter + + +class HostedGraphiteReporter(Reporter): + + """ + This reporter requires an api key from http://www.hostedgraphite.com + """ + + def __init__( + self, hosted_graphite_api_key, registry=None, reporting_interval=10, url="https://hostedgraphite.com/api/v1/sink", + clock=None): + super(HostedGraphiteReporter, self).__init__( + registry, reporting_interval, clock) + self.url = url + self.api_key = hosted_graphite_api_key + + def report_now(self, registry=None, timestamp=None): + metrics = self._collect_metrics(registry or self._registry, timestamp) + if metrics: + try: + # XXX: better use http-keepalive/pipelining somehow? + request = urllib2.Request(self.url, metrics) + request.add_header("Authorization", "Basic %s" % + base64.encodestring(self.api_key).strip()) + result = urllib2.urlopen(request) + except Exception as e: + print(e, file=sys.stderr) + + def _collect_metrics(self, registry, timestamp=None): + timestamp = timestamp or int(round(self.clock.time())) + metrics = registry.dump_metrics() + metrics_data = [] + for key in metrics.keys(): + for value_key in metrics[key].keys(): + metric_line = "%s.%s %s %s\n" % ( + key, value_key, metrics[key][value_key], timestamp) + metrics_data.append(metric_line) + return ''.join(metrics_data) diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/reporter.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/reporter.py new file mode 100644 index 00000000000..5c2fc03cc5f --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/reporters/reporter.py @@ -0,0 +1,45 @@ +import time +from threading import Thread, Event +from ..registry import global_registry, get_qualname + + +class Reporter(object): + + def create_thread(self): + # noinspection PyAttributeOutsideInit + self._loop_thread = Thread(target=self._loop, name="pyformance reporter {0}".format(get_qualname(type(self)))) + self._loop_thread.setDaemon(True) + + def __init__(self, registry=None, reporting_interval=30, clock=None): + self.registry = registry or global_registry() + self.reporting_interval = reporting_interval + self.clock = clock or time + self._stopped = Event() + self.create_thread() + + def start(self): + if self._stopped.is_set(): + return False + + r = str(self._loop_thread) + if "stopped" in r: + # has to be recreated in a celery worker + self.create_thread() + elif "started" in r: + # already started + return False + + self._loop_thread.start() + return True + + def stop(self): + self._stopped.set() + + def _loop(self): + while not self._stopped.is_set(): + self.report_now(self.registry) + time.sleep(self.reporting_interval) + # self._stopped.clear() + + def report_now(self, registry=None, timestamp=None): + raise NotImplementedError(self.report_now) diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/__init__.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/__init__.py new file mode 100644 index 00000000000..486f658147c --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/__init__.py @@ -0,0 +1,3 @@ +from .samples import ExpDecayingSample +from .moving_average import ExpWeightedMovingAvg +from .snapshot import Snapshot diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/moving_average.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/moving_average.py new file mode 100644 index 00000000000..0fa57076e95 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/moving_average.py @@ -0,0 +1,69 @@ +import math +import time + + +class ExpWeightedMovingAvg(object): + + """ + An exponentially-weighted moving average. + """ + INTERVAL = 5.0 # seconds + SECONDS_PER_MINUTE = 60.0 + + def __init__(self, period, interval=INTERVAL, clock=time): + """ + Create a new EWMA with a specific smoothing constant. + + :type period: C{int} + :param period: the time in minutes it takes to reach a given significance level + :type interval: C{int} + :param interval: the expected tick interval, defaults to 5s + """ + super(ExpWeightedMovingAvg, self).__init__() + self.clock = clock + self.uncounted = 0.0 + self.interval = interval + self.rate = -1 + self.period = period * ExpWeightedMovingAvg.SECONDS_PER_MINUTE + self.last_tick = self.clock.time() + + def get_rate(self): + if self.clock.time() - self.last_tick >= self.interval: + self.tick() + if self.rate >= 0: + return self.rate + return 0 + + def add(self, value): + self.uncounted += value + + def tick(self): + """ + Mark the passage of time and decay the current rate accordingly. + """ + prev = self.last_tick + now = self.clock.time() + interval = now - prev + if interval <= 0: + return + + instant_rate = self.uncounted / interval + self.uncounted = 0.0 + + if self.rate >= 0: + self.rate += (self._alpha(interval) * (instant_rate - self.rate)) + else: + self.rate = instant_rate + + self.last_tick = now + + def _alpha(self, interval): + """ + Calculate the alpha based on the time since the last tick. This is + necessary because a single threaded Python program loses precision + under high load, so we can't assume a consistant I{EWMA._interval}. + + :type interval: C{float} + :param interval: the interval we use to calculate the alpha + """ + return 1 - math.exp(-interval / self.period) diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/samples.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/samples.py new file mode 100644 index 00000000000..618b4c3ca45 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/samples.py @@ -0,0 +1,112 @@ +import time +import random +import math +import heapq +from .snapshot import Snapshot + +DEFAULT_SIZE = 1028 +DEFAULT_ALPHA = 0.015 + + +class ExpDecayingSample(object): + + """ + An exponentially-decaying random sample of longs. Uses Cormode et al's + forward-decaying priority reservoir sampling method to produce a + statistically representative sample, exponentially biased towards newer + entries. + + @see: + Cormode et al. Forward Decay: A Practical Time Decay Model for + Streaming Systems. ICDE '09: Proceedings of the 2009 IEEE + International Conference on Data Engineering (2009) + """ + + RESCALE_THREASHOLD = 3600.0 # 1 hour + + def __init__(self, size=DEFAULT_SIZE, alpha=DEFAULT_ALPHA, clock=time): + """ + Creates a new L{ExponentiallyDecayingSample}. + + :type size: C{int} + :param size: the number of samples to keep in the sampling reservoir + :type alpha: C{float} + :param alpha: the exponential decay factor; the higher this is, the more + biased the sample will be towards newer values + :type clock: C{function} + :param clock: the function used to return the current time, default to + seconds since the epoch; to be used with other time + units, or with the twisted clock for our testing purposes + """ + super(ExpDecayingSample, self).__init__() + self.clock = clock + self.size = size + self.alpha = alpha + self.clear() + + def clear(self): + self.values = {} + self.priorities = [] + self.counter = 0 + self.start_time = self.clock.time() + self.next_time = self.clock.time() + \ + ExpDecayingSample.RESCALE_THREASHOLD + + def get_size(self): + return self.counter if self.counter < self.size else self.size + + def update(self, value): + """ + Adds a value to the sample. + + :type value: C{int} or C{float} + :param value: the value to be added + """ + if self.size == 0: + return + self._rescale_if_necessary() + priority = self._weight( + self.clock.time() - self.start_time) / random.random() + new_counter = self.counter + 1 + self.counter = new_counter + + if new_counter <= self.size: + self.values[priority] = value + heapq.heappush(self.priorities, priority) + else: + first = heapq.heappop(self.priorities) + if first < priority: + if priority not in self.values: + self.values[priority] = value + heapq.heappush(self.priorities, priority) + while first not in self.values: + first = heapq.heappop(self.priorities) + del self.values[first] + else: + heapq.heappush(self.priorities, first) + + def _rescale_if_necessary(self): + if self.clock.time() >= self.next_time: + self._rescale() + + def _rescale(self): + self.next_time = self.clock.time() + \ + ExpDecayingSample.RESCALE_THREASHOLD + old_start_time = self.start_time + self.start_time = self.clock.time() + new_values = {} + new_priorities = [] + for key, val in self.values.items(): + priority = key * \ + math.exp(-self.alpha * (self.start_time - old_start_time)) + new_values[priority] = val + heapq.heappush(new_priorities, priority) + self.values = new_values + self.priorities = new_priorities + self.counter = len(self.values) + + def _weight(self, value): + return math.exp(self.alpha * value) + + def get_snapshot(self): + return Snapshot(self.values.values()) diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/snapshot.py b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/snapshot.py new file mode 100644 index 00000000000..ac9e693353f --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyformance/stats/snapshot.py @@ -0,0 +1,59 @@ +class Snapshot(object): + + """ + This class is used by the histogram meter + """ + + MEDIAN = 0.5 + P75_Q = 0.75 + P95_Q = 0.95 + P99_Q = 0.99 + P999_Q = 0.999 + + def __init__(self, values): + super(Snapshot, self).__init__() + self.values = sorted(values) + + def get_size(self): + "get current size" + return len(self.values) + + def get_median(self): + "get current median" + return self.get_percentile(Snapshot.MEDIAN) + + def get_75th_percentile(self): + "get current 75th percentile" + return self.get_percentile(Snapshot.P75_Q) + + def get_95th_percentile(self): + "get current 95th percentile" + return self.get_percentile(Snapshot.P95_Q) + + def get_99th_percentile(self): + "get current 99th percentile" + return self.get_percentile(Snapshot.P99_Q) + + def get_999th_percentile(self): + "get current 999th percentile" + return self.get_percentile(Snapshot.P999_Q) + + def get_percentile(self, percentile): + """ + get custom percentile + + :param percentile: float value between 0 and 1 + """ + if percentile < 0 or percentile > 1: + raise ValueError("{0} is not in [0..1]".format(percentile)) + length = len(self.values) + if length == 0: + return 0 + pos = percentile * (length + 1) + if pos < 1: + return self.values[0] + if pos >= length: + return self.values[-1] + lower = self.values[int(pos) - 1] + upper = self.values[int(pos)] + return lower + (pos - int(pos)) * (upper - lower) diff --git a/desktop/core/ext-py3/pyformance-0.3.2/pyproject.toml b/desktop/core/ext-py3/pyformance-0.3.2/pyproject.toml new file mode 100644 index 00000000000..9787c3bdf00 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/desktop/core/ext-py3/pyformance-0.3.2/setup.py b/desktop/core/ext-py3/pyformance-0.3.2/setup.py new file mode 100644 index 00000000000..7898dc38e10 --- /dev/null +++ b/desktop/core/ext-py3/pyformance-0.3.2/setup.py @@ -0,0 +1,22 @@ +# +# pyformance 0.3.2 (https://github.com/omergertel/pyformance), vendored for Hue. +# +# Sources are the pyformance 0.3.2 release unchanged, except pyformance/__init__.py: +# its pkg_resources.declare_namespace() call is removed (see the comment there). +# +from setuptools import setup, find_packages + +setup( + name="pyformance", + version="0.3.2", + author="Omer Getrel", + author_email="omer.gertel@gmail.com", + description="Performance metrics, based on Coda Hale's Yammer metrics", + license="Apache 2.0", + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python :: 2.7", + ], + packages=find_packages(), +) diff --git a/desktop/core/ext-py3/pysaml2-7.3.1/src/saml2/cert.py b/desktop/core/ext-py3/pysaml2-7.3.1/src/saml2/cert.py index c5f626601ac..45f97b6e020 100644 --- a/desktop/core/ext-py3/pysaml2-7.3.1/src/saml2/cert.py +++ b/desktop/core/ext-py3/pysaml2-7.3.1/src/saml2/cert.py @@ -5,6 +5,10 @@ from os import remove from os.path import join +from cryptography import x509 +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.x509.oid import NameOID from OpenSSL import crypto import dateutil.parser import pytz @@ -24,6 +28,34 @@ class PayloadError(Exception): pass +def _to_bytes(value): + return value if isinstance(value, bytes) else value.encode("utf-8") + + +def _hash_algorithm(name): + """ + Hue change: pyOpenSSL 26 removed X509Req, so certificate requests are built with cryptography, which wants a + hash object instead of a name like "sha256". + """ + try: + return getattr(hashes, name.upper())() + except AttributeError: + raise WrongInput(f"Unsupported hash algorithm: {name}") + + +def _subject_name(cert_info): + return x509.Name( + [ + x509.NameAttribute(NameOID.COUNTRY_NAME, cert_info["country_code"]), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, cert_info["state"]), + x509.NameAttribute(NameOID.LOCALITY_NAME, cert_info["city"]), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, cert_info["organization"]), + x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, cert_info["organization_unit"]), + x509.NameAttribute(NameOID.COMMON_NAME, cert_info["cn"]), + ] + ) + + class OpenSSLWrapper: def __init__(self): pass @@ -134,31 +166,34 @@ def create_certificate( k = crypto.PKey() k.generate_key(crypto.TYPE_RSA, key_length) - # create a self-signed cert - cert = crypto.X509() - - if request: - cert = crypto.X509Req() - if len(cert_info["country_code"]) != 2: raise WrongInput("Country code must be two letters!") - cert.get_subject().C = cert_info["country_code"] - cert.get_subject().ST = cert_info["state"] - cert.get_subject().L = cert_info["city"] - cert.get_subject().O = cert_info["organization"] # noqa: E741 - cert.get_subject().OU = cert_info["organization_unit"] - cert.get_subject().CN = cn - if not request: + + if request: + csr = ( + x509.CertificateSigningRequestBuilder() + .subject_name(_subject_name(cert_info)) + .sign(k.to_cryptography_key(), _hash_algorithm(hash_alg)) + ) + else: + # create a self-signed cert + cert = crypto.X509() + cert.get_subject().C = cert_info["country_code"] + cert.get_subject().ST = cert_info["state"] + cert.get_subject().L = cert_info["city"] + cert.get_subject().O = cert_info["organization"] # noqa: E741 + cert.get_subject().OU = cert_info["organization_unit"] + cert.get_subject().CN = cn cert.set_serial_number(sn) cert.gmtime_adj_notBefore(valid_from) # Valid before present time cert.gmtime_adj_notAfter(valid_to) # 3 650 days cert.set_issuer(cert.get_subject()) - cert.set_pubkey(k) - cert.sign(k, hash_alg) + cert.set_pubkey(k) + cert.sign(k, hash_alg) try: if request: - tmp_cert = crypto.dump_certificate_request(crypto.FILETYPE_PEM, cert) + tmp_cert = csr.public_bytes(serialization.Encoding.PEM) else: tmp_cert = crypto.dump_certificate(crypto.FILETYPE_PEM, cert) tmp_key = None @@ -237,27 +272,24 @@ def create_cert_signed_certificate( :return: String representation of the signed certificate. """ - ca_cert = crypto.load_certificate(crypto.FILETYPE_PEM, sign_cert_str) - ca_key = None - if passphrase is not None: - ca_key = crypto.load_privatekey(crypto.FILETYPE_PEM, sign_key_str, passphrase) - else: - ca_key = crypto.load_privatekey(crypto.FILETYPE_PEM, sign_key_str) - req_cert = crypto.load_certificate_request(crypto.FILETYPE_PEM, request_cert_str) - - cert = crypto.X509() - cert.set_subject(req_cert.get_subject()) - cert.set_serial_number(sn) - cert.gmtime_adj_notBefore(valid_from) - cert.gmtime_adj_notAfter(valid_to) - cert.set_issuer(ca_cert.get_subject()) - cert.set_pubkey(req_cert.get_pubkey()) - cert.sign(ca_key, hash_alg) - - cert_dump = crypto.dump_certificate(crypto.FILETYPE_PEM, cert) - if isinstance(cert_dump, str): - return cert_dump - return cert_dump.decode("utf-8") + ca_cert = x509.load_pem_x509_certificate(_to_bytes(sign_cert_str)) + ca_key = serialization.load_pem_private_key( + _to_bytes(sign_key_str), password=_to_bytes(passphrase) if passphrase is not None else None + ) + req_cert = x509.load_pem_x509_csr(_to_bytes(request_cert_str)) + + now = datetime.datetime.now(datetime.timezone.utc) + cert = ( + x509.CertificateBuilder() + .subject_name(req_cert.subject) + .issuer_name(ca_cert.subject) + .public_key(req_cert.public_key()) + .serial_number(sn) + .not_valid_before(now + datetime.timedelta(seconds=valid_from)) + .not_valid_after(now + datetime.timedelta(seconds=valid_to)) + .sign(ca_key, _hash_algorithm(hash_alg)) + ) + return cert.public_bytes(serialization.Encoding.PEM).decode("utf-8") def verify_chain(self, cert_chain_str_list, cert_str): """ @@ -321,16 +353,13 @@ def verify(self, signing_cert_str, cert_str): if ca_cert.get_subject().CN == cert.get_subject().CN: return False, ("CN may not be equal for CA certificate and the " "signed certificate.") - cert_algorithm = cert.get_signature_algorithm() - cert_algorithm = cert_algorithm.decode("ascii") - cert_str = cert_str.encode("ascii") - - cert_crypto = saml2.cryptography.pki.load_pem_x509_certificate(cert_str) + cert_crypto = saml2.cryptography.pki.load_pem_x509_certificate(cert_str.encode("ascii")) + # Hue change: pyOpenSSL 26 removed crypto.verify(), so check the signature with cryptography. try: - crypto.verify(ca_cert, cert_crypto.signature, cert_crypto.tbs_certificate_bytes, cert_algorithm) + cert_crypto.verify_directly_issued_by(ca_cert.to_cryptography()) return True, "Signed certificate is valid and correctly signed by CA certificate." - except crypto.Error as e: + except (InvalidSignature, ValueError, TypeError) as e: return False, f"Certificate is incorrectly signed: {str(e)}" except Exception as e: return False, f"Certificate is not valid for an unknown reason. {str(e)}" diff --git a/desktop/core/generate_requirements.py b/desktop/core/generate_requirements.py index 680e78e4ba7..d3dbd2c5a4f 100755 --- a/desktop/core/generate_requirements.py +++ b/desktop/core/generate_requirements.py @@ -39,6 +39,8 @@ def __init__(self): "boto-2.49.0", # Vendored to relax the hard Mako==1.2.3 pin of djangomako 1.3.2 (see its setup.py). "djangomako-1.3.2", + # Vendored without its pkg_resources.declare_namespace() call, which setuptools >= 82 can no longer serve. + "pyformance-0.3.2", "pysaml2-7.3.1", "python-sasl-0.3.1", ] @@ -54,10 +56,10 @@ def __init__(self): # also drops the remaining emSign roots. It is a transitive dependency of requests; pin it so the CA # bundle never depends on whatever is newest at build time. "certifi==2026.10.23", - # cryptography 46.x requires cffi >= 2.0.0. Held at 2.0.0 because 2.1.0 needs Python >= 3.10. + # cryptography requires cffi >= 2.0.0. Held at 2.0.0 because 2.1.0 needs Python >= 3.10. "cffi==2.0.0", "configobj==5.0.9", - "cryptography==46.0.7", + "cryptography==50.0.2", "cx-Oracle==8.3.0", "Django==5.2.18", "django-auth-ldap==5.3.0", @@ -97,10 +99,10 @@ def __init__(self): "psutil==5.8.0", "pyarrow==23.0.1", "pydantic==2.10.6", - "pyformance==0.3.2", "PyJWT==2.15.1", - # Held at 26.0.0: 26.4.0 drops X509Req, which pysaml2 still needs. Caps cryptography at <47. - "pyOpenSSL==26.0.0", + # 26.3.0 is the first release that accepts cryptography 49 and 50. It no longer has X509Req, so the certificate + # request code of the vendored pysaml2 uses cryptography directly. + "pyOpenSSL==26.4.0", "python-daemon==2.2.4", "python-dateutil==2.8.2", "python-ldap==3.4.5", @@ -135,7 +137,7 @@ def __init__(self): "numpy==1.24.4", "pandas==2.0.3", "pure-sasl==0.6.2", - "setuptools==80.9.0", + "setuptools==84.1.0", "sqlparse==0.6.0", ] diff --git a/desktop/core/generate_requirements_test.py b/desktop/core/generate_requirements_test.py index 17dcad0aed6..951f365492e 100644 --- a/desktop/core/generate_requirements_test.py +++ b/desktop/core/generate_requirements_test.py @@ -49,7 +49,7 @@ def setup_method(self): os.makedirs(self.ext_py3_dir) # Create empty placeholder files that simulate local requirement packages - for req in ["boto-2.49.0", "djangomako-1.3.2", "pysaml2-7.3.1", "python-sasl-0.3.1"]: + for req in ["boto-2.49.0", "djangomako-1.3.2", "pyformance-0.3.2", "pysaml2-7.3.1", "python-sasl-0.3.1"]: with open(os.path.join(self.ext_py3_dir, req), "w") as f: f.write("") diff --git a/desktop/core/src/desktop/appmanager.py b/desktop/core/src/desktop/appmanager.py index 05464a36140..e754095abda 100644 --- a/desktop/core/src/desktop/appmanager.py +++ b/desktop/core/src/desktop/appmanager.py @@ -15,14 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os import re import sys -import logging import traceback from builtins import object +from importlib.metadata import entry_points -import pkg_resources from django.utils.translation import gettext as _ import desktop @@ -56,7 +56,7 @@ def _import_module_or_none(module): try: __import__(module) return sys.modules[module] - except ImportError as ie: + except ImportError: # If the exception came from us importing, we want to just # return None. We need to inspect the stack, though, so we properly # reraise in the case that the module we're importing triggered @@ -78,7 +78,7 @@ class DesktopModuleInfo(object): A Desktop app is encapsulated by a module. These modules are found using the "desktop.sdk.application" - entrypoint (from pkg_resources/setup.py). + entrypoint (declared in setup.py). Each desktop app may have a settings file, which lists DJANGO_APPS to be installed as @@ -184,7 +184,7 @@ def get_bootstrap_file(self): if self.static_dir: f = os.path.join(self.static_dir, "bootstrap.js") if os.path.exists(f): - return file(f) + return open(f) else: return None @@ -236,7 +236,7 @@ def load_libs(): raise Exception("load_apps already has been called.") DESKTOP_LIBS = [] - for lib in pkg_resources.iter_entry_points("desktop.sdk.lib"): + for lib in entry_points(group="desktop.sdk.lib"): m = lib.load() DESKTOP_LIBS.append(DesktopModuleInfo(m)) @@ -260,7 +260,7 @@ def load_apps(app_blacklist): raise Exception(_("load_apps has already been called.")) DESKTOP_APPS = [] - for sdk_app in pkg_resources.iter_entry_points("desktop.sdk.application"): + for sdk_app in entry_points(group="desktop.sdk.application"): if desktop.conf.CSP_NONCE.get(): # This Apps are not supported CSP Nonce is enabled. diff --git a/desktop/core/src/desktop/lib/gunicorn_server_utils.py b/desktop/core/src/desktop/lib/gunicorn_server_utils.py index 7dbca37bcdd..0d9bdde06f7 100644 --- a/desktop/core/src/desktop/lib/gunicorn_server_utils.py +++ b/desktop/core/src/desktop/lib/gunicorn_server_utils.py @@ -29,9 +29,9 @@ import sys import tempfile import time +from importlib.metadata import version import gunicorn.app.base -import pkg_resources import psutil import redis from django.core.wsgi import get_wsgi_application @@ -90,7 +90,7 @@ def worker_int(worker): def enable_logging(args, options): """Enable basic logging for the process""" - HUE_DESKTOP_VERSION = pkg_resources.get_distribution("desktop").version or "Unknown" + HUE_DESKTOP_VERSION = version("desktop") or "Unknown" # Start basic logging as soon as possible. if "HUE_PROCESS_NAME" not in os.environ: _proc = os.path.basename(len(sys.argv) > 1 and sys.argv[1] or sys.argv[0]) diff --git a/desktop/core/src/desktop/settings.py b/desktop/core/src/desktop/settings.py index 356041d8df2..2d071439181 100644 --- a/desktop/core/src/desktop/settings.py +++ b/desktop/core/src/desktop/settings.py @@ -28,8 +28,8 @@ import sys import uuid from builtins import map, zip +from importlib.metadata import version -import pkg_resources from django.utils.translation import gettext_lazy as _ import desktop.redaction @@ -44,7 +44,7 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__)), '..', '..', '..')) -HUE_DESKTOP_VERSION = pkg_resources.get_distribution("desktop").version or "Unknown" +HUE_DESKTOP_VERSION = version("desktop") or "Unknown" NICE_NAME = "Hue" ENV_HUE_PROCESS_NAME = "HUE_PROCESS_NAME" diff --git a/desktop/core/src/desktop/supervisor.py b/desktop/core/src/desktop/supervisor.py index b77b26662ab..e59a0677496 100644 --- a/desktop/core/src/desktop/supervisor.py +++ b/desktop/core/src/desktop/supervisor.py @@ -32,26 +32,25 @@ from __future__ import print_function -import os import grp -import pwd -import sys -import time -import signal import logging import optparse -import threading +import os +import pwd +import signal import subprocess +import sys +import threading +import time from builtins import object, range +from importlib.metadata import entry_points import daemon -import pkg_resources -from daemon.daemon import DaemonContext from daemon.pidfile import TimeoutPIDLockFile -import desktop.log -import desktop.lib.paths import desktop.lib.daemon_utils +import desktop.lib.paths +import desktop.log PROC_NAME = 'supervisor' LOG = logging.getLogger() @@ -164,7 +163,7 @@ def run(self): return LOG.error("Process %s exited abnormally. Restarting it." % (proc_str,)) - except BaseException as ex: + except BaseException: LOG.exception("Uncaught exception. Supervisor exiting.") self.state = Supervisor.ERROR @@ -229,7 +228,7 @@ def get_pid_cmdline(pid): def get_supervisees(): """Pull the supervisor specifications out of the entry point.""" - eps = list(pkg_resources.iter_entry_points(ENTRY_POINT_GROUP)) + eps = list(entry_points(group=ENTRY_POINT_GROUP)) return dict((ep.name, ep.load()) for ep in eps) @@ -366,7 +365,7 @@ def main(): sups.append(sup) wait_loop(sups, options) - except BaseException as ex: + except BaseException: LOG.exception("Exception in supervisor main loop") shutdown(sups) # shutdown() exits the process diff --git a/desktop/libs/libsaml/src/libsaml/cert_tests.py b/desktop/libs/libsaml/src/libsaml/cert_tests.py new file mode 100644 index 00000000000..5e02d69600a --- /dev/null +++ b/desktop/libs/libsaml/src/libsaml/cert_tests.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# Licensed to Cloudera, Inc. under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. Cloudera, Inc. licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Tests for the certificate handling of the vendored pysaml2 (saml2.cert). + +pyOpenSSL 26 removed X509Req, load_certificate_request and verify(), which the SAML signing code uses to create a +temporary signing certificate. These tests run that flow against the installed pyOpenSSL and cryptography. +""" + +import pytest +from cryptography import x509 +from cryptography.x509.oid import NameOID +from saml2.cert import CertificateError, OpenSSLWrapper, WrongInput + +CA_INFO = { + "cn": "ca.example.com", + "country_code": "se", + "state": "AC", + "city": "Dorotea", + "organization": "Company", + "organization_unit": "Sales", +} +SERVICE_INFO = dict(CA_INFO, cn="service.example.com") + + +def common_name(pem): + certificate = x509.load_pem_x509_certificate(pem if isinstance(pem, bytes) else pem.encode()) + return certificate.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value + + +class TestOpenSSLWrapper: + def setup_method(self): + self.wrapper = OpenSSLWrapper() + self.ca_cert, self.ca_key = self.wrapper.create_certificate(CA_INFO, request=False, key_length=2048) + + def create_signed_certificate(self, **kwargs): + request, _key = self.wrapper.create_certificate(SERVICE_INFO, request=True, key_length=2048) + return self.wrapper.create_cert_signed_certificate(self.ca_cert.decode(), self.ca_key.decode(), request.decode(), **kwargs) + + def test_create_self_signed_certificate(self): + assert common_name(self.ca_cert) == "ca.example.com" + assert self.ca_key.startswith(b"-----BEGIN") + + def test_create_certificate_request(self): + request, key = self.wrapper.create_certificate(SERVICE_INFO, request=True, key_length=2048) + + csr = x509.load_pem_x509_csr(request) + assert csr.is_signature_valid + assert csr.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value == "service.example.com" + assert csr.subject.get_attributes_for_oid(NameOID.COUNTRY_NAME)[0].value == "se" + assert key.startswith(b"-----BEGIN") + + def test_certificate_request_requires_a_two_letter_country_code(self): + with pytest.raises(WrongInput): + self.wrapper.create_certificate(dict(SERVICE_INFO, country_code="swe"), request=True) + + def test_unsupported_hash_algorithm_is_rejected(self): + with pytest.raises(WrongInput): + self.wrapper.create_certificate(SERVICE_INFO, request=True, hash_alg="nosuchhash") + + def test_encrypted_private_key(self): + _cert, key = self.wrapper.create_certificate( + CA_INFO, key_length=2048, cipher_passphrase={"cipher": "aes256", "passphrase": "secret"} + ) + + assert b"ENCRYPTED" in key + + def test_sign_certificate_request_with_ca(self): + signed = self.create_signed_certificate() + + certificate = x509.load_pem_x509_certificate(signed.encode()) + assert common_name(signed) == "service.example.com" + assert certificate.issuer.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value == "ca.example.com" + + def test_sign_certificate_request_with_encrypted_ca_key(self): + ca_cert, ca_key = self.wrapper.create_certificate( + CA_INFO, key_length=2048, cipher_passphrase={"cipher": "aes256", "passphrase": "secret"} + ) + request, _key = self.wrapper.create_certificate(SERVICE_INFO, request=True, key_length=2048) + + signed = self.wrapper.create_cert_signed_certificate(ca_cert.decode(), ca_key.decode(), request.decode(), passphrase="secret") + + assert common_name(signed) == "service.example.com" + + def test_verify_accepts_certificate_signed_by_ca(self): + valid, _message = self.wrapper.verify(self.ca_cert.decode(), self.create_signed_certificate()) + + assert valid + + def test_verify_rejects_certificate_signed_by_another_ca(self): + other_ca_cert, _other_key = self.wrapper.create_certificate(dict(CA_INFO, cn="other.example.com"), key_length=2048) + + valid, message = self.wrapper.verify(other_ca_cert.decode(), self.create_signed_certificate()) + + assert not valid + assert "incorrectly signed" in message + + def test_verify_rejects_certificate_with_same_common_name_as_the_ca(self): + request, _key = self.wrapper.create_certificate(CA_INFO, request=True, key_length=2048) + signed = self.wrapper.create_cert_signed_certificate(self.ca_cert.decode(), self.ca_key.decode(), request.decode()) + + valid, message = self.wrapper.verify(self.ca_cert.decode(), signed) + + assert not valid + assert "CN may not be equal" in message + + def test_certificate_error_is_raised_when_the_key_cannot_be_dumped(self): + with pytest.raises(CertificateError): + self.wrapper.create_certificate(CA_INFO, key_length=2048, cipher_passphrase={"cipher": "nosuchcipher", "passphrase": "secret"}) diff --git a/desktop/libs/notebook/src/notebook/connectors/sql_alchemy.py b/desktop/libs/notebook/src/notebook/connectors/sql_alchemy.py index db95cadf907..ba2900269fe 100644 --- a/desktop/libs/notebook/src/notebook/connectors/sql_alchemy.py +++ b/desktop/libs/notebook/src/notebook/connectors/sql_alchemy.py @@ -56,11 +56,13 @@ import re import textwrap import uuid +from importlib.metadata import entry_points from string import Template from urllib.parse import parse_qs as urllib_parse_qs, quote_plus as urllib_quote_plus, urlparse as urllib_urlparse from past.builtins import long from sqlalchemy import create_engine, inspect, MetaData, Table +from sqlalchemy.dialects import registry from sqlalchemy.exc import CompileError, NoSuchTableError, OperationalError, ProgrammingError, UnsupportedCompilationError from desktop.lib.i18n import force_unicode @@ -76,6 +78,26 @@ LOG = logging.getLogger() +_BUILTIN_DIALECT_LOADER = registry.auto_fn + + +def _load_dialect(name): + ''' + SQLAlchemy 1.3 finds third-party dialects (trino://, phoenix://...) via pkg_resources, which setuptools >= 82 + no longer ships. Look up the same "sqlalchemy.dialects" entry points with importlib.metadata instead, after + the built-in dialects, in the order SQLAlchemy uses. + ''' + loader = _BUILTIN_DIALECT_LOADER(name) + if loader: + return loader + + entry_point = next(iter(entry_points(group='sqlalchemy.dialects', name=name)), None) + return entry_point.load if entry_point else None + + +registry.auto_fn = _load_dialect + + def query_error_handler(func): def decorator(*args, **kwargs): try: diff --git a/desktop/libs/notebook/src/notebook/connectors/sql_alchemy_tests.py b/desktop/libs/notebook/src/notebook/connectors/sql_alchemy_tests.py index 20cab695a22..08c70d3427c 100644 --- a/desktop/libs/notebook/src/notebook/connectors/sql_alchemy_tests.py +++ b/desktop/libs/notebook/src/notebook/connectors/sql_alchemy_tests.py @@ -16,19 +16,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import logging from builtins import object from unittest.mock import MagicMock, Mock, patch import pytest +from sqlalchemy.dialects import registry from sqlalchemy.exc import UnsupportedCompilationError -from sqlalchemy.types import ARRAY, JSON, VARCHAR, NullType +from sqlalchemy.types import ARRAY, JSON, NullType, VARCHAR from desktop.auth.backend import rewrite_user from desktop.lib.django_test_util import make_logged_in_client from notebook.connectors.base import AuthenticationRequired -from notebook.connectors.sql_alchemy import Assist, SqlAlchemyApi +from notebook.connectors.sql_alchemy import _load_dialect, Assist, SqlAlchemyApi from useradmin.models import User LOG = logging.getLogger() @@ -74,7 +74,7 @@ def test_create_athena_engine(self): } } - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: + with patch('notebook.connectors.sql_alchemy.create_engine'): SqlAlchemyApi(self.user, interpreter)._create_engine() def test_fetch_result_empty(self): @@ -141,7 +141,7 @@ def test_create_engine_auth_error(self): } } - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: + with patch('notebook.connectors.sql_alchemy.create_engine'): with pytest.raises(AuthenticationRequired): SqlAlchemyApi(self.user, interpreter)._create_engine() @@ -165,7 +165,7 @@ def test_create_engine_auth(self): } } - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: + with patch('notebook.connectors.sql_alchemy.create_engine'): SqlAlchemyApi(self.user, interpreter)._create_engine() def test_create_connection_error(self): @@ -176,7 +176,7 @@ def test_create_connection_error(self): } } - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: + with patch('notebook.connectors.sql_alchemy.create_engine'): with pytest.raises(AuthenticationRequired): engine = SqlAlchemyApi(self.user, interpreter)._create_engine() SqlAlchemyApi(self.user, interpreter)._create_connection(engine) @@ -201,7 +201,7 @@ def test_create_connection(self): } } - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: + with patch('notebook.connectors.sql_alchemy.create_engine'): engine = SqlAlchemyApi(self.user, interpreter)._create_engine() SqlAlchemyApi(self.user, interpreter)._create_connection(engine) @@ -216,14 +216,14 @@ def test_create_engine_with_impersonation(self): } with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: - engine = SqlAlchemyApi(self.user, interpreter)._create_engine() + SqlAlchemyApi(self.user, interpreter)._create_engine() create_engine.assert_called_with('presto://hue:8080/hue', pool_pre_ping=True) interpreter['options']['has_impersonation'] = True # On with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: - engine = SqlAlchemyApi(self.user, interpreter)._create_engine() + SqlAlchemyApi(self.user, interpreter)._create_engine() create_engine.assert_called_with('presto://hue:8080/hue', connect_args={'principal_username': 'test'}, @@ -240,14 +240,14 @@ def test_create_engine_with_impersonation_phoenix(self): } with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: - engine = SqlAlchemyApi(self.user, interpreter)._create_engine() + SqlAlchemyApi(self.user, interpreter)._create_engine() create_engine.assert_called_with('phoenix://hue:8080/hue', pool_pre_ping=False) interpreter['options']['has_impersonation'] = True # On with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: - engine = SqlAlchemyApi(self.user, interpreter)._create_engine() + SqlAlchemyApi(self.user, interpreter)._create_engine() create_engine.assert_called_with('phoenix://test@hue:8080/hue', pool_pre_ping=False) @@ -275,7 +275,7 @@ def test_explain(self): def test_check_status(self): notebook = Mock() - with patch('notebook.connectors.sql_alchemy.CONNECTIONS') as CONNECTIONS: + with patch('notebook.connectors.sql_alchemy.CONNECTIONS'): snippet = {'result': {'handle': {'guid': 'guid-1', 'has_result_set': False}}} response = SqlAlchemyApi(self.user, self.interpreter).check_status(notebook, snippet) @@ -288,9 +288,9 @@ def test_check_status(self): def test_get_sample_data(self): snippet = Mock() - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: + with patch('notebook.connectors.sql_alchemy.create_engine'): with patch('notebook.connectors.sql_alchemy.Assist.get_sample_data') as get_sample_data: - with patch('notebook.connectors.sql_alchemy.inspect') as inspect: + with patch('notebook.connectors.sql_alchemy.inspect'): get_sample_data.return_value = (['col1'], [[1], [2]]) response = SqlAlchemyApi(self.user, self.interpreter).get_sample_data(snippet) @@ -303,8 +303,8 @@ def test_get_sample_data(self): def test_get_tables(self): snippet = MagicMock() - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: - with patch('notebook.connectors.sql_alchemy.inspect') as inspect: + with patch('notebook.connectors.sql_alchemy.create_engine'): + with patch('notebook.connectors.sql_alchemy.inspect'): with patch('notebook.connectors.sql_alchemy.Assist.get_table_names') as get_table_names: with patch('notebook.connectors.sql_alchemy.Assist.get_view_names') as get_view_names: get_table_names.return_value = ['table1'] @@ -319,9 +319,9 @@ def test_get_tables(self): def test_get_sample_data_table(self): snippet = Mock() - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: + with patch('notebook.connectors.sql_alchemy.create_engine'): with patch('notebook.connectors.sql_alchemy.Assist.get_sample_data') as get_sample_data: - with patch('notebook.connectors.sql_alchemy.inspect') as inspect: + with patch('notebook.connectors.sql_alchemy.inspect'): get_sample_data.return_value = (['col1'], [[1], [2]]) response = SqlAlchemyApi(self.user, self.interpreter).get_sample_data(snippet, database='database1', table='table1') @@ -349,7 +349,7 @@ def test_dialect_trim_statement_semicolon(self): snippet = {'statement': 'SELECT 1;', 'result': {}} # Trim - engine = SqlAlchemyApi(self.user, interpreter).execute(notebook, snippet) + SqlAlchemyApi(self.user, interpreter).execute(notebook, snippet) execute.assert_called_with('SELECT 1') @@ -357,7 +357,7 @@ def test_dialect_trim_statement_semicolon(self): interpreter['options']['url'] = 'mysql://hue:3306/hue' interpreter['dialect_properties']['sql_identifier_quote'] = '`' - engine = SqlAlchemyApi(self.user, interpreter).execute(notebook, snippet) + SqlAlchemyApi(self.user, interpreter).execute(notebook, snippet) execute.assert_called_with('SELECT 1') @@ -423,8 +423,8 @@ def test_empty_database_names(self): } snippet = MagicMock() - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: - with patch('notebook.connectors.sql_alchemy.inspect') as inspect: + with patch('notebook.connectors.sql_alchemy.create_engine'): + with patch('notebook.connectors.sql_alchemy.inspect'): with patch('notebook.connectors.sql_alchemy.Assist') as Assist: Assist.return_value = Mock(get_databases=Mock(return_value=['SYSTEM', None])) @@ -439,8 +439,8 @@ def test_columns_with_null_type(self): } snippet = MagicMock() - with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: - with patch('notebook.connectors.sql_alchemy.inspect') as inspect: + with patch('notebook.connectors.sql_alchemy.create_engine'): + with patch('notebook.connectors.sql_alchemy.inspect'): with patch('notebook.connectors.sql_alchemy.Assist') as Assist: def col1_dict(key): return { @@ -525,3 +525,29 @@ def test_fix_bigquery_db_prefixes(self): assert api._fix_bigquery_db_prefixes('table') == 'table' assert api._fix_bigquery_db_prefixes('db.table') == 'table' + + +class TestLoadDialect(object): + + def test_registry_uses_load_dialect(self): + assert registry.auto_fn is _load_dialect + + def test_builtin_dialect_before_entry_points(self): + with patch('notebook.connectors.sql_alchemy.entry_points') as entry_points: + loader = _load_dialect('sqlite') + + assert loader().name == 'sqlite' + entry_points.assert_not_called() + + def test_entry_point_dialect(self): + entry_point = Mock() + + with patch('notebook.connectors.sql_alchemy.entry_points', return_value=[entry_point]) as entry_points: + loader = _load_dialect('trino') + + entry_points.assert_called_once_with(group='sqlalchemy.dialects', name='trino') + assert loader is entry_point.load + + def test_unknown_dialect(self): + with patch('notebook.connectors.sql_alchemy.entry_points', return_value=[]): + assert _load_dialect('nosuchdialect') is None