[release] Prepare Hue 4.12.0

VERSION

@@ -17,4 +17,4 @@
 # This file should be the one source of truth for for versions within HUE.
 # It is at least included by each of the default hue app's setup.py.
 
-VERSION="4.11.0"
+VERSION="4.12.0"

desktop/core/src/desktop/appmanager.py

@@ -37,6 +37,22 @@
 
 LOG = logging.getLogger()
 
+
+def _unique_entry_points(group):
+  """
+  Entry points of a group, without the duplicates.
+
+  A distribution found at several places of sys.path (e.g. an editable install and its source egg-info) is
+  returned several times by the older versions of importlib.metadata, which would load an app twice.
+  """
+  seen = set()
+
+  for entry_point in entry_points(group=group):
+    if entry_point.name not in seen:
+      seen.add(entry_point.name)
+      yield entry_point
+
+
 ######################################################################
 # Global variables set after calling load_apps()
 ######################################################################
@@ -236,7 +252,7 @@ def load_libs():
     raise Exception("load_apps already has been called.")
   DESKTOP_LIBS = []
 
-  for lib in entry_points(group="desktop.sdk.lib"):
+  for lib in _unique_entry_points("desktop.sdk.lib"):
     m = lib.load()
     DESKTOP_LIBS.append(DesktopModuleInfo(m))
 
@@ -260,7 +276,7 @@ def load_apps(app_blacklist):
     raise Exception(_("load_apps has already been called."))
   DESKTOP_APPS = []
 
-  for sdk_app in entry_points(group="desktop.sdk.application"):
+  for sdk_app in _unique_entry_points("desktop.sdk.application"):
 
     if desktop.conf.CSP_NONCE.get():
       # This Apps are not supported CSP Nonce is enabled.

desktop/core/src/desktop/appmanager_tests.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from importlib.metadata import EntryPoint
+from unittest.mock import patch
+
+from desktop.appmanager import _unique_entry_points
+
+
+def test_unique_entry_points_skips_a_distribution_found_twice():
+  # Older importlib.metadata versions return a distribution once per sys.path entry
+  duplicated = [
+    EntryPoint(name='dashboard', value='dashboard', group='desktop.sdk.application'),
+    EntryPoint(name='search', value='search', group='desktop.sdk.application'),
+    EntryPoint(name='dashboard', value='dashboard', group='desktop.sdk.application'),
+  ]
+
+  with patch('desktop.appmanager.entry_points', return_value=duplicated) as entry_points:
+    names = [entry_point.name for entry_point in _unique_entry_points('desktop.sdk.application')]
+
+  entry_points.assert_called_once_with(group='desktop.sdk.application')
+  assert names == ['dashboard', 'search']

desktop/libs/librdbms/java/pom.xml

@@ -26,7 +26,7 @@
       <groupId>com.cloudera.hue</groupId>
       <artifactId>hue-parent</artifactId>
       <relativePath>../../../../maven/pom.xml</relativePath>
-      <version>4.11.0-SNAPSHOT</version>
+      <version>4.12.0-SNAPSHOT</version>
     </parent>
 
     <groupId>com.cloudera.hue</groupId>

desktop/libs/notebook/src/notebook/connectors/sql_alchemy.py

@@ -174,12 +174,11 @@ def _create_engine(self):
       url = url.replace(url[17:37], urllib_quote_plus(url[17:37]))
       url = url.replace(url[38:50], urllib_quote_plus(url[38:50]))
       parsed = urllib_urlparse(url)
-      s3_staging_dir = urllib_parse_qs(parsed.query)['s3_staging_dir'][0]
-      url = url.replace(s3_staging_dir, urllib_quote_plus(s3_staging_dir))
-      work_group = urllib_parse_qs(parsed.query)['work_group'][0]
-      url = url.replace(work_group, urllib_quote_plus(work_group))
-      catalog_name = urllib_parse_qs(parsed.query)['catalog_name'][0]
-      url = url.replace(catalog_name, urllib_quote_plus(catalog_name))
+      query = urllib_parse_qs(parsed.query)
+      # The work group and the catalog name are optional, Athena uses its defaults when they are missing
+      for param in ('s3_staging_dir', 'work_group', 'catalog_name'):
+        for value in query.get(param, [])[:1]:
+          url = url.replace(value, urllib_quote_plus(value))
 
     m = re.search(URL_PATTERN, url)
     driver_name = m.group('driver_name')

desktop/libs/notebook/src/notebook/connectors/sql_alchemy_tests.py

@@ -74,9 +74,28 @@ def test_create_athena_engine(self):
       }
     }
 
-    with patch('notebook.connectors.sql_alchemy.create_engine'):
+    with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine:
+      SqlAlchemyApi(self.user, interpreter)._create_engine()
+
+    url = create_engine.call_args.args[0]
+    assert 's3_staging_dir=s3%3A%2F%2Fgethue-athena%2Fscratch' in url
+    assert 'work_group=demo_group' in url
+    assert 'catalog_name=AwsDataCatalog' in url
+
+  def test_create_athena_engine_without_work_group_and_catalog_name(self):
+    interpreter = {
+      'name': 'hive',
+      'options': {
+        'url': 'awsathena+rest://XXXXXXXXXXXXXXX:XXXXXXXXXXXXXXXXXXX@athena.us-west-2.amazonaws.com:443/default?'
+            's3_staging_dir=s3://gethue-athena/scratch'
+      }
+    }
+
+    with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine:
       SqlAlchemyApi(self.user, interpreter)._create_engine()
 
+    assert 's3_staging_dir=s3%3A%2F%2Fgethue-athena%2Fscratch' in create_engine.call_args.args[0]
+
   def test_fetch_result_empty(self):
     notebook = Mock()
     snippet = {'result': {'handle': {'guid': 'guid-1'}}}

docs/docs-site/content/administrator/installation/dependencies/_index.md

@@ -62,6 +62,8 @@ Hue sets `options={"threaded": true}` for Oracle by default. If you override `op
 
 ## OS packages
 
+The [File Import API](/developer/api/rest/importer/) detects the file type with the `libmagic` library. Install it with the OS package manager: `libmagic1` on Ubuntu, `file-libs` on RHEL and `brew install libmagic` on macOS. Without it, Hue logs a warning and cannot guess the file type.
+
 ### Ubuntu
 
 ```bash

docs/docs-site/content/developer/api/rest/_index.md

@@ -802,7 +802,7 @@ It returns `200 OK` when all operations succeed. Otherwise, it returns `500` wit
 
 ## Importer
 
-Use the Importer endpoints to upload a file, detect its format and preview it before you create a table. Read the [File Import API](/developer/api/rest/importer/) for the parameters, responses and a full workflow.
+Use the Importer endpoints to upload a file, detect its format and preview it. These endpoints are under development and do not create the table yet. Read the [File Import API](/developer/api/rest/importer/) for the parameters, responses and a full workflow.
 
 | Purpose | Method and URL |
 |---------|----------------|

docs/docs-site/content/developer/api/rest/importer.md

@@ -13,6 +13,8 @@ The examples use `hue.example.com` as the server. Replace `<ACCESS_TOKEN>` with
 
 ## Overview
 
+> **Note:** The File Import API is under development. It uploads a file, detects its format and header, previews it and lists the SQL types of a dialect. It does not create the table yet.
+
 The importer is turned off by default. An administrator turns it on with `is_enabled=true` in the `[desktop] [[importer]]` section of `hue.ini`. The `get_config` response reports the setting in `importer.is_enabled`.
 
 The File Import API allows you to: