Merge branch 'master' into trino_query_explain

Author: agl29

apps/impala/src/impala/conf.py

@@ -73,6 +73,12 @@
   type=coerce_bool,
   dynamic_default=is_impersonation_enabled)
 
+COORDINATOR_UI_SPNEGO = Config(
+  key='coordinator_ui_spnego',
+  help=_t("Impala Coordinator Web Server has Spnego enabled."),
+  type=coerce_bool,
+  default=False)
+
 QUERYCACHE_ROWS = Config(
   key='querycache_rows',
   help=_t("Number of initial rows of a resultset to ask Impala to cache in order to"
@@ -121,7 +127,8 @@
 USER_SCRATCH_DIR_PERMISSION = Config(
   key="user_scratch_dir_permission",
   help=_t("Due to IMPALA-10272, the importer fails with READ permissions."
-          "Setting this to True, means setting the scratch directory and its file to 777 so the importer does not fail with permission issue."),
+          "Setting this to True, means setting the scratch directory and its file to 777 so the importer "
+          "does not fail with permission issue."),
   type=coerce_bool,
   default=False
 )

apps/jobbrowser/src/jobbrowser/apis/query_api.py

@@ -27,7 +27,10 @@
 from babel import localtime
 import os
 
+from urllib.parse import urlparse
+
 from desktop.lib import export_csvxls
+from impala.conf import COORDINATOR_UI_SPNEGO
 from libanalyze import analyze as analyzer, rules
 from notebook.conf import ENABLE_QUERY_ANALYSIS
 
@@ -158,12 +161,16 @@ def app(self, appid):
       }
     app = apps.get('apps')[0]
     progress_groups = re.search(r"([\d\.\,]+)%", app.get('progress'))
+    parsed_api_url = urlparse(self.api.url)
+
     app.update({
       'progress': float(progress_groups.group(1)) \
           if progress_groups and progress_groups.group(1) else 100 \
             if self._api_status(app.get('status')) in ['SUCCEEDED', 'FAILED'] else 1,
       'type': 'queries',
-      'doc_url': "%s/query_plan?query_id=%s" % (self.api.url, appid),
+      'doc_url': '%s/query_plan?query_id=%s' % (self.api.url, appid) if not COORDINATOR_UI_SPNEGO.get() else
+                 '%s/query_plan?scheme=%s&host=%s&port=%s&query_id=%s' %
+                   (self.api.url, parsed_api_url.scheme, parsed_api_url.hostname, parsed_api_url.port, appid),
       'properties': {
         'memory': '',
         'profile': '',

apps/jobbrowser/src/jobbrowser/apis/query_api_tests.py

@@ -27,6 +27,7 @@
 
 from desktop.auth.backend import rewrite_user
 from desktop.lib.django_test_util import make_logged_in_client
+from impala.conf import COORDINATOR_UI_SPNEGO
 from useradmin.models import User
 
 from jobbrowser.apis.query_api import QueryApi
@@ -102,3 +103,74 @@ def test_download_profile(self):
         assert_equal(resp.status_code, 200)
         assert_equal(resp['Content-Disposition'], 'attachment; filename="query-profile_00001.txt"')
         assert_equal(resp.content, b'Query (id=d94d2fb4815a05c4:b1ccec1500000000):\n  Summary:...')
+
+  def test_doc_url(self):
+    with patch('jobbrowser.apis.query_api._get_api') as _get_api:
+      with patch('jobbrowser.apis.query_api.QueryApi.apps') as apps:
+        _get_api.return_value = Mock(
+          url='https://coordinator:25000'
+        )
+
+        apps.return_value = {
+          "apps": [{
+            "id": "b246701d30ab0dd1:afc9f65900000000",
+            "name": "SELECT *\nFROM `default`.web_logs\nLIMIT 100",
+            "status": "FINISHED",
+            "apiStatus": "SUCCEEDED",
+            "type": "QUERY",
+            "user": "test",
+            "queue": "root.test",
+            "progress": "4 / 4 ( 100%)",
+            "isRunning": False,
+            "canWrite": True,
+            "duration": 1291.0,
+            "submitted": "2024-03-06 17:59:15.304000",
+            "rows_fetched": 100,
+            "waiting": True,
+            "waiting_time": "2s610ms"
+          }],
+          "total": 1
+        }
+
+        result = QueryApi(self.user).app('b246701d30ab0dd1:afc9f65900000000')
+        assert_equal(result.get('doc_url'),
+                     'https://coordinator:25000/query_plan?query_id=b246701d30ab0dd1:afc9f65900000000')
+
+  def test_doc_url_spnego(self):
+    resets = [
+      COORDINATOR_UI_SPNEGO.set_for_testing(True)
+    ]
+    with patch('jobbrowser.apis.query_api._get_api') as _get_api:
+      with patch('jobbrowser.apis.query_api.QueryApi.apps') as apps:
+        _get_api.return_value = Mock(
+          url='https://coordinator:25000'
+        )
+
+        apps.return_value = {
+          "apps": [{
+            "id": "b246701d30ab0dd1:afc9f65900000000",
+            "name": "SELECT *\nFROM `default`.web_logs\nLIMIT 100",
+            "status": "FINISHED",
+            "apiStatus": "SUCCEEDED",
+            "type": "QUERY",
+            "user": "test",
+            "queue": "root.test",
+            "progress": "4 / 4 ( 100%)",
+            "isRunning": False,
+            "canWrite": True,
+            "duration": 1291.0,
+            "submitted": "2024-03-06 17:59:15.304000",
+            "rows_fetched": 100,
+            "waiting": True,
+            "waiting_time": "2s610ms"
+          }],
+          "total": 1
+        }
+        try:
+          result = QueryApi(self.user).app('b246701d30ab0dd1:afc9f65900000000')
+          assert_equal(result.get('doc_url'),
+                       'https://coordinator:25000/query_plan?'
+                       'scheme=https&host=coordinator&port=25000&query_id=b246701d30ab0dd1:afc9f65900000000')
+        finally:
+          for reset in resets:
+            reset()

desktop/conf.dist/hue.ini

@@ -1506,6 +1506,9 @@ submit_to=True
 # Turn on/off impersonation mechanism when talking to Impala
 ## impersonation_enabled=False
 
+# Impala Coordinator Web Server has Spnego enabled
+## coordinator_ui_spnego=false
+
 # Number of initial rows of a result set to ask Impala to cache in order
 # to support re-fetching them for downloading them.
 # Set to 0 for disabling the option and backward compatibility.

desktop/conf/pseudo-distributed.ini.tmpl

@@ -1488,6 +1488,9 @@
   # Turn on/off impersonation mechanism when talking to Impala
   ## impersonation_enabled=False
 
+  # Impala Coordinator Web Server has Spnego enabled
+  ## coordinator_ui_spnego=false
+
   # Number of initial rows of a result set to ask Impala to cache in order
   # to support re-fetching them for downloading them.
   # Set to 0 for disabling the option and backward compatibility.