[pull] main from swirlai:main

.github/workflows/db-dist.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Set Up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.12.8'
+          python-version: '3.12.9'
           cache: 'pip'
       - name: Install Swirl (with a timeout)
         run: ./install.sh

.github/workflows/qa-suite.yml

@@ -28,7 +28,7 @@ jobs:
       - name: Set Up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.12.8'
+          python-version: '3.12.9'
           cache: 'pip'
       - name: Install Swirl
         run: ./install.sh
@@ -46,6 +46,37 @@ jobs:
           python swirl.py start
         env:
           ALLOWED_HOSTS: localhost,host.docker.internal
+      - name: PATCH the Web PSE Config
+        env:
+          JSON: ${{ secrets.QA_PSE_WEB }}
+          API_ENDPOINT: http://localhost:8000/swirl/searchproviders/1/
+          PASSWORD: ${{ secrets.QA_ADMIN_PW }}
+        run: |
+          sleep 15
+          curl -X PATCH "$API_ENDPOINT" \
+            -u admin:"$PASSWORD" \
+            -H "Content-Type: application/json" \
+            -d "$JSON"
+      - name: PATCH the LinkedIn PSE Config
+        env:
+          JSON: ${{ secrets.QA_PSE_LINKEDIN }}
+          API_ENDPOINT: http://localhost:8000/swirl/searchproviders/3/
+          PASSWORD: ${{ secrets.QA_ADMIN_PW }}
+        run: |
+          curl -X PATCH "$API_ENDPOINT" \
+            -u admin:"$PASSWORD" \
+            -H "Content-Type: application/json" \
+            -d "$JSON"
+      - name: PATCH the SWIRL Docs PSE Config
+        env:
+          JSON: ${{ secrets.QA_PSE_DOCS }}
+          API_ENDPOINT: http://localhost:8000/swirl/searchproviders/4/
+          PASSWORD: ${{ secrets.QA_ADMIN_PW }}
+        run: |
+          curl -X PATCH "$API_ENDPOINT" \
+            -u admin:"$PASSWORD" \
+            -H "Content-Type: application/json" \
+            -d "$JSON"
       - name: Decode BigQuery token and create JSON file
         run: |
           echo "$BIGQUERY_TOKEN_BASE64" | base64 --decode > "${{ github.workspace }}/token.json"

.github/workflows/test-build-pipeline.yml

@@ -25,7 +25,7 @@ jobs:
         - name: Set Up Python
           uses: actions/setup-python@v5
           with:
-            python-version: '3.12.8'
+            python-version: '3.12.9'
             cache: 'pip'
         - name: Install Swirl
           run: ./install.sh
@@ -56,7 +56,7 @@ jobs:
         - name: Set Up Python
           uses: actions/setup-python@v5
           with:
-            python-version: '3.12.8'
+            python-version: '3.12.9'
             cache: 'pip'
         - name: Install Swirl (with a timeout)
           run: ./install.sh
@@ -125,7 +125,7 @@ jobs:
         - name: Set Up Python
           uses: actions/setup-python@v5
           with:
-            python-version: '3.12.8'
+            python-version: '3.12.9'
             cache: 'pip'
         - name: Install Swirl
           run: ./install.sh
@@ -143,6 +143,37 @@ jobs:
             python swirl.py start
           env:
             ALLOWED_HOSTS: localhost,host.docker.internal
+        - name: PATCH the Web PSE Config
+          env:
+            JSON: ${{ secrets.QA_PSE_WEB }}
+            API_ENDPOINT: http://localhost:8000/swirl/searchproviders/1/
+            PASSWORD: ${{ secrets.QA_ADMIN_PW }}
+          run: |
+            sleep 15
+            curl -X PATCH "$API_ENDPOINT" \
+              -u admin:"$PASSWORD" \
+              -H "Content-Type: application/json" \
+              -d "$JSON"
+        - name: PATCH the LinkedIn PSE Config
+          env:
+            JSON: ${{ secrets.QA_PSE_LINKEDIN }}
+            API_ENDPOINT: http://localhost:8000/swirl/searchproviders/3/
+            PASSWORD: ${{ secrets.QA_ADMIN_PW }}
+          run: |
+            curl -X PATCH "$API_ENDPOINT" \
+              -u admin:"$PASSWORD" \
+              -H "Content-Type: application/json" \
+              -d "$JSON"
+        - name: PATCH the SWIRL Docs PSE Config
+          env:
+            JSON: ${{ secrets.QA_PSE_DOCS }}
+            API_ENDPOINT: http://localhost:8000/swirl/searchproviders/4/
+            PASSWORD: ${{ secrets.QA_ADMIN_PW }}
+          run: |
+            curl -X PATCH "$API_ENDPOINT" \
+              -u admin:"$PASSWORD" \
+              -H "Content-Type: application/json" \
+              -d "$JSON"
         - name: Decode BigQuery token and create JSON file
           run: |
             echo "$BIGQUERY_TOKEN_BASE64" | base64 --decode > "${{ github.workspace }}/token.json"

.github/workflows/testing-wip.yml

@@ -41,7 +41,7 @@ jobs:
       - name: Set Up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.12.8'
+          python-version: '3.12.9'
           cache: 'pip'
       - name: Install Swirl
         run: ./install.sh
@@ -59,6 +59,37 @@ jobs:
           python swirl.py start
         env:
           ALLOWED_HOSTS: localhost,host.docker.internal
+      - name: PATCH the Web PSE Config
+        env:
+          JSON: ${{ secrets.QA_PSE_WEB }}
+          API_ENDPOINT: http://localhost:8000/swirl/searchproviders/1/
+          PASSWORD: ${{ secrets.QA_ADMIN_PW }}
+        run: |
+          sleep 15
+          curl -X PATCH "$API_ENDPOINT" \
+            -u admin:"$PASSWORD" \
+            -H "Content-Type: application/json" \
+            -d "$JSON"
+      - name: PATCH the LinkedIn PSE Config
+        env:
+          JSON: ${{ secrets.QA_PSE_LINKEDIN }}
+          API_ENDPOINT: http://localhost:8000/swirl/searchproviders/3/
+          PASSWORD: ${{ secrets.QA_ADMIN_PW }}
+        run: |
+          curl -X PATCH "$API_ENDPOINT" \
+            -u admin:"$PASSWORD" \
+            -H "Content-Type: application/json" \
+            -d "$JSON"
+      - name: PATCH the SWIRL Docs PSE Config
+        env:
+          JSON: ${{ secrets.QA_PSE_DOCS }}
+          API_ENDPOINT: http://localhost:8000/swirl/searchproviders/4/
+          PASSWORD: ${{ secrets.QA_ADMIN_PW }}
+        run: |
+          curl -X PATCH "$API_ENDPOINT" \
+            -u admin:"$PASSWORD" \
+            -H "Content-Type: application/json" \
+            -d "$JSON"
       - name: Decode BigQuery token and create JSON file
         run: |
           echo "$BIGQUERY_TOKEN_BASE64" | base64 --decode > "${{ github.workspace }}/token.json"

.github/workflows/unit-tests.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Set Up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.12.8'
+          python-version: '3.12.9'
           cache: 'pip'
       - name: Install Swirl
         run: ./install.sh

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12.8-slim-bookworm
+FROM python:3.12.9-slim-bookworm
 
 # Update, upgrade and install packages in a single RUN to reduce layers
 RUN apt-get update && apt-get install -y \

SearchProviders/google.json

@@ -1,8 +1,8 @@
 [
     {
         "name": "Google Web - pse.google.com",
-        "active": true,
-        "default": true,
+        "active": false,
+        "default": false,
         "connector": "RequestsGet",
         "url": "https://www.googleapis.com/customsearch/v1",
         "query_template": "{url}?cx={cx}&key={key}&q={query_string}",
@@ -33,15 +33,15 @@
         "query_processors": [
             "AdaptiveQueryProcessor"
         ],
-        "query_mappings": "cx=7d473806dcdde5bc6,DATE_SORT=sort=date,PAGE=start=RESULT_INDEX,NOT_CHAR=-",
+        "query_mappings": "cx=<your-Google-PSE-code>,DATE_SORT=sort=date,PAGE=start=RESULT_INDEX,NOT_CHAR=-",
         "result_processors": [
             "MappingResultProcessor",
             "DateFinderResultProcessor",
             "CosineRelevancyResultProcessor"
         ],
         "response_mappings": "FOUND=searchInformation.totalResults,RETRIEVED=queries.request[0].count,RESULTS=items",
         "result_mappings": "url=link,body=snippet,author=displayLink,cacheId,pagemap.metatags[*].['og:type'],pagemap.metatags[*].['og:site_name'],pagemap.metatags[*].['og:description'],NO_PAYLOAD",
-        "credentials": "key=AIzaSyDvVeE-L6nCC9u-TTGuhggvSmzhtiTHJsA",
+        "credentials": "key=<your-Google-API-key>",
         "tags": [
             "Web", "WWW", "Google"
         ]
@@ -87,7 +87,7 @@
         "query_processors": [
             "AdaptiveQueryProcessor"
         ],
-        "query_mappings": "cx=333906f8f74144c5c,DATE_SORT=sort=date,PAGE=start=RESULT_INDEX,NOT_CHAR=-",
+        "query_mappings": "cx=<your-Google-PSE-code>,DATE_SORT=sort=date,PAGE=start=RESULT_INDEX,NOT_CHAR=-",
         "result_grouping_field": "",
         "result_processors": [
             "MappingResultProcessor",
@@ -98,7 +98,7 @@
         "response_mappings": "FOUND=searchInformation.totalResults,RETRIEVED=queries.request[0].count,RESULTS=items",
         "result_mappings": "title= htmlTitle,url=link,body=snippet,cacheId,pagemap.metatags[*].['og:type'],pagemap.metatags[*].locale,pagemap.metatags[*].['og:image'],NO_PAYLOAD",
         "results_per_query": 10,
-        "credentials": "key=AIzaSyDvVeE-L6nCC9u-TTGuhggvSmzhtiTHJsA",
+        "credentials": "key=<your-Google-API-key>",
         "eval_credentials": "",
         "tags": [
             "LinkedIn",
@@ -107,8 +107,8 @@
     },
     {
         "name": "Swirl Documentation - docs.swirlaiconnect.com",
-        "active": true,
-        "default": true,
+        "active": false,
+        "default": false,
         "connector": "RequestsGet",
         "url": "https://www.googleapis.com/customsearch/v1",
         "query_template": "{url}?cx={cx}&key={key}&q={query_string}",
@@ -139,7 +139,7 @@
         "query_processors": [
             "AdaptiveQueryProcessor"
         ],
-        "query_mappings": "cx=f047885d9696f447c,DATE_SORT=sort=date,PAGE=start=RESULT_INDEX,NOT_CHAR=-",
+        "query_mappings": "cx=<your-Google-PSE-code>,DATE_SORT=sort=date,PAGE=start=RESULT_INDEX,NOT_CHAR=-",
         "result_grouping_field": "",
         "result_processors": [
             "MappingResultProcessor",
@@ -149,7 +149,7 @@
         "response_mappings": "FOUND=searchInformation.totalResults,RETRIEVED=queries.request[0].count,RESULTS=items",
         "result_mappings": "title=title,url=link,body=snippet,author=displayLink,cacheId,pagemap.metatags[*].['og:type'],pagemap.metatags[*].['og:site_name'],pagemap.metatags[*].['og:description'],NO_PAYLOAD",
         "results_per_query": 10,
-        "credentials": "key=AIzaSyDvVeE-L6nCC9u-TTGuhggvSmzhtiTHJsA",
+        "credentials": "key=<your-Google-API-key>",
         "eval_credentials": "",
         "tags": [
             "Swirl",

SearchProviders/preloaded.json

@@ -1,8 +1,8 @@
 [
     {
         "name": "Google Web - pse.google.com",
-        "active": true,
-        "default": true,
+        "active": false,
+        "default": false,
         "connector": "RequestsGet",
         "url": "https://www.googleapis.com/customsearch/v1",
         "query_template": "{url}?cx={cx}&key={key}&q={query_string}",
@@ -33,7 +33,7 @@
         "query_processors": [
             "AdaptiveQueryProcessor"
         ],
-        "query_mappings": "cx=7d473806dcdde5bc6,DATE_SORT=sort=date,PAGE=start=RESULT_INDEX,NOT_CHAR=-",
+        "query_mappings": "cx=<your-Google-PSE-code>,DATE_SORT=sort=date,PAGE=start=RESULT_INDEX,NOT_CHAR=-",
         "result_processors": [
             "MappingResultProcessor",
             "DateFinderResultProcessor",
@@ -42,7 +42,7 @@
         "response_mappings": "FOUND=searchInformation.totalResults,RETRIEVED=queries.request[0].count,RESULTS=items",
         "result_mappings": "url=link,body=snippet,author=displayLink,cacheId,pagemap.metatags[*].['og:type'],pagemap.metatags[*].['og:site_name'],pagemap.metatags[*].['og:description'],NO_PAYLOAD",
         "results_per_query": 10,
-        "credentials": "key=AIzaSyDvVeE-L6nCC9u-TTGuhggvSmzhtiTHJsA",
+        "credentials": "key=<your-Google-API-key>",
         "tags": [
             "Google",
             "Web",
@@ -90,7 +90,7 @@
         "query_processors": [
             "AdaptiveQueryProcessor"
         ],
-        "query_mappings": "cx=333906f8f74144c5c,DATE_SORT=sort=date,PAGE=start=RESULT_INDEX,NOT_CHAR=-",
+        "query_mappings": "cx=<your-Google-PSE-code>,DATE_SORT=sort=date,PAGE=start=RESULT_INDEX,NOT_CHAR=-",
         "result_grouping_field": "",
         "result_processors": [
             "MappingResultProcessor",
@@ -101,7 +101,7 @@
         "response_mappings": "FOUND=searchInformation.totalResults,RETRIEVED=queries.request[0].count,RESULTS=items",
         "result_mappings": "title= htmlTitle,url=link,body=snippet,cacheId,pagemap.metatags[*].['og:type'],pagemap.metatags[*].locale,pagemap.metatags[*].['og:image'],NO_PAYLOAD",
         "results_per_query": 10,
-        "credentials": "key=AIzaSyDvVeE-L6nCC9u-TTGuhggvSmzhtiTHJsA",
+        "credentials": "key=<your-Google-API-key>",
         "eval_credentials": "",
         "tags": [
             "LinkedIn",
@@ -110,8 +110,8 @@
     },
     {
         "name": "Swirl Documentation - docs.swirlaiconnect.com",
-        "active": true,
-        "default": true,
+        "active": false,
+        "default": false,
         "connector": "RequestsGet",
         "url": "https://www.googleapis.com/customsearch/v1",
         "query_template": "{url}?cx={cx}&key={key}&q={query_string}",
@@ -142,7 +142,7 @@
         "query_processors": [
             "AdaptiveQueryProcessor"
         ],
-        "query_mappings": "cx=f047885d9696f447c,DATE_SORT=sort=date,PAGE=start=RESULT_INDEX,NOT_CHAR=-",
+        "query_mappings": "cx=<your-Google-PSE-code>,DATE_SORT=sort=date,PAGE=start=RESULT_INDEX,NOT_CHAR=-",
         "result_grouping_field": "",
         "result_processors": [
             "MappingResultProcessor",
@@ -152,7 +152,7 @@
         "response_mappings": "FOUND=searchInformation.totalResults,RETRIEVED=queries.request[0].count,RESULTS=items",
         "result_mappings": "title=title,url=link,body=snippet,author=displayLink,cacheId,pagemap.metatags[*].['og:type'],pagemap.metatags[*].['og:site_name'],pagemap.metatags[*].['og:description'],NO_PAYLOAD",
         "results_per_query": 10,
-        "credentials": "key=AIzaSyDvVeE-L6nCC9u-TTGuhggvSmzhtiTHJsA",
+        "credentials": "key=<your-Google-API-key>",
         "eval_credentials": "",
         "tags": [
             "Swirl",