From b09494c615307ac6affd1dda65749869981cc6be Mon Sep 17 00:00:00 2001 From: Harshil Khamar Date: Fri, 27 Oct 2023 16:25:40 +0530 Subject: [PATCH 1/5] Yahoo Search Added --- SearchProviders/preloaded.json | 24 +++++++ SearchProviders/yahoosearch.json | 24 +++++++ swirl/connectors/yahoosearch.py | 103 +++++++++++++++++++++++++++++++ swirl/models.py | 1 + swirl_server/settings.py | 4 ++ 5 files changed, 156 insertions(+) create mode 100644 SearchProviders/yahoosearch.json create mode 100644 swirl/connectors/yahoosearch.py diff --git a/SearchProviders/preloaded.json b/SearchProviders/preloaded.json index 80d22b62a..5da3c5e61 100644 --- a/SearchProviders/preloaded.json +++ b/SearchProviders/preloaded.json @@ -1158,5 +1158,29 @@ "Swirl", "Documentation" ] + }, + { + "name": "Yahoo Search", + "active": false, + "default": true, + "connector": "yahoosearch", + "url": "", + "query_template": "", + "query_processors": [ + "AdaptiveQueryProcessor" + ], + "query_mappings": "p='{query_to_provider}'", + "result_processors": [ + "GenericResultProcessor" + ], + "response_mappings": "", + "result_mappings": "BLOCK=ai_summary", + "results_per_query": 10, + "credentials": "your-serpapi-API-key-here", + "tags": [ + "Yahoo", + "Query", + "SearchEngine" + ] } ] diff --git a/SearchProviders/yahoosearch.json b/SearchProviders/yahoosearch.json new file mode 100644 index 000000000..c211fd4e4 --- /dev/null +++ b/SearchProviders/yahoosearch.json @@ -0,0 +1,24 @@ +{ + "name": "Yahoo Search", + "active": false, + "default": true, + "connector": "yahoosearch", + "url": "", + "query_template": "", + "query_processors": [ + "AdaptiveQueryProcessor" + ], + "query_mappings": "p='{query_to_provider}'", + "result_processors": [ + "GenericResultProcessor" + ], + "response_mappings": "", + "result_mappings": "BLOCK=ai_summary", + "results_per_query": 10, + "credentials": "your-serpapi-API-key-here", + "tags": [ + "Yahoo", + "Query", + "SearchEngine" + ] +} \ No newline at end of file diff --git a/swirl/connectors/yahoosearch.py b/swirl/connectors/yahoosearch.py new file mode 100644 index 000000000..5cd0ee890 --- /dev/null +++ b/swirl/connectors/yahoosearch.py @@ -0,0 +1,103 @@ + +''' +@author: Harshil Khamar +@contact: harshilkhamar1@gmail.com +''' + +from sys import path +from os import environ + +import django + +from swirl.utils import swirl_setdir +from serpapi import GoogleSearch +path.append(swirl_setdir()) # path to settings.py file +environ.setdefault('DJANGO_SETTINGS_MODULE', 'swirl_server.settings') +django.setup() + +from django.conf import settings + +from celery.utils.log import get_task_logger +from logging import DEBUG, INFO, WARNING +logger = get_task_logger(__name__) + +from swirl.connectors.mappings import * +from swirl.connectors.connector import Connector + +from datetime import datetime + +from serpapi import YahooSearch + +class YahooSearch(Connector): + + type = "YahooSearch" + + def __init__(self, provider_id, search_id, update, request_id=''): + super().__init__(provider_id, search_id, update, request_id) + + + def execute_search(self, session=None): + apikey = None + if self.provider.credentials: + apikey = self.provider.credentials + else: + if getattr(settings, 'SERPAPI_API_KEY', None): + apikey = settings.SERPAPI_API_KEY + else: + self.found = 0 + self.retrieved = 0 + self.response = [] + self.status = "ERR_NO_CREDENTIALS" + return + + prompted_query = "" + if self.query_to_provider.endswith('?'): + prompted_query = self.query_to_provider + else: + if 'p' in self.query_mappings: + prompted_query = self.query_mappings['p'].format(query_to_provider=self.query_to_provider) + else: + prompted_query = self.query_to_provider + self.warning(f'p not found in query_mappings!') + + if not prompted_query: + self.found = 0 + self.retrieved = 0 + self.response = [] + self.status = "ERR_PROMPT_FAILED" + return + + self.query_to_provider = prompted_query + params = { + "p": "coffee", + "engine": "yahoo", + "api_key": apikey, + "output": "json" + } + search = GoogleSearch(params) + data = search.get_dict() + completions = data + message = completions['organic_results'] # FROM API Doc + + self.found = 1 + self.retrieved = 1 + self.response = message + + return + + ######################################## + + def normalize_response(self): + + logger.debug(f"{self}: normalize_response()") + + self.results = [ + { + 'title': self.query_string_to_provider, + 'body': f'{self.response}', + 'author': 'YahooSearch', + 'date_published': str(datetime.now()) + } + ] + + return diff --git a/swirl/models.py b/swirl/models.py index 837f3cf0f..be7ca93df 100644 --- a/swirl/models.py +++ b/swirl/models.py @@ -68,6 +68,7 @@ class SearchProvider(models.Model): }) CONNECTOR_CHOICES = [ ('ChatGPT', 'ChatGPT Query String'), + ('YahooSearch', 'YahooSearch Query String'), ('RequestsGet', 'HTTP/GET returning JSON'), ('RequestsPost', 'HTTP/POST returning JSON'), ('Elastic', 'Elasticsearch Query String'), diff --git a/swirl_server/settings.py b/swirl_server/settings.py index 2b85b7561..a914e6ec2 100644 --- a/swirl_server/settings.py +++ b/swirl_server/settings.py @@ -267,6 +267,10 @@ if 'OPENAI_API_KEY' in env: OPENAI_API_KEY = env('OPENAI_API_KEY') +if 'SERPAPI_API_KEY' in env: + SERPAPI_API_KEY = env('SERPAPI_API_KEY') + + MICROSOFT_CLIENT_ID= env('MICROSOFT_CLIENT_ID') MICROSOFT_CLIENT_SECRET = env('MICROSOFT_CLIENT_SECRET') MICROSOFT_REDIRECT_URI = env('MICROSOFT_REDIRECT_URI') From 8f4ae5948c7f9459965e7d89ada399b9919a16eb Mon Sep 17 00:00:00 2001 From: Harshil Khamar Date: Fri, 3 Nov 2023 03:23:29 +0530 Subject: [PATCH 2/5] Changes Done --- SearchProviders/preloaded.json | 24 ------------------- SearchProviders/yahoosearch.json | 8 ++++--- .../connectors/{ => untested}/yahoosearch.py | 0 swirl/models.py | 1 - swirl_server/settings.py | 4 ---- 5 files changed, 5 insertions(+), 32 deletions(-) rename swirl/connectors/{ => untested}/yahoosearch.py (100%) diff --git a/SearchProviders/preloaded.json b/SearchProviders/preloaded.json index 5da3c5e61..80d22b62a 100644 --- a/SearchProviders/preloaded.json +++ b/SearchProviders/preloaded.json @@ -1158,29 +1158,5 @@ "Swirl", "Documentation" ] - }, - { - "name": "Yahoo Search", - "active": false, - "default": true, - "connector": "yahoosearch", - "url": "", - "query_template": "", - "query_processors": [ - "AdaptiveQueryProcessor" - ], - "query_mappings": "p='{query_to_provider}'", - "result_processors": [ - "GenericResultProcessor" - ], - "response_mappings": "", - "result_mappings": "BLOCK=ai_summary", - "results_per_query": 10, - "credentials": "your-serpapi-API-key-here", - "tags": [ - "Yahoo", - "Query", - "SearchEngine" - ] } ] diff --git a/SearchProviders/yahoosearch.json b/SearchProviders/yahoosearch.json index c211fd4e4..7a77a709b 100644 --- a/SearchProviders/yahoosearch.json +++ b/SearchProviders/yahoosearch.json @@ -6,14 +6,16 @@ "url": "", "query_template": "", "query_processors": [ - "AdaptiveQueryProcessor" + "AdaptiveQueryProcessor", + "MappingResultProcessor", + "CosineRelevancyResultProcessor" ], "query_mappings": "p='{query_to_provider}'", "result_processors": [ "GenericResultProcessor" ], - "response_mappings": "", - "result_mappings": "BLOCK=ai_summary", + "response_mappings": "RESULTS=body", + "result_mappings": "title=title,body=body,author=author,date_published=date_published", "results_per_query": 10, "credentials": "your-serpapi-API-key-here", "tags": [ diff --git a/swirl/connectors/yahoosearch.py b/swirl/connectors/untested/yahoosearch.py similarity index 100% rename from swirl/connectors/yahoosearch.py rename to swirl/connectors/untested/yahoosearch.py diff --git a/swirl/models.py b/swirl/models.py index be7ca93df..837f3cf0f 100644 --- a/swirl/models.py +++ b/swirl/models.py @@ -68,7 +68,6 @@ class SearchProvider(models.Model): }) CONNECTOR_CHOICES = [ ('ChatGPT', 'ChatGPT Query String'), - ('YahooSearch', 'YahooSearch Query String'), ('RequestsGet', 'HTTP/GET returning JSON'), ('RequestsPost', 'HTTP/POST returning JSON'), ('Elastic', 'Elasticsearch Query String'), diff --git a/swirl_server/settings.py b/swirl_server/settings.py index c9b5343d5..a097bf824 100644 --- a/swirl_server/settings.py +++ b/swirl_server/settings.py @@ -266,10 +266,6 @@ OPENAI_API_KEY = env.get_value('OPENAI_API_KEY', default='') -if 'SERPAPI_API_KEY' in env: - SERPAPI_API_KEY = env('SERPAPI_API_KEY') - - MICROSOFT_CLIENT_ID= env('MICROSOFT_CLIENT_ID') MICROSOFT_CLIENT_SECRET = env('MICROSOFT_CLIENT_SECRET') MICROSOFT_REDIRECT_URI = env('MICROSOFT_REDIRECT_URI') From d2b1ac86342487c4fe02c63d0748c1d268ab8f5e Mon Sep 17 00:00:00 2001 From: Harshil Khamar Date: Fri, 3 Nov 2023 03:31:16 +0530 Subject: [PATCH 3/5] yahoosearch.json moved --- SearchProviders/{ => untested}/yahoosearch.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename SearchProviders/{ => untested}/yahoosearch.json (100%) diff --git a/SearchProviders/yahoosearch.json b/SearchProviders/untested/yahoosearch.json similarity index 100% rename from SearchProviders/yahoosearch.json rename to SearchProviders/untested/yahoosearch.json From b3b7ec78d5f2541ad62585f9f3d1bff8d1d6ddfe Mon Sep 17 00:00:00 2001 From: Harshil Khamar Date: Fri, 3 Nov 2023 03:44:50 +0530 Subject: [PATCH 4/5] Result Mapping Updated --- SearchProviders/untested/yahoosearch.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SearchProviders/untested/yahoosearch.json b/SearchProviders/untested/yahoosearch.json index 7a77a709b..56584e7c6 100644 --- a/SearchProviders/untested/yahoosearch.json +++ b/SearchProviders/untested/yahoosearch.json @@ -15,7 +15,7 @@ "GenericResultProcessor" ], "response_mappings": "RESULTS=body", - "result_mappings": "title=title,body=body,author=author,date_published=date_published", + "result_mappings": "body=body,author=author,date_published=date_published", "results_per_query": 10, "credentials": "your-serpapi-API-key-here", "tags": [ From 88fa469432bcbeef0068fcd630f6884929e65b02 Mon Sep 17 00:00:00 2001 From: Harshil Khamar Date: Fri, 3 Nov 2023 04:25:30 +0530 Subject: [PATCH 5/5] Title Added --- SearchProviders/untested/yahoosearch.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SearchProviders/untested/yahoosearch.json b/SearchProviders/untested/yahoosearch.json index 56584e7c6..7a77a709b 100644 --- a/SearchProviders/untested/yahoosearch.json +++ b/SearchProviders/untested/yahoosearch.json @@ -15,7 +15,7 @@ "GenericResultProcessor" ], "response_mappings": "RESULTS=body", - "result_mappings": "body=body,author=author,date_published=date_published", + "result_mappings": "title=title,body=body,author=author,date_published=date_published", "results_per_query": 10, "credentials": "your-serpapi-API-key-here", "tags": [