add configuration for bulk request timeout (#57)

hysds · Mar 21, 2023 · 60ab5fd · 60ab5fd
1 parent 7e7b0ff
commit 60ab5fd
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/config/settings.cfg.tmpl b/config/settings.cfg.tmpl
@@ -14,6 +14,9 @@ GRQ_INDEX = "grq"
 # ElasticSearch geonames index
 GEONAMES_INDEX = "geonames"
 
+# timeout value for ElasticSearch bulk requests (defaults to 10)
+BULK_REQUEST_TIMEOUT = 30
+
 # Redis URL
 REDIS_URL = "redis://{{ MOZART_REDIS_PVT_IP }}:6379/0"
 

diff --git a/grq2/services/api_v02/datasets.py b/grq2/services/api_v02/datasets.py
@@ -123,6 +123,9 @@ class IndexDataset(Resource):
     @grq_ns.marshal_with(resp_model)
     @grq_ns.expect(parser, validate=True)
     def post(self):
+        # get bulk request timeout from config
+        bulk_request_timeout = app.config.get('BULK_REQUEST_TIMEOUT', 10)
+
         try:
             datasets = json.loads(request.json)
 
@@ -134,14 +137,14 @@ def post(self):
                 docs_bulk.append({"index": {"_index": index, "_id": _id}})
                 docs_bulk.append(ds)
 
-            response = grq_es.es.bulk(body=docs_bulk)
+            response = grq_es.es.bulk(body=docs_bulk, request_timeout=bulk_request_timeout)
             if response["errors"] is True:
                 app.logger.error(response)
                 delete_docs = []
                 for doc in docs_bulk:
                     if "index" in doc:
                         delete_docs.append({"delete": doc["index"]})
-                grq_es.es.bulk(delete_docs)
+                grq_es.es.bulk(delete_docs, request_timeout=bulk_request_timeout)
                 return {
                     "success": False,
                     "message": response["items"]