diff --git a/config/settings.cfg.tmpl b/config/settings.cfg.tmpl index 76501e8..f84b3d9 100644 --- a/config/settings.cfg.tmpl +++ b/config/settings.cfg.tmpl @@ -14,6 +14,9 @@ GRQ_INDEX = "grq" # ElasticSearch geonames index GEONAMES_INDEX = "geonames" +# timeout value for ElasticSearch bulk requests (defaults to 10) +BULK_REQUEST_TIMEOUT = 30 + # Redis URL REDIS_URL = "redis://{{ MOZART_REDIS_PVT_IP }}:6379/0" diff --git a/grq2/services/api_v02/datasets.py b/grq2/services/api_v02/datasets.py index bbaf10c..35b8c1e 100644 --- a/grq2/services/api_v02/datasets.py +++ b/grq2/services/api_v02/datasets.py @@ -123,6 +123,9 @@ class IndexDataset(Resource): @grq_ns.marshal_with(resp_model) @grq_ns.expect(parser, validate=True) def post(self): + # get bulk request timeout from config + bulk_request_timeout = app.config.get('BULK_REQUEST_TIMEOUT', 10) + try: datasets = json.loads(request.json) @@ -134,14 +137,14 @@ def post(self): docs_bulk.append({"index": {"_index": index, "_id": _id}}) docs_bulk.append(ds) - response = grq_es.es.bulk(body=docs_bulk) + response = grq_es.es.bulk(body=docs_bulk, request_timeout=bulk_request_timeout) if response["errors"] is True: app.logger.error(response) delete_docs = [] for doc in docs_bulk: if "index" in doc: delete_docs.append({"delete": doc["index"]}) - grq_es.es.bulk(delete_docs) + grq_es.es.bulk(delete_docs, request_timeout=bulk_request_timeout) return { "success": False, "message": response["items"]