add cronjob to clean successful jobs

Dockerfile

@@ -23,7 +23,7 @@ FROM wmoim/dim_eccodes_baseimage:2.31.0
 ENV PYGEOAPI_CONFIG=/data/wis2box/config/pygeoapi/local.config.yml
 ENV PYGEOAPI_OPENAPI=/data/wis2box/config/pygeoapi/local.openapi.yml
 
-RUN apt-get update -y && apt-get install curl python3-pip git unzip -y
+RUN apt-get update -y && apt-get install cron curl python3-pip git unzip -y
 # install gunicorn, gevent, gdal, elasticsearch
 RUN apt-get install -y --no-install-recommends \
     libgdal-dev gunicorn python3-gevent python3-gdal python3-elasticsearch libudunits2-dev \
@@ -48,4 +48,9 @@ RUN cd /app \
     && pip3 install -e . \
     && chmod +x /app/docker/es-entrypoint.sh /app/docker/wait-for-elasticsearch.sh
 
+# add wis2box.cron to crontab
+COPY ./docker/wis2box-api.cron /etc/cron.d/wis2box-api.cron
+
+RUN chmod 0644 /etc/cron.d/wis2box-api.cron && crontab /etc/cron.d/wis2box-api.cron
+
 ENTRYPOINT [ "/app/docker/es-entrypoint.sh" ]

README.md

@@ -80,3 +80,4 @@ All bugs, enhancements and issues are managed on [GitHub](https://github.com/wmo
 ## Contact
 
 * [Tom Kralidis](https://github.com/tomkralidis)
+* [Maaike Limper](https://github.com/maaikelimper)

docker/clean-old-jobs.py

@@ -0,0 +1,78 @@
+###############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# 'License'); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+###############################################################################
+
+# a simple script to clean old jobs from the api
+
+import logging
+import os
+
+from datetime import datetime
+
+import requests
+
+logging.basicConfig(level=logging.INFO)
+
+LOGGER = logging.getLogger('clean-old-jobs')
+JOB_RETENTION_MINUTES = int(os.environ.get('JOB_RETENTION_MINUTES', 60))
+LOCAL_API_URL = os.environ.get('LOCAL_API_URL', 'http://localhost/oapi')
+
+
+def clean_jobs():
+    # get jobs
+    url = f'{LOCAL_API_URL}/jobs'
+
+    headers = {
+        'Accept': 'application/json'
+    }
+
+    try:
+        response = requests.get(url, headers=headers)
+        result = response.json()
+        if len(result['jobs']) == 0:
+            LOGGER.info('No jobs to clean')
+            return
+        jobs = result['jobs']
+    except Exception as err:
+        LOGGER.error(f'Error getting jobs: {err}')
+        return
+
+    job_ids = []
+    for job in jobs:
+        if job.get('job_end_datetime', None) is None:
+            continue
+        if job.get('status', 'unknown') != 'successful':
+            continue
+        endtime_str = job['job_end_datetime']
+        endtime = datetime.strptime(endtime_str, '%Y-%m-%dT%H:%M:%S.%fZ')
+        minutes_since_end = (datetime.utcnow() - endtime).total_seconds() / 60
+        logging.debug(f'Job {job["jobID"]} ended {minutes_since_end} min. ago')
+        if minutes_since_end > JOB_RETENTION_MINUTES:
+            job_ids.append(job['jobID'])
+
+    LOGGER.info(f'Found {len(job_ids)} jobs to clean')
+    for id in job_ids:
+        LOGGER.info(f'Cleaning job {id}')
+        url = f'{LOCAL_API_URL}/jobs/{id}'
+        requests.delete(url, headers=headers)
+
+
+if __name__ == '__main__':
+    clean_jobs()

docker/entrypoint.sh

@@ -26,6 +26,13 @@ echo "START /entrypoint.sh"
 
 set +e
 
+#ensure environment-variables are available for cronjob
+printenv | grep -v "no_proxy" >> /etc/environment
+
+# ensure cron is running
+service cron start
+service cron status
+
 # gunicorn env settings with defaults
 SCRIPT_NAME="/"
 CONTAINER_NAME="wis2box-api"

docker/wis2box-api.cron

@@ -0,0 +1 @@
+*/5 * * * * python3 /app/docker/clean-old-jobs.py > /proc/1/fd/1 2>/proc/1/fd/2

wis2box_api/__init__.py

@@ -19,4 +19,4 @@
 #
 ###############################################################################
 
-__version__ = '0.7.dev0'
+__version__ = 'wis2box-1.0b7-dev'