Merge pull request #146 from nansencenter/update_cmems_provider

Update CMEMS provider
nansencenter · Oct 8, 2024 · 2083f64 · 2083f64
2 parents 6e36560 + ed173db
commit 2083f64
Show file tree

Hide file tree

Showing 13 changed files with 1,822 additions and 47 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -12,15 +12,14 @@ jobs:
     strategy:
       matrix:
         python_version:
-          - '3.7'
-          - '3.8'
           - '3.9'
           - '3.10'
           - '3.11'
     env:
       BASE_IMAGE: "${{ vars.DOCKER_ORG }}/geospaas:2.5.2-python${{ matrix.python_version }}"
       IMAGE_NAME: "${{ vars.DOCKER_ORG }}/geospaas_harvesting"
       METANORM_VERSION: '4.2.2'
+      PYTHESINT_VERSION: 1.7.0.dev0
       GEOSPAAS_DB_HOST: 'db'
       GEOSPAAS_DB_USER: 'test'
       GEOSPAAS_DB_PASSWORD: "${{ secrets.GEOSPAAS_DB_PASSWORD }}"
@@ -58,6 +57,7 @@ jobs:
           build-args: |
             BASE_IMAGE=${{ env.BASE_IMAGE }}
             METANORM_VERSION=${{ env.METANORM_VERSION }}
+            PYTHESINT_VERSION=${{ env.PYTHESINT_VERSION }}
           push: false
           load: true
           tags: 'harvesting_tests'
@@ -109,6 +109,7 @@ jobs:
           build-args: |
             BASE_IMAGE=${{ env.BASE_IMAGE }}
             METANORM_VERSION=${{ env.METANORM_VERSION }}
+            PYTHESINT_VERSION=${{ env.PYTHESINT_VERSION }}
           push: ${{ github.event_name == 'release' }}
           tags: |
             ${{ env.IMAGE_NAME }}:${{ github.ref_name }}-python${{ matrix.python_version }}

diff --git a/Dockerfile b/Dockerfile
@@ -1,15 +1,19 @@
 ARG BASE_IMAGE=nansencenter/geospaas:latest
 
-FROM ${BASE_IMAGE} as base
+FROM ${BASE_IMAGE} AS base
 
 ARG METANORM_VERSION
 RUN pip install --upgrade --no-cache-dir \
     https://github.com/nansencenter/metanorm/releases/download/${METANORM_VERSION}/metanorm-${METANORM_VERSION}-py3-none-any.whl \
+    'copernicusmarine' \
     'feedparser==6.0.*' \
     'graypy==2.1.*' \
     'requests_oauthlib==1.3.*' \
     'tblib'
 
+ARG PYTHESINT_VERSION=''
+RUN bash -c "[ -n '$PYTHESINT_VERSION' ] && pip install --upgrade 'pythesint==$PYTHESINT_VERSION' || true"
+
 FROM base
 
 COPY . /tmp/setup

diff --git a/geospaas_harvesting/config.py b/geospaas_harvesting/config.py
@@ -60,7 +60,7 @@ class ProvidersArgument(DictArgument):
     provider_types = {
         'aviso': providers_aviso.AVISOProvider,
         'ceda': providers_ceda.CEDAProvider,
-        'cmems_ftp': providers_cmems.CMEMSFTPProvider,
+        'cmems': providers_cmems.CMEMSProvider,
         'copernicus_scihub': providers_copernicus_scihub.CopernicusScihubProvider,
         'earthdata_cmr': providers_earthdata_cmr.EarthDataCMRProvider,
         'ftp': providers_ftp.FTPProvider,

diff --git a/geospaas_harvesting/config.yml b/geospaas_harvesting/config.yml
@@ -8,7 +8,7 @@ providers:
     username: 'anonymous'
     password: ''
   cmems:
-    type: 'cmems_ftp'
+    type: 'cmems'
     username: !ENV CMEMS_USERNAME
     password: !ENV CMEMS_PASSWORD
   cmems_in_situ:

diff --git a/geospaas_harvesting/crawlers.py b/geospaas_harvesting/crawlers.py
@@ -46,6 +46,9 @@ def __init__(self, url, metadata=None):
         self.url = url
         self.metadata = metadata
 
+    def __repr__(self):
+        return f"DatasetInfo(url='{self.url}', metadata={self.metadata})"
+
     def __eq__(self, other):
         return self.url == other.url and self.metadata == other.metadata