fixup csv uploads to preserve entry data and existing prefix

Author: ashimali

.flaskenv

@@ -1,5 +1,5 @@
 FLASK_DEBUG=True
-FLASK_CONFIG=config.DevelopmentConfig
+FLASK_CONFIG=application.config.DevelopmentConfig
 FLASK_APP=application.wsgi:app
 SECRET_KEY=replaceinprod
 DATABASE_URL=postgresql://localhost/dluhc-datasets

application/blueprints/main/views.py

@@ -7,6 +7,7 @@
 from flask import (
     Blueprint,
     abort,
+    current_app,
     flash,
     make_response,
     redirect,
@@ -176,7 +177,11 @@ def add_record(id):
             data["start-date"] = start_date
 
         # set prefix to as it is not in form
-        data["prefix"] = dataset.dataset
+        data["prefix"] = (
+            dataset.dataset
+            if dataset.dataset not in current_app.config.WIKIDATA_PREFIX_DATASETS
+            else "wikidata"
+        )
         data["entry-date"] = datetime.datetime.today().strftime("%Y-%m-%d")
 
         last_record = (

application/commands.py

@@ -727,6 +727,22 @@ def remove_orgs():
         db.session.commit()
 
 
+@data_cli.command("wikidata-prefix")
+def wikidata_prefix():
+    from flask import current_app
+
+    for dataset in current_app.config["WIKIDATA_PREFIX_DATASETS"]:
+        print(f"Setting wikidata prefix for {dataset}")
+        dataset = Dataset.query.filter(Dataset.dataset == dataset).one_or_none()
+        if dataset is None:
+            print(f"Dataset {dataset} does not exist")
+            continue
+        for record in dataset.records:
+            record.prefix = "wikidata"
+            db.session.add(record)
+        db.session.commit()
+
+
 def _get_repo(config):
     app_id = config.get("GITHUB_APP_ID")
     repo_name = config.get("DATASETS_REPO")

config/config.py application/config.py

@@ -24,6 +24,17 @@ class Config:
     SPECIFICATION_REPO_URL = os.getenv("SPECIFICATION_REPO_URL")
     PLATFORM_URL = os.getenv("PLATFORM_URL")
     PLANNING_DATA_DESIGN_URL = os.getenv("PLANNING_DATA_DESIGN_URL")
+    WIKIDATA_PREFIX_DATASETS = set(
+        [
+            "development-corporation",
+            "national-park-authority",
+            "nonprofit",
+            "public-authority",
+            "passenger-transport-executive",
+            "regional-park-authority",
+            "waste-authority",
+        ]
+    )
 
 
 class DevelopmentConfig(Config):

application/wsgi.py

@@ -2,4 +2,4 @@
 
 from application.factory import create_app
 
-app = create_app(os.getenv("FLASK_CONFIG") or "config.DevelopmentConfig")
+app = create_app(os.getenv("FLASK_CONFIG") or "application.config.DevelopmentConfig")

config/__init__.py

@@ -10,12 +10,12 @@
 def app():
     from application.factory import create_app
 
-    app = create_app("config.TestConfig")
+    app = create_app("application.config.TestConfig")
     yield app
 
 
 fields = [
-    Field(field="organisation", datatype="curie", name="Orgasnisation"),
+    Field(field="organisation", datatype="curie", name="Organisation"),
     Field(field="prefix", datatype="string", name="Prefix"),
     Field(field="reference", datatype="string", name="Reference"),
     Field(field="documentation-url", datatype="url", name="Documentation url"),