Added award, fund and quality

Author: psd

Makefile

@@ -96,6 +96,9 @@ specification::
 	curl -qfsL '$(SOURCE_URL)/specification/main/specification/provision.csv' > specification/provision.csv
 	curl -qfsL '$(SOURCE_URL)/specification/main/specification/provision-rule.csv' > specification/provision-rule.csv
 	curl -qfsL '$(SOURCE_URL)/specification/main/specification/provision-reason.csv' > specification/provision-reason.csv
+	curl -qfsL '$(SOURCE_URL)/specification/main/specification/fund.csv' > specification/fund.csv
+	curl -qfsL '$(SOURCE_URL)/specification/main/specification/award.csv' > specification/award.csv
+	curl -qfsL '$(SOURCE_URL)/specification/main/specification/quality.csv' > specification/quality.csv
 
 check-performance::
-	python check/performance.py
+	python check/performance.py

bin/guess-lpa.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+import csv
+
+
+organisations = {}
+datasets = {}
+
+
+for row in csv.DictReader(open("collection/source.csv")):
+    organisation = row["organisation"]
+    pipelines = row.get("datasets", "") or row.get("pipelines", "")
+
+    for dataset in pipelines.split(";"):
+        organisations.setdefault(organisation, set())
+        organisations[organisation].add(dataset)
+
+        datasets.setdefault(dataset, set())
+        datasets[dataset].add(organisation)
+
+
+#for organisation in sorted(organisations):
+    #if not (organisation.startswith("local-authority") or organisation.startswith("development-corporation") or organisation.startswith("national-park")): 
+        #print('"' + organisation + '":', sorted(organisations[organisation]))
+
+for organisation in sorted(organisations):
+    if (organisation.startswith("local-authority") or organisation.startswith("development-corporation") or organisation.startswith("national-park")): 
+        print(organisation)

bin/load.py

@@ -4,7 +4,6 @@
 
 import os
 import sys
-import csv
 import logging
 import sqlite3
 
@@ -23,12 +22,14 @@
     "old-resource": "collection",
     "log": "collection",
 
+    "award": "specification",
     "collection": "specification",
     "theme": "specification",
     "typology": "specification",
     "dataset": "specification",
     "dataset-field": "specification",
     "field": "specification",
+    "fund": "specification",
     "datatype": "specification",
     "prefix": "specification",
     "severity": "specification",
@@ -46,6 +47,8 @@
     "provision": "specification",
     "provision-rule": "specification",
     "provision-reason": "specification",
+    "quality": "specification",
+    "datatype": "specification",
     "specification": "specification",
     "specification-status": "specification",
 
@@ -206,4 +209,3 @@ def create_parquet_from_table(df, name, output_dir):
         where t1.rn = 1              
     """)
     conn.close()
-