Specify the version for files downloaded from the CDN. (#48)

* Specify the version for files downloaded from the CDN.

* Missed setting variable.

Author: cjohns-scottlogic

bin/download-collection.sh

@@ -3,6 +3,7 @@
 set -e
 
 s3="https://files.planning.data.gov.uk/"
+timestamp=`date +%s`
 
 csvcut -c collection specification/collection.csv | tail -n +2 |
 while read collection
@@ -14,7 +15,7 @@ do
         if [ ! -f $path ] ; then
             mkdir -p $dir
             set -x
-            curl -qsfL $flags --retry 3 -o $path "$s3$collection-collection/collection/$file"
+            curl -qsfL $flags --retry 3 -o $path "$s3$collection-collection/collection/$file?version=$timestamp"
             set +x
         fi
     done

bin/download-column-field.sh

@@ -2,6 +2,7 @@
 
 s3="https://files.planning.data.gov.uk/"
 column_field_dir="column-field/"
+timestamp=`date +%s`
 
 python3 bin/resources.py |
 while read collection pipeline resource
@@ -14,7 +15,7 @@ do
     if [ ! -f $path ] ; then
         mkdir -p $dir
         set -x
-        curl -qsfL $flags "$s3$collection-collection/var/column-field/$pipeline/$resource.csv" > $path
+        curl -qsfL $flags "$s3$collection-collection/var/column-field/$pipeline/$resource.csv?version=$timestamp" > $path
         set +x
     fi
 done

bin/download-converted-resources.sh

@@ -3,6 +3,7 @@ set -e
 
 s3="https://files.planning.data.gov.uk/"
 dir=var/converted-resource
+timestamp=`date +%s`
 
 python3 bin/resources.py |
 while read collection pipeline resource
@@ -14,7 +15,7 @@ do
     if [ ! -f $path ] ; then
         mkdir -p $dir/$pipeline
         set -x
-        curl -qsfL $flags "$s3$collection-collection/$dir/$pipeline/$resource.csv" -o $path ||:
+        curl -qsfL $flags "$s3$collection-collection/$dir/$pipeline/$resource.csv?version=$timestamp" -o $path ||:
         set +x
     fi
 done

bin/download-digital-land.sh

@@ -3,12 +3,13 @@
 set -e
 
 s3="https://files.planning.data.gov.uk/"
+timestamp=`date +%s`
 
 # Directory to save the database
 db_dir="dataset"
 db_file="digital-land.sqlite3"
 db_path="${db_dir}/${db_file}"
-db_url="${s3}digital-land-builder/dataset/${db_file}"
+db_url="${s3}digital-land-builder/dataset/${db_file}?version=${timestamp}"
 
 # Create directory if it does not exist
 mkdir -p $db_dir

bin/download-expectations.sh

@@ -4,6 +4,7 @@ set -e
 
 s3="https://files.planning.data.gov.uk/"
 expectations_dir="expectations/"
+timestamp=`date +%s`
 
 dir=var/$expectations_dir
 mkdir -p $dir
@@ -19,7 +20,7 @@ do
             path=$dir$file
             if [ ! -f $path ] ; then
                 set -x
-                if ! curl -qsfL $flags --retry 3 -o $path $s3$collection-collection/dataset/$file; then
+                if ! curl -qsfL $flags --retry 3 -o $path $s3$collection-collection/dataset/$file?version=$timestamp ; then
                     echo "*** UNABLE TO DOWNLOAD $collection FROM $path ***"
                 fi
                 set +x

bin/download-issues.sh

@@ -3,6 +3,7 @@
 #set -e
 
 s3="https://files.planning.data.gov.uk/"
+timestamp=`date +%s`
 
 python3 bin/resources.py |
 while read collection pipeline resource
@@ -14,7 +15,7 @@ do
     if [ ! -f $path ] ; then
         mkdir -p $dir
         set -x
-        curl -qsfL $flags "$s3$collection-collection/issue/$pipeline/$resource.csv" > $path
+        curl -qsfL $flags "$s3$collection-collection/issue/$pipeline/$resource.csv?version=$timestamp" > $path
         set +x
     fi
 done

bin/download-operational-issues.sh

@@ -4,6 +4,7 @@
 
 s3="https://files.planning.data.gov.uk/"
 operational_issue_dir="performance/operational_issue/"
+timestamp=`date +%s`
 
 mkdir -p $operational_issue_dir
 
@@ -15,7 +16,7 @@ do
     if [ ! -f $path ] ; then
         mkdir -p $dir
         set -x
-        curl -qsfL $flags "$s3$operational_issue_dir$dataset/operational-issue.csv" > $path
+        curl -qsfL $flags "$s3$operational_issue_dir$dataset/operational-issue.csv?version=$timestamp" > $path
         set +x
     fi
 done

bin/download-resources.sh

@@ -3,6 +3,7 @@
 set -e
 
 s3="https://files.planning.data.gov.uk/"
+timestamp=`date +%s`
 
 python3 bin/resources.py |
 while read collection pipeline resource
@@ -14,7 +15,7 @@ do
     if [ ! -f $path ] ; then
         mkdir -p $dir
         set -x
-        curl -qsfL $flags "$s3$collection-collection/collection/resource/$resource" > $path
+        curl -qsfL $flags "$s3$collection-collection/collection/resource/$resource?version=$timestamp" > $path
         set +x
     fi
 done