Additional pipelines geocodeConfig, vocabularies and location configuration (#591)

* Additional geocodeConfig variables

* Added download of vocabularies and configuration. Customization of stateProvinces centroids

* Added tag

Author: vjrj

ansible/roles/pipelines/tasks/main.yml

@@ -146,6 +146,7 @@
   tags:
     - pipelines
     - la-pipelines
+    - la-pipelines-apt
     - apt
 
 - name: Install pipelines config
@@ -362,6 +363,45 @@
     - hadoop
     - hadoop_dir
 
+- name: Get Gbif API vocabularies
+  run_once: true
+  uri:
+    url: "http://api.gbif.org/v1/vocabularies/{{ item }}/releases/latest/export"
+    dest: /tmp/{{ item }}.json
+  with_items:
+    - LifeStage
+    - DegreeOfEstablishment
+    - EstablishmentMeans
+    - Pathway
+  tags:
+    - pipelines
+    - hadoop
+    - hadoop_vocab
+
+- name: Copy Gbif API vocabularies to dfs
+  run_once: true
+  shell: "{{ hadoop_install_dir }}/bin/hdfs dfs -copyFromLocal -f /tmp/{{ item }}.json /pipelines-vocabularies/ "
+  with_items:
+    - LifeStage
+    - DegreeOfEstablishment
+    - EstablishmentMeans
+    - Pathway
+  become_user: "{{ hadoop.user }}"
+  tags:
+    - pipelines
+    - hadoop
+    - hadoop_vocab
+
+- name: Copy state/province vocab
+  copy: src={{ item.src }} dest={{ data_dir }}/la-pipelines/config/{{ item.dest }} mode=0644
+  with_items:
+    - { src: "{{ state_province_centre_points_file }}", dest: stateProvinceCentrePoints.txt }
+  when: state_province_centre_points_file is defined
+  tags:
+    - pipelines
+    - hadoop
+    - hadoop_vocab
+
 - name: Download SHP files (includes global layer and EEZ derived from GBIFs geocode DB)
   get_url:
     url: "{{ pipelines_shapefiles_url | default('https://pipelines-shp.s3-ap-southeast-2.amazonaws.com/pipelines-shapefiles.zip') }}"

ansible/roles/pipelines/templates/la-pipelines-local.yaml

@@ -34,14 +34,22 @@ speciesListService:
 
 geocodeConfig:
   country:
+    path: {{ data_dir }}/pipelines-shp/{{ geocode_country_layer | default('political') }}
+    field: {{ geocode_country_field | default('ISO_A2') }}
     intersectMapping:
       CX: {{geocode_region | default('AU')}}
       CC: {{geocode_region | default('AU')}}
       HM: {{geocode_region | default('AU')}}
       NF: {{geocode_region | default('AU')}}
   stateProvince:
     path: /data/pipelines-shp/{{ geocode_state_province_layer | default('cw_state_poly') }}
+    field: {{ geocode_state_province_field | default('FEATURE') }}
 
+{% if state_province_centre_points_file is defined %}
+locationInfoConfig:
+    stateProvinceCentrePointsFile : {{ data_dir }}/la-pipelines/config/stateProvinceCentrePoints.txt
+
+{% endif %}
 gbifConfig:
   vocabularyConfig:
     vocabulariesPath: hdfs://{{ hadoop_master }}:{{ hadoop_port }}{{ vocabulary_dir }}