Skip to content

Commit 01dbae3

Browse files
djtfmartindjtfmartin
authored
Events prod deployment (#635)
* Scripts for AtlasOfLivingAustralia/extended-data-model#80 * service fixes * set initial_master_nodes, index_buffer_size, configurable cluster.name * es_index config * updated docker images * changes to allow multiple environments on the same VM
1 parent 9c1ce12 commit 01dbae3

File tree

11 files changed

+259
-0
lines changed

11 files changed

+259
-0
lines changed

‎ansible/events.yml

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
- hosts: events
2+
roles:
3+
- common
4+
- webserver
5+
- events
6+
vars:
7+
docker_dir: "/data/docker"
8+
9+
- hosts: elasticsearch
10+
vars:
11+
es_version: "7.17.7"
12+
es_heap_dump_path: "/data/"
13+
es_api_host: "{{ elasticsearch_host | default('localhost') }}"
14+
es_heap_size: "8g"
15+
es_config:
16+
cluster.name: "{{ elasticsearch_cluster_name | default('extended-data') }}"
17+
node.name: "{{ elasticsearch_host | default('localhost') }}"
18+
network.host: "{{ elasticsearch_network_host }}"
19+
discovery.seed_hosts: "{{ elasticsearch_seed_hosts }}"
20+
bootstrap.memory_lock: false
21+
cluster.initial_master_nodes: "{{ elasticsearch_cluster_initial_master_nodes }}"
22+
indices.memory.index_buffer_size: "{{ elasticsearch_index_buffer_size | default('40%') }}"
23+
roles:
24+
- common
25+
- java
26+
- { role: ansible-elasticsearch, es_templates: false, es_instance_name: 'ecodata-elasticsearch', es_data_dirs: ['/data/elasticsearch'], tags: ['elasticsearch'], when: install_elasticsearch | bool }

‎ansible/roles/events/handlers/main.yml

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
- name: restart graphql_service
2+
service: name={{ env }}-graphql state=restarted enabled="yes"

‎ansible/roles/events/tasks/main.yaml

+145
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
- include: ../../common/tasks/setfacts.yml
2+
tags:
3+
- docker
4+
- update-images
5+
- events
6+
- service
7+
- docker-config
8+
- events_vhost
9+
10+
- name: Add Docker GPG apt Key
11+
apt_key:
12+
url: https://download.docker.com/linux/ubuntu/gpg
13+
state: present
14+
tags:
15+
- docker
16+
- events
17+
18+
- name: Add Docker Repository
19+
apt_repository:
20+
repo: deb https://download.docker.com/linux/ubuntu bionic stable
21+
state: present
22+
tags:
23+
- docker
24+
- events
25+
26+
- name: Update apt and install docker-ce
27+
apt: update_cache=yes name={{ item }} state=latest
28+
with_items:
29+
- docker-ce
30+
- docker-ce-cli
31+
- docker-compose
32+
- containerd.io
33+
tags:
34+
- docker
35+
- events
36+
37+
- name: Update docker directory
38+
template: src={{ item }} dest=/etc/docker/{{ item }}
39+
with_items:
40+
- daemon.json
41+
tags:
42+
- docker
43+
- events
44+
45+
- name: Restart docker
46+
service:
47+
name: docker
48+
state: restarted
49+
when: use_docker_with_pipelines is defined and use_docker_with_pipelines | bool == True
50+
tags:
51+
- docker
52+
- events
53+
- docker-restart
54+
55+
- name: ensures {{ data_dir }}/{{ env }} dir exists
56+
file:
57+
path: "{{ data_dir }}/{{ env }}"
58+
state: directory
59+
tags:
60+
- docker
61+
- events
62+
- update-images
63+
- docker-config
64+
65+
- name: Copy docker YAML files to {{ data_dir }}
66+
template: src={{ item }} dest={{ data_dir }}/{{ env }}/{{ item }}
67+
with_items:
68+
- graphql-service.yml
69+
- es2vt.env
70+
- es-api.env
71+
- graphql-api.env
72+
notify:
73+
- restart graphql_service
74+
tags:
75+
- docker
76+
- events
77+
- update-images
78+
- docker-config
79+
80+
- name: Copy service scripts to /usr/bin
81+
template: src={{ item }} dest=/usr/bin/{{ env }}-{{ item }} mode=777
82+
with_items:
83+
- graphql-service.sh
84+
tags:
85+
- docker
86+
- events
87+
- service
88+
89+
- name: Copy docker service files to /etc/systemd/system
90+
template: src={{ item }} dest=/etc/systemd/system/{{ env }}-{{ item }}
91+
with_items:
92+
- graphql.service
93+
tags:
94+
- docker
95+
- events
96+
- service
97+
98+
- name: enable services
99+
service: name="{{ env }}-{{ item }}" enabled=yes
100+
with_items:
101+
- graphql.service
102+
tags:
103+
- docker
104+
- events
105+
- service
106+
107+
- name: Start service graphql-service, if not running
108+
service:
109+
name: "{{ env }}-graphql"
110+
state: started
111+
tags:
112+
- docker
113+
- events
114+
- service
115+
116+
- name: add nginx vhost if configured
117+
include_role:
118+
name: nginx_vhost
119+
vars:
120+
appname: "events"
121+
hostname: "{{ events_hostname }}"
122+
context_path: ""
123+
nginx_paths:
124+
- path: "/es/"
125+
sort_label: "1_es"
126+
is_proxy: true
127+
use_cache: true
128+
force_cache: true
129+
proxy_pass: "http://127.0.0.1:{{ es_port }}/"
130+
- path: "/graphql"
131+
sort_label: "1_graphql"
132+
is_proxy: true
133+
use_cache: true
134+
force_cache: true
135+
proxy_pass: "http://127.0.0.1:{{ graphql_port }}"
136+
- path: "/tile/"
137+
sort_label: "1_tile"
138+
is_proxy: true
139+
use_cache: true
140+
force_cache: true
141+
proxy_pass: "http://127.0.0.1:{{ es2vt_port }}/"
142+
tags:
143+
- events
144+
- events_vhost
145+

‎ansible/roles/events/templates/daemon.json

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"data-root": "{{ docker_dir }}",
3+
"live-restore": true
4+
}

‎ansible/roles/events/templates/es-api.env

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
event:
2+
hosts: [{{ es_hosts }}]
3+
requestTimeout: 30000
4+
maxRetries: 3
5+
maxResultWindow: 100000
6+
index: {{ es_index | default('event') }}
7+
eventOccurrence:
8+
hosts: [{{ es_hosts }}]
9+
requestTimeout: 30000
10+
maxRetries: 3
11+
maxResultWindow: 100000
12+
index: {{ es_index | default('event') }}
13+
apiKey: {{ es_api_key }}
14+
port: {{ es_port | default('4001') }}

‎ansible/roles/events/templates/es2vt.env

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
event:
2+
hosts: [{{ es_hosts }}]
3+
index: {{ es_index | default('event') }}
4+
requestTimeout: 60000
5+
maxRetries: 3
6+
maxResultWindow: 100000
7+
port: {{ es2vt_port | default('4002') }}
8+
apiKey: {{ es_api_key }}

‎ansible/roles/events/templates/graphql-api.env

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
port: {{ graphql_port }}
2+
debug: true
3+
NODE_DEBUG: http node index.js
4+
organization: ala
5+
apiEs: http:///es-api:{{ es_port | default('4001') }}
6+
apiEsKey: {{ es_api_key }}
7+
es2vt: http:///es2vt:{{ es2vt_port | default('4002') }}
8+
apiDownloads: "{{ downloads_url }}/dataset/{datasetKey}"
9+
datasetEml: "{{ collectory_url }}/ws/eml/{datasetKey}"
10+
alaNamematching: "{{ namesearch_url }}/api"
11+
ala:
12+
namematching: "{{ namesearch_url }}/api"
13+
bie: {{ bie_service_url }}
14+
biocache: {{ biocache_ws_url }}
15+
images: {{ media_store_url }}

‎ansible/roles/events/templates/graphql-service.sh

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
if [ $1 = "start" ]; then
3+
docker-compose -f /data/{{ env }}/graphql-service.yml up -d
4+
else
5+
docker-compose -f /data/{{ env }}/graphql-service.yml kill
6+
fi

‎ansible/roles/events/templates/graphql-service.yml

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
version: "2.2"
2+
3+
services:
4+
5+
es-api:
6+
container_name: "{{ env }}-es-api"
7+
image: "{{ es_docker_image }}"
8+
ports:
9+
- "{{ es_port | default('4001') }}:{{ es_port | default('4001') }}"
10+
volumes:
11+
- /data/{{ env }}/es-api.env:/usr/src/.env
12+
graphql-api:
13+
container_name: "{{ env }}-graphql-api"
14+
image: "{{ graphql_docker_image }}"
15+
ports:
16+
- "{{ graphql_port | default('4000') }}:{{ graphql_port | default('4000') }}"
17+
volumes:
18+
- /data/{{ env }}/graphql-api.env:/usr/src/.env
19+
es2vt:
20+
container_name: "{{ env }}-es2vt"
21+
image: "{{ es2vt_docker_image }}"
22+
ports:
23+
- "{{ es2vt_port | default('4002') }}:{{ es2vt_port | default('4002') }}"
24+
volumes:
25+
- /data/{{ env }}/es2vt.env:/usr/src/.env

‎ansible/roles/events/templates/graphql.service

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
[Unit]
2+
Description=ALA {{ env }} graphql service
3+
4+
[Service]
5+
Restart=always
6+
ExecStart=/usr/bin/{{ env }}-graphql-service.sh start
7+
ExecStop=/usr/bin/{{ env }}-graphql-service.sh stop
8+
RemainAfterExit=yes
9+
10+
[Install]
11+
WantedBy=multi-user.target

‎ansible/roles/events/vars/main.yml

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
graphql_image: "{{ graphql_image | default('atlasoflivingaustralia/graphql-api:5cf0cb50')}}"
2+
es_image: "{{ es_image | default('atlasoflivingaustralia/es-api:79e116ca')}}"
3+
es2vt_image: "{{ es2vt_image | default('atlasoflivingaustralia/es2vt:8af41c39')}}"

0 commit comments

Comments
 (0)