Skip to content

Commit 592c6df

Browse files
chini007chini007
committedJul 8, 2024
July 2024 Release
1 parent cb25764 commit 592c6df

16 files changed

+420
-172
lines changed
 

‎Release.txt

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,9 @@
88
3. Bug fixes
99
##### January 2024 Release ########
1010
1. External Test Credentials from post environment files
11-
2. Project ID fix: Create a temp flow to export import project to maintain id in all environment.
11+
2. Project ID fix: Create a temp flow to export import project to maintain id in all environment.
12+
##### July 2024 Release ########
13+
1. Support for API
14+
2. Support for Parameters (with environment specific values)
15+
3. Environemnt specific configs filenames aligned from environment config yaml.
16+
4. Bug fixes

‎configs/dev.yml

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎configs/env/dev.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
tenant:
2+
hostname: "psprod.int-aws-de.webmethods.io" # webMethodsIO Integration hostname or ip address
3+
port: "443" # webMethodsIO Integration port
4+
admin_username: "gitops" # webMethodsIO Integration user
5+
type: "dev" #DEV

‎configs/env/play.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
tenant:
2+
hostname: "psdev.int-aws-de.webmethods.io" # webMthodsIO Integration hostname or ip address
3+
port: "443" # webMthodsIO Integration port
4+
admin_username: "gitops" # webMthodsIO Integration user
5+
type: "play" #Dev

‎configs/prod.yml renamed to ‎configs/env/prod.yml

File renamed without changes.

‎configs/qa.yml renamed to ‎configs/env/qa.yml

File renamed without changes.

‎configs/play.yml

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎pipelines/InitiateTesting.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ steps:
5050
cd ../self
5151
pwd
5252
ls -ltr
53-
echo "##vso[task.setvariable variable=admin_user]`yq -e ".tenant.admin_username" configs/${source}.yml`"
53+
echo "##vso[task.setvariable variable=admin_user]`yq -e ".tenant.admin_username" configs/env/${source}.yml`"
5454
5555
echo $(System.DefaultWorkingDirectory)
5656
workingDirectory: '$(System.DefaultWorkingDirectory)'

‎pipelines/initialize_pipeline.yml

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -51,23 +51,23 @@ steps:
5151
cd ../self
5252
pwd
5353
ls -ltr
54-
echo "##vso[task.setvariable variable=source_environment_hostname]`yq -e ".tenant.hostname" configs/play.yml`"
55-
echo "##vso[task.setvariable variable=source_environment_port]`yq -e ".tenant.port" configs/play.yml`"
56-
echo "##vso[task.setvariable variable=admin_user]`yq -e ".tenant.admin_username" configs/play.yml`"
57-
echo "##vso[task.setvariable variable=source_type]`yq -e ".tenant.type" configs/play.yml`"
54+
echo "##vso[task.setvariable variable=source_environment_hostname]`yq -e ".tenant.hostname" configs/env/play.yml`"
55+
echo "##vso[task.setvariable variable=source_environment_port]`yq -e ".tenant.port" configs/env/play.yml`"
56+
echo "##vso[task.setvariable variable=admin_user]`yq -e ".tenant.admin_username" configs/env/play.yml`"
57+
echo "##vso[task.setvariable variable=source_type]`yq -e ".tenant.type" configs/env/play.yml`"
5858
echo "##vso[task.setvariable variable=repo_user]`yq -e ".repo.user" configs/repo.yml`"
5959
60-
echo "##vso[task.setvariable variable=dev_environment_hostname]`yq -e ".tenant.hostname" configs/dev.yml`"
61-
echo "##vso[task.setvariable variable=dev_environment_port]`yq -e ".tenant.port" configs/dev.yml`"
62-
echo "##vso[task.setvariable variable=dev_admin_user]`yq -e ".tenant.admin_username" configs/dev.yml`"
60+
echo "##vso[task.setvariable variable=dev_environment_hostname]`yq -e ".tenant.hostname" configs/env/dev.yml`"
61+
echo "##vso[task.setvariable variable=dev_environment_port]`yq -e ".tenant.port" configs/env/dev.yml`"
62+
echo "##vso[task.setvariable variable=dev_admin_user]`yq -e ".tenant.admin_username" configs/env/dev.yml`"
6363
64-
echo "##vso[task.setvariable variable=qa_environment_hostname]`yq -e ".tenant.hostname" configs/qa.yml`"
65-
echo "##vso[task.setvariable variable=qa_environment_port]`yq -e ".tenant.port" configs/qa.yml`"
66-
echo "##vso[task.setvariable variable=qa_admin_user]`yq -e ".tenant.admin_username" configs/qa.yml`"
64+
echo "##vso[task.setvariable variable=qa_environment_hostname]`yq -e ".tenant.hostname" configs/env/qa.yml`"
65+
echo "##vso[task.setvariable variable=qa_environment_port]`yq -e ".tenant.port" configs/env/qa.yml`"
66+
echo "##vso[task.setvariable variable=qa_admin_user]`yq -e ".tenant.admin_username" configs/env/qa.yml`"
6767
68-
echo "##vso[task.setvariable variable=prod_environment_hostname]`yq -e ".tenant.hostname" configs/prod.yml`"
69-
echo "##vso[task.setvariable variable=prod_environment_port]`yq -e ".tenant.port" configs/prod.yml`"
70-
echo "##vso[task.setvariable variable=prod_admin_user]`yq -e ".tenant.admin_username" configs/prod.yml`"
68+
echo "##vso[task.setvariable variable=prod_environment_hostname]`yq -e ".tenant.hostname" configs/env/prod.yml`"
69+
echo "##vso[task.setvariable variable=prod_environment_port]`yq -e ".tenant.port" configs/env/prod.yml`"
70+
echo "##vso[task.setvariable variable=prod_admin_user]`yq -e ".tenant.admin_username" configs/env/prod.yml`"
7171
7272
echo "##vso[build.updatebuildnumber]$(Build.BuildNumber)-$(repoName)"
7373
echo $(System.DefaultWorkingDirectory)

‎pipelines/scripts/exportAsset.sh

Lines changed: 124 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ HOME_DIR=$7
1515
synchProject=$8
1616
source_type=$9
1717
inlcudeAllReferenceData=${10}
18+
envTypes=${11}
1819
debug=${@: -1}
1920

2021
if [ -z "$LOCAL_DEV_URL" ]; then
@@ -56,7 +57,14 @@ debug=${@: -1}
5657
echo "Missing template parameter source_type"
5758
exit 1
5859
fi
59-
60+
if [ -z "$inlcudeAllReferenceData" ]; then
61+
echo "Missing template parameter inlcudeAllReferenceData"
62+
exit 1
63+
fi
64+
if [ -z "$envTypes" ]; then
65+
echo "Missing template parameter envTypes"
66+
exit 1
67+
fi
6068
if [ "$debug" == "debug" ]; then
6169
echo "******* Running in Debug mode ******"
6270
fi
@@ -113,9 +121,7 @@ function exportSingleReferenceData () {
113121
metadataJson=$(echo "$metadataJson"| jq 'del(.columnNames, .dataRecords, .revisionData)')
114122
echo "$metadataJson" > metadata.json
115123
echo "$datajson" > ${source_type}.csv
116-
cp -n ./${source_type}.csv dev.csv
117-
cp -n ./${source_type}.csv qa.csv
118-
cp -n ./${source_type}.csv prod.csv
124+
configPerEnv . ${envTypes} "referenceData" ${source_type}.csv
119125
cd -
120126
fi
121127
cd ${HOME_DIR}/${repoName}
@@ -173,6 +179,28 @@ function exportReferenceData (){
173179
cd ${HOME_DIR}/${repoName}
174180
}
175181

182+
function configPerEnv(){
183+
localtion=$1
184+
envTypes=$2
185+
configType=$3
186+
sourceFile=$4
187+
key=$5
188+
189+
IFS=, read -ra values <<< "$envTypes"
190+
for v in "${values[@]}"
191+
do
192+
# things with "$v"
193+
if [ ${configType} == "referenceData" ]; then
194+
cp ./$sourceFile ./$v.csv
195+
else
196+
if [ ${configType} == "project_parameter" ]; then
197+
cp ./$sourceFile ./${key}_${v}.json
198+
fi
199+
fi
200+
done
201+
202+
}
203+
176204
function exportAsset(){
177205

178206
LOCAL_DEV_URL=$1
@@ -216,22 +244,28 @@ function exportAsset(){
216244
echod "Rest_API Export:" ${EXPORT_URL} "with JSON: "${rest_api_json}
217245
echod $(ls -ltr)
218246
else
219-
if [[ $assetType = workflow* ]]; then
247+
if [[ $assetType = project_parameter* ]]; then
220248
echod $assetType
221-
EXPORT_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/workflows/${assetID}/export
222-
cd ${HOME_DIR}/${repoName}
223-
mkdir -p ./assets/workflows
224-
cd ./assets/workflows
225-
echod "Workflow Export:" ${EXPORT_URL}
226-
echod $(ls -ltr)
249+
exportProjectParameters ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${assetID} ${assetType} ${HOME_DIR} ${synchProject} ${inlcudeAllReferenceData}
250+
return
227251
else
228-
if [[ $assetType = flowservice* ]]; then
229-
EXPORT_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/flows/${assetID}/export
252+
if [[ $assetType = workflow* ]]; then
253+
echod $assetType
254+
EXPORT_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/workflows/${assetID}/export
230255
cd ${HOME_DIR}/${repoName}
231-
mkdir -p ./assets/flowservices
232-
cd ./assets/flowservices
233-
echo "Flowservice Export:" ${EXPORT_URL}
256+
mkdir -p ./assets/workflows
257+
cd ./assets/workflows
258+
echod "Workflow Export:" ${EXPORT_URL}
234259
echod $(ls -ltr)
260+
else
261+
if [[ $assetType = flowservice* ]]; then
262+
EXPORT_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/flows/${assetID}/export
263+
cd ${HOME_DIR}/${repoName}
264+
mkdir -p ./assets/flowservices
265+
cd ./assets/flowservices
266+
echo "Flowservice Export:" ${EXPORT_URL}
267+
echod $(ls -ltr)
268+
fi
235269
fi
236270
fi
237271
fi
@@ -246,7 +280,6 @@ function exportAsset(){
246280
--header 'Accept: application/json' \
247281
-u ${admin_user}:${admin_password})
248282
fi
249-
250283
downloadURL=$(echo "$linkJson" | jq -r '.output.download_link')
251284

252285
regex='(https?|ftp|file)://[-[:alnum:]\+&@#/%?=~_|!:,.;]*[-[:alnum:]\+&@#/%=~_|]'
@@ -278,6 +311,74 @@ function exportAsset(){
278311

279312
}
280313

314+
function exportProjectParameters(){
315+
316+
LOCAL_DEV_URL=$1
317+
admin_user=$2
318+
admin_password=$3
319+
repoName=$4
320+
assetID=$5
321+
assetType=$6
322+
HOME_DIR=$7
323+
synchProject=$8
324+
inlcudeAllReferenceData=$9
325+
326+
if [ ${synchProject} == true ]; then
327+
PROJECT_PARAM_GET_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/params
328+
else
329+
PROJECT_PARAM_GET_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/params/${assetID}
330+
fi
331+
332+
ppListJson=$(curl --location --request GET ${PROJECT_PARAM_GET_URL} \
333+
--header 'Content-Type: application/json' \
334+
--header 'Accept: application/json' \
335+
-u ${admin_user}:${admin_password})
336+
337+
ppListExport=$(echo "$ppListJson" | jq '. // empty')
338+
339+
if [ -z "$ppListExport" ]; then
340+
echo "No Project Parameters retreived:" ${ppListJson}
341+
else
342+
mkdir -p ./assets/projectConfigs/parameters
343+
cd ./assets/projectConfigs/parameters
344+
if [ ${synchProject} != true ]; then
345+
parameterUID=$(jq -r '.output.uid' <<< "$ppListJson")
346+
mkdir -p ./${parameterUID}
347+
cd ./${parameterUID}
348+
data=$(jq -r '.output.param' <<< "$ppListJson")
349+
key=$(jq -r '.output.param.key' <<< "$ppListJson")
350+
metadataJson='{ "uid":"'${parameterUID}'" }'
351+
echo ${metadataJson} > ./metadata.json
352+
echo ${data} > ./${key}_${source_type}.json
353+
configPerEnv . ${envTypes} "project_parameter" ${key}_${source_type}.json ${key}
354+
#cp -n ./${key}_${source_type}.json ${key}_dev.json
355+
#cp -n ./${key}_${source_type}.json ${key}_qa.json
356+
#cp -n ./${key}_${source_type}.json ${key}_prod.json
357+
cd ..
358+
else
359+
for item in $(jq -c -r '.output[]' <<< "$ppListJson"); do
360+
echod "Inside Parameters Loop"
361+
parameterUID=$(jq -r '.uid' <<< "$item")
362+
mkdir -p ./${parameterUID}
363+
cd ./${parameterUID}
364+
data=$(jq -r '.param' <<< "$item")
365+
key=$(jq -r '.param.key' <<< "$item")
366+
metadataJson='{ "uid":"'${parameterUID}'" }'
367+
echo ${metadataJson} > ./metadata.json
368+
echo ${data} > ./${key}_${source_type}.json
369+
configPerEnv . ${envTypes} "project_parameter" ${key}_${source_type}.json ${key}
370+
#cp -n ./${key}_${source_type}.json ${key}_dev.json
371+
#cp -n ./${key}_${source_type}.json ${key}_qa.json
372+
#cp -n ./${key}_${source_type}.json ${key}_prod.json
373+
cd ..
374+
done
375+
fi
376+
echo "Project Parameters export Succeeded"
377+
fi
378+
cd ${HOME_DIR}/${repoName}
379+
}
380+
381+
281382
if [ ${synchProject} == true ]; then
282383
echod "Listing All Assets"
283384
echod $assetType
@@ -288,7 +389,7 @@ if [ ${synchProject} == true ]; then
288389
--header 'Accept: application/json' \
289390
-u ${admin_user}:${admin_password})
290391

291-
# Exporing APIs
392+
# Exporting APIs
292393
for item in $(jq -c -r '.output.rest_api[]' <<< "$projectListJson"); do
293394
echod "Inside REST_API Loop"
294395
assetID=$item
@@ -297,7 +398,7 @@ if [ ${synchProject} == true ]; then
297398
exportAsset ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${assetID} ${assetType} ${HOME_DIR} ${synchProject} ${inlcudeAllReferenceData}
298399
done
299400

300-
# Exporing Workflows
401+
# Exporting Workflows
301402
for item in $(jq -c -r '.output.workflows[]' <<< "$projectListJson"); do
302403
echod "Inside Workflow Loop"
303404
assetID=$item
@@ -339,34 +440,11 @@ if [ ${synchProject} == true ]; then
339440
# Exporting Project Referencedata
340441

341442
exportReferenceData ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${assetID} ${assetType} ${HOME_DIR}
342-
343-
344443
# Exporting Project Parameters
345-
: ' PP Export
346-
PROJECT_PARAM_GET_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/params
347-
348-
ppListJson=$(curl --location --request GET ${PROJECT_PARAM_GET_URL} \
349-
--header 'Content-Type: application/json' \
350-
--header 'Accept: application/json' \
351-
-u ${admin_user}:${admin_password})
352-
353-
ppListExport=$(echo "$ppListJson" | jq '. // empty')
354-
355-
if [ -z "$ppListExport" ]; then
356-
echo "No Project Parameters retreived:" ${ppListJson}
357-
else
358-
mkdir -p ./assets/projectConfigs/parameters
359-
cd ./assets/projectConfigs/parameters
360-
for item in $(jq -c -r '.output[]' <<< "$ppListJson"); do
361-
echod "Inside Parameters Loop"
362-
parameterUID=$(jq -r '.uid' <<< "$item")
363-
data=$(jq -r '.param' <<< "$item")
364-
echo ${data} > ./${parameterUID}.json
365-
done
366-
echo "Project Parameters export Succeeded"
367-
fi
368-
cd ${HOME_DIR}/${repoName}
369-
'
444+
#PP Export
445+
assetType=project_parameter
446+
exportAsset ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${assetID} ${assetType} ${HOME_DIR} ${synchProject} ${inlcudeAllReferenceData}
447+
370448
else
371449
exportAsset ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${assetID} ${assetType} ${HOME_DIR} ${synchProject} ${inlcudeAllReferenceData}
372450
fi

0 commit comments

Comments
 (0)