Merge pull request #593 from boozallen/591-improve-arch-test

#591 improve archetype test

Author: ewilkins-csi

foundation/foundation-archetype/extensions-to-add-to-archetype_test-generator-deploy_pom.xml

@@ -0,0 +1,35 @@
+            <plugin>
+                <groupId>com.boozallen.aissemble</groupId>
+                <artifactId>mda-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-spark-application</id>
+                        <phase>prepare-package</phase>
+                        <goals>
+                            <goal>copy-pipeline-artifacts</goal>
+                        </goals>
+                        <configuration>
+                            <targetPipelineTypes>
+                                <targetPipelineType>data-flow</targetPipelineType>
+                            </targetPipelineTypes>
+                            <targetArtifactTypes>
+                                <targetArtifactType>VALUES_FILES</targetArtifactType>
+                            </targetArtifactTypes>
+                            <outputDirectory>${project.basedir}/src/main/resources/apps/pipeline-invocation-service/target/</outputDirectory>
+                        </configuration>
+                    </execution>
+                </executions>
+                <configuration>
+                    <groupId>${project.groupId}</groupId>
+                    <modelsArtifactId>${project.parent.artifactId}-pipeline-models</modelsArtifactId>
+                    <pipelinesDirectory>${project.parent.basedir}/${project.parent.artifactId}-pipelines/</pipelinesDirectory>
+                    <habushuArtifactVersion>${version.habushu.dist.artifact}</habushuArtifactVersion>
+                </configuration>
+                <dependencies>
+                    <dependency>
+                        <groupId>${project.groupId}</groupId>
+                        <artifactId>${project.parent.artifactId}-pipeline-models</artifactId>
+                        <version>${project.version}</version>
+                    </dependency>
+                </dependencies>
+            </plugin>

foundation/foundation-archetype/plugins-to-add-to-archetype_test-generator-deploy_pom.xml

@@ -10,6 +10,7 @@
 # #L%
 ###
 
+set -o pipefail
 echo -e "\n\n **** TESTING ARCHETYPE GENERATION **** \n\n"
 
 if [ $# -ne 1 ]; then
@@ -21,27 +22,73 @@ fi
 mkdir -p target/temp
 cd target/temp
 
+#---
+## Replacement for `sed -i` that formats the command correctly for MacOS and Linux
+##
+## @args: $1 and $2 are passed in unchanged to `sed`
+#---
+function sub {
+  if [[ $OSTYPE == "darwin"* ]]; then
+    sed -i '' "$1" "$2" || exit 1
+  else
+    sed -i "$1" "$2" || exit 1
+  fi
+}
+
+#---
+## Escapes the contents of a stdin for use with a `sed` substitution command. Namely ensures dollar signs, new lines
+## and backslashes are escaped.
+##
+## @stdin: the content to escape
+## @stdout: the input with '$', '\', and '\n' escaped
+#---
+function esc {
+  # sed is used for a straightforward substitution of \ and $
+  # awk is used to escape newlines. Essentially, replaces the newlines between lines of input with a backslash and newline.
+  # Hence ORS (output record sep) is `\\\n`. However, because trailing newlines are always chomped in pipes, the last
+  # newline in the input must not be escaped. Therfore, input is buffered by one line in the `last` var so we can turn set
+  # the ORS to blank for the final line.
+  sed 's/\([\/\$]\)/\\\1/g' | awk 'NR > 1 {ORS="\\\n"} END {ORS=""; print last} {print last} {last=$0}' ORS=''
+}
+
 function updatePomBasedOnChildDirs {
   # The 'TODO' text line that we're going to replace in the file
     # should be passed in as the first param to this method. i.e.
     #    updatePomBasedOnChildDirs '<!-- TODO: this is where manual actions go -->'
   todoText=$1
-
-    modules=""
+  modules=""
   for d in */ ; do
     if [ $d != "target/" ]; then
       # Drop / from the end of the directory
       d="${d/\//}"
-      modules+="<module>$d</module>"
+      modules+="<module>$d</module>"$'\n        '
     fi
   done
 
   echo -e "\nINFO: Adding \n\t$modules\n\tto $PWD/pom.xml\n"
-  fileContents=`cat pom.xml`
-  fileContents="${fileContents/$todoText/$modules}"
+  # command substitution trims trailing newlines so add it back
+  sub "s/$todoText/$(esc <<< "$modules")/" pom.xml
+}
 
-  # NOTE - the formatting get's all messed up
-  echo -e $fileContents > pom.xml
+#---
+## Runs maven build and captures any manual actions for adding Fermenter executions to the deploy POM, then updates the
+## deploy POM based on the detected manual actions.
+#---
+function runBuildAndUpdateDeploy {
+  deployInsert="<!-- Add executions for each deployment module -->"
+  outputstart='executions to test-generator-deploy'
+  outputend='\[WARNING\]'
+  # $outputstart match at end ensures the match line isn't captured. NF ensures blank lines aren't captured.
+  execs=$(./mvnw clean install | tee /dev/tty | \
+            awk "BEGIN {output=0} /$outputend/ {output=0} NF && output {print} /$outputstart/ {output=1}")\
+      || { echo -e '\n\n\t**** MAVEN BUILD FAILED ****\n\n' ; exit 1; }
+  if [ -n "$execs" ]; then
+    #re-adding insert comment allows for subsequent insertions
+    execs=$deployInsert$'\n'$execs
+    execs=$(esc <<< "$execs")
+    echo -e "\n INFO: Adding execution profiles to deploy POM: \n$execs"
+    sub "s/$deployInsert/$execs/" test-generator-deploy/pom.xml
+  fi
 }
 
 package='com.bah.aiops'
@@ -61,7 +108,7 @@ mvn archetype:generate -B \
   -DmavenRepositoryUrl=https://repo1.maven.org/maven2 \
   -DmavenSnapshotRepositoryUrl=https://s01.oss.sonatype.org/content/repositories/snapshots
 
-cd test-generator/
+cd test-generator/ || exit
 
 # Add pipelines files necessary for testing.
 
@@ -201,49 +248,41 @@ echo "{
 echo -e "\n# maven-suppress-warnings" >> Tiltfile
 
 echo -e "\nINFO: Running full build to generate project structure\n"
-./mvnw clean install || { echo -e '\n\n\t**** MAVEN BUILD FAILED ****\n\n' ; exit 1; }
+runBuildAndUpdateDeploy
 
-
-cd test-generator-shared/
+cd test-generator-shared/ || exit
 updatePomBasedOnChildDirs "<!-- TODO: replace with your project-specific modules here -->"
 
-cd ../test-generator-docker/
+cd ../test-generator-docker/ || exit
 updatePomBasedOnChildDirs "<!-- TODO: Add docker modules here -->"
 
-cd ../test-generator-pipelines/
+cd ../test-generator-pipelines/ || exit
 updatePomBasedOnChildDirs "<!-- TODO: replace with your pipeline-specific modules here -->"
 
 cd ..
 
 echo -e "\nINFO: Generating project structure for pipelines and data records\n"
-./mvnw clean install || { echo -e '\n\n\t**** MAVEN BUILD FAILED ****\n\n' ; exit 1; }
+runBuildAndUpdateDeploy
 
-cd test-generator-pipelines/example-machine-learning-pipeline/
+cd test-generator-pipelines/example-machine-learning-pipeline/ || exit
 updatePomBasedOnChildDirs "<!-- TODO: replace with your step-specific modules here -->"
 
 cd ../..
 
 echo -e "\nINFO: Generating project structure for ml pipelines\n"
-./mvnw clean install || { echo -e '\n\n\t**** MAVEN BUILD FAILED ****\n\n' ; exit 1; }
-
-
-echo -e "\nINFO: updating the test-generator-deploy/pom.xml based on static code as it's hard to predict without folder structure\n"
-executions=`cat ../../../extensions-to-add-to-archetype_test-generator-deploy_pom.xml`
-
-fileContents=`cat test-generator-deploy/pom.xml`
-todoText="<!-- Add executions for each deployment module -->"
-fileContents="${fileContents/$todoText/$executions}"
-echo -e $fileContents > test-generator-deploy/pom.xml
-
+runBuildAndUpdateDeploy
 
+echo -e "\nINFO: updating the test-generator-deploy/pom.xml based on static code as it's hard to predict\n"
+plugins=$(esc < ../../../plugins-to-add-to-archetype_test-generator-deploy_pom.xml)
+sub "s/ *<\/plugins>/$plugins        <\/plugins>/" test-generator-deploy/pom.xml
 
 echo -e "\nINFO: Running full build to check that the build passes w/o any manual actions needed\n"
-./mvnw clean install || { echo -e '\n\n\t**** MAVEN BUILD FAILED ****\n\n' ; exit 1; } > maven-build.log
-buildLog=`cat maven-build.log`
-if [[ $buildLog == *"MANUAL ACTION NEEDED"* ]]; then
+# NOTE: because fermenter results are cached, the build-cache will hide remaining manual actions that were missed in previous steps
+./mvnw clean install -Dmaven.build.cache.skipCache -Dfermenter.display.message.keys=true | tee /dev/tty | awk '/WARNING/ {print}' > maven-build.log \
+    || { echo -e '\n\n\t**** MAVEN BUILD FAILED ****\n\n' ; exit 1; }
+if grep -iq 'Manual action' maven-build.log; then
   echo -e "\n\n **** ERROR: Manual action still found in build **** \n    Look at **archetype/target/temp/test-generator/maven-build.log** to see what the problem was. \n\n"
   exit 1
 else
   echo -e "\n\n **** SUCCESS: Completed the full build successfully ****\n\n"
 fi
-

foundation/foundation-archetype/test-project-archetype.sh

@@ -14,6 +14,14 @@ mp.messaging.outgoing.metadata-ingest.topic=metadata-ingest
 mp.messaging.outgoing.metadata-ingest.value.serializer=com.boozallen.aissemble.core.metadata.producer.MetadataSerializer
 #end
 
+#if (${pipeline.isAlertingSupportNeeded()})
+$header Alert Producer configs $header
+mp.messaging.outgoing.alerts.connector=smallrye-kafka
+mp.messaging.outgoing.alerts.topic=alerts
+##TODO: StringSerializer actually throws an exception. We need an AlertSerializer in alerting-core
+mp.messaging.outgoing.alerts.value.serializer=org.apache.kafka.common.serialization.StringSerializer
+#end
+
 #if (${pipeline.getDataLineage()})
 $header Data Lineage Emitter configs $header
 $comment By default, uses in-memory transport and logs events to console.