#95 Added Spark Infrastructure V2 integrations

Signed-off-by: Peter McClonski <mcclonski_peter@bah.com>

Author: peter-mcclonski

extensions/extensions-helm/aissemble-spark-application-chart/values-migrate.yaml

@@ -0,0 +1,57 @@
+########################################
+## CONFIG | Spark Configs
+########################################
+metadata:
+  namespace: default
+sparkApp:
+  spec:
+    sparkConf:
+      spark.eventLog.enabled: "true"
+      spark.eventLog.dir: "/opt/spark/spark-events"
+    type: "placeholder" #required for a dry run test to pass, this should always be overridden
+    mode: cluster
+    imagePullPolicy: IfNotPresent
+    restartPolicy:
+      type: Never
+    sparkVersion: "3.4.0"
+    sparkConfigMap: spark-config
+    dynamicAllocation:
+      enabled: true
+      initialExecutors: 0
+      minExecutors: 0
+      maxExecutors: 4
+    volumes:
+      - name: ivy-cache
+        persistentVolumeClaim:
+          claimName: ivy-cache
+      - name: spark-events
+        persistentVolumeClaim:
+          claimName: spark-events-claim
+    driver:
+      cores: 1
+      coreLimit: "1200m"
+      memory: "512m"
+      serviceAccount: spark
+      volumeMounts:
+        - name: ivy-cache
+          mountPath: "/opt/spark/.ivy2"
+        - name: spark-events
+          mountPath: "/opt/spark/spark-events"
+    executor:
+      cores: 1
+      coreLimit: "1200m"
+      memory: "512m"
+      labels:
+        version: 3.4.0
+      volumeMounts:
+        - name: ivy-cache
+          mountPath: "/opt/spark/.ivy2"
+        - name: spark-events
+          mountPath: "/opt/spark/spark-events"
+service:
+  enabled: false
+  spec:
+    ports:
+      - name: "debug"
+        port: 4747
+        targetPort: 4747

extensions/extensions-helm/aissemble-spark-history-chart/.helmignore extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/.helmignore

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>com.boozallen.aissemble</groupId>
-        <artifactId>extensions-helm</artifactId>
+        <artifactId>extensions-helm-spark-infrastructure</artifactId>
         <version>1.7.0-SNAPSHOT</version>
     </parent>
 

extensions/extensions-helm/aissemble-spark-history-chart/Chart.yaml extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/Chart.yaml

@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    
+    <parent>
+        <groupId>com.boozallen.aissemble</groupId>
+        <artifactId>extensions-helm</artifactId>
+        <version>1.7.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>extensions-helm-spark-infrastructure</artifactId>
+    <name>aiSSEMBLE::Extensions::Helm::Spark Infrastructure</name>
+
+    <packaging>pom</packaging>
+
+	<modules>
+        <module>aissemble-spark-history-chart</module>
+	</modules>
+    <build>
+        <plugins>
+<!--            Disable unit testing on aggregator modules -->
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <version>3.1.0</version>
+                <executions>
+                    <execution>
+                        <id>run tests</id>
+                        <inherited>false</inherited>
+                        <phase>none</phase>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

extensions/extensions-helm/aissemble-spark-history-chart/LICENSE.txt extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/LICENSE.txt

@@ -21,7 +21,6 @@
         <module>aissemble-inference-chart</module>
         <module>aissemble-versioning-chart</module>
         <module>aissemble-kafka-chart</module>
-        <module>aissemble-spark-history-chart</module>
         <module>extensions-helm-hive</module>
         <module>aissemble-keycloak-chart</module>
         <module>aissemble-elasticsearch-operator-chart</module>
@@ -31,6 +30,7 @@
         <module>aissemble-lineage-http-consumer-chart</module>
         <module>aissemble-jenkins-chart</module>
         <module>extensions-helm-pipeline-invocation</module>
+        <module>extensions-helm-spark-infrastructure</module>
         <module>aissemble-policy-decision-point-chart</module>
         <module>aissemble-mlflow-chart</module>
         <module>aissemble-localstack-chart</module>

extensions/extensions-helm/aissemble-spark-history-chart/README.md extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/README.md

@@ -174,6 +174,9 @@ public class KubernetesGenerator extends AbstractKubernetesGenerator {
      * | sharedInfrastructureValuesDevFile             | deployment/shared-infrastructure/shared-infrastructure.values-dev.yaml.vm                 | apps/${appName}/values-dev.yaml                                            |
      * | sharedInfrastructureValuesFile                | deployment/shared-infrastructure/shared-infrastructure.values.yaml.vm                     | apps/${appName}/values.yaml                                                |
      * | sparkInfrastructureConfigMapFile              | deployment/spark-infrastructure/templates/configmap.yaml.vm                               | apps/${appName}/templates/configmap.yaml                                   |
+     * | sparkInfrastructureHelmChartFileV2            | deployment/spark-infrastructure/v2/spark.infrastructure.chart.yaml.vm                     | apps/${appName}/Chart.yaml                                                 |
+     * | sparkInfrastructureHelmValuesDevFileV2        | deployment/spark-infrastructure/v2/spark.infrastructure.values.dev.yaml.vm                | apps/${appName}/values-dev.yaml                                            |
+     * | sparkInfrastructureHelmValuesFileV2           | deployment/spark-infrastructure/v2/spark.infrastructure.values.yaml.vm                    | apps/${appName}/values.yaml                                                |
      * | sparkOperatorScheduledApplicationsCRD         | deployment/spark-operator/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml.vm    | apps/${appName}/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml  |
      * | sparkOperatorApplicationsCRD                  | deployment/spark-operator/crds/sparkoperator.k8s.io_sparkapplications.yaml.vm             | apps/${appName}/crds/sparkoperator.k8s.io_sparkapplications.yaml           |
      * | sparkOperatorValuesDevFile                    | deployment/spark-operator/spark-operator.values-dev.yaml.vm                               | apps/${appName}/values-dev.yaml                                            |

extensions/extensions-helm/aissemble-spark-history-chart/examples/README.md extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/examples/README.md

@@ -292,6 +292,28 @@
 			}
 		]
 	},
+	{
+		"name": "aissemble-spark-infrastructure-deploy-v2",
+		"profileReferences": [
+			{
+				"name": "spark-infrastructure-kubernetes-v2"
+			}
+		]
+	},
+	{
+		"name": "spark-infrastructure-kubernetes-v2",
+		"targetReferences": [
+			{
+				"name": "sparkInfrastructureHelmChartFileV2"
+			},
+			{
+				"name": "sparkInfrastructureHelmValuesFileV2"
+			},
+			{
+				"name": "sparkInfrastructureHelmValuesDevFileV2"
+			}
+		]
+	},
 	{
 		"name": "aissemble-spark-operator-deploy",
 		"profileReferences": [

extensions/extensions-helm/aissemble-spark-history-chart/examples/values-cloud.yaml extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/examples/values-cloud.yaml

@@ -3166,5 +3166,29 @@
     "generator": "com.boozallen.aiops.mda.generator.KubernetesGenerator",
     "metadataContext": "targeted",
     "overwritable": false
+  },
+  {
+    "name": "sparkInfrastructureHelmChartFileV2",
+    "templateName": "templates/deployment/spark-infrastructure/v2/spark.infrastructure.chart.yaml.vm",
+    "outputFile": "apps/${appName}/Chart.yaml",
+    "generator": "com.boozallen.aiops.mda.generator.KubernetesGenerator",
+    "metadataContext": "targeted",
+    "overwritable": false
+  },
+  {
+    "name": "sparkInfrastructureHelmValuesFileV2",
+    "templateName": "templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.yaml.vm",
+    "outputFile": "apps/${appName}/values.yaml",
+    "generator": "com.boozallen.aiops.mda.generator.KubernetesGenerator",
+    "metadataContext": "targeted",
+    "overwritable": false
+  },
+  {
+    "name": "sparkInfrastructureHelmValuesDevFileV2",
+    "templateName": "templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.dev.yaml.vm",
+    "outputFile": "apps/${appName}/values-dev.yaml",
+    "generator": "com.boozallen.aiops.mda.generator.KubernetesGenerator",
+    "metadataContext": "targeted",
+    "overwritable": false
   }
 ]

extensions/extensions-helm/aissemble-spark-history-chart/examples/values-local.yaml extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/examples/values-local.yaml

@@ -0,0 +1,35 @@
+apiVersion: v2
+name: ${appName}
+description: A Helm chart for ${appName}
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: ${version}
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.0.0"
+
+# To include an aiSSEMBLE chart with ArgoCD we must add it as a dependency until including from multiple sources
+# is moved out of beta and into the latest version of Argo.
+# https://argo-cd.readthedocs.io/en/stable/user-guide/multiple_sources/
+# Note that this requires prefixing values with the name of the dependency (see test/src/test/resources in the aiSSEMBLE repo
+# for example usage) in order to override values.
+dependencies:
+  - name: aissemble-spark-history-chart
+    version: ${versionTag}
+    repository: oci://ghcr.io/boozallen
+

extensions/extensions-helm/aissemble-spark-history-chart/examples/values-s3.yaml extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/examples/values-s3.yaml

@@ -0,0 +1,8 @@
+# This file contains the default values for the aiSSEMBLE Spark History Helm chart.
+
+aissemble-spark-history-chart:
+    service:
+      type: LoadBalancer
+
+    ingress:
+      enabled: false

extensions/extensions-helm/aissemble-spark-history-chart/pom.xml extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/pom.xml

@@ -0,0 +1,5 @@
+# This file contains the default values for the aiSSEMBLE Spark History Helm chart.
+
+aissemble-spark-history-chart:
+    eventVolume:
+      enabled: true