[#538] Encryption of relation records (#567)

aaron-gary · web-flow · commit bff8bfdfdeb3 · 2025-02-05T14:00:21.000-08:00
Signed-off-by: Aaron Gary &lt;agary@cpointe-inc.com&gt;
diff --git a/foundation/foundation-mda/pom.xml b/foundation/foundation-mda/pom.xml
@@ -170,6 +170,12 @@
             <groupId>net.masterthought</groupId>
             <artifactId>cucumber-reporting</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.boozallen.aissemble</groupId>
+            <artifactId>foundation-encryption-policy-java</artifactId>
+            <version>${version.aissemble}</version>
+        </dependency>
+
     </dependencies>
 
 </project>
diff --git a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/BaseRecordDecorator.java b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/BaseRecordDecorator.java
@@ -11,6 +11,7 @@
  */
 
 import com.boozallen.aiops.mda.generator.util.PipelineUtils;
+import com.boozallen.aiops.mda.metamodel.AissembleModelInstanceRepository;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.commons.lang3.StringUtils;
@@ -106,6 +107,14 @@ public List<RecordField> getFields() {
         return wrapped.getFields();
     }
 
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public List<String> getFieldIds(AissembleModelInstanceRepository metadataRepo) {
+        return wrapped.getFieldIds(metadataRepo);
+    }
+
     /**
      * {@inheritDoc}
      */
diff --git a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/BaseStepDecorator.java b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/BaseStepDecorator.java
@@ -440,4 +440,21 @@ public List<FileStore> getDecoratedFileStores() {
 	public boolean isGeneric() {
 		return "generic".equalsIgnoreCase(getType());
 	}
+
+	/**
+	 * Whether or not the step has an inbound native collection type defined
+	 * and also contains relations.
+	 *
+	 * @return true if the step has an inbound native collection type defined and has relations.
+	 */
+	public boolean hasInboundNativeCollectionTypeAndRelations() {
+		boolean isNative = getInbound() != null && getInbound().getNativeCollectionType() != null;
+		boolean hasRelations = false;
+		if(getInbound() != null && getInbound().getRecordType() != null) {
+			Record thisRecord = getInbound().getRecordType().getRecordType();
+			hasRelations = thisRecord.getRelations() != null && !thisRecord.getRelations().isEmpty();
+		}
+
+		return isNative && hasRelations;
+	}
 }
diff --git a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/Record.java b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/Record.java
@@ -10,6 +10,7 @@
  * #L%
  */
 
+import com.boozallen.aiops.mda.metamodel.AissembleModelInstanceRepository;
 import org.technologybrewery.fermenter.mda.metamodel.element.NamespacedMetamodel;
 
 import java.util.Collection;
@@ -49,6 +50,16 @@ public interface Record extends NamespacedMetamodel {
      */
     List<RecordField> getFields();
 
+
+    /**
+     * Returns the fields ids contained in this field container, including field ids from related records.
+     *
+     * @param metadataRepo a configured MetadataRepo
+     * @return a list of field ids
+     */
+    List<String> getFieldIds(AissembleModelInstanceRepository metadataRepo);
+
+
     /**
      * Returns the list of supported Frameworks (e.g. PySpark).
      * @return list of supported Frameworks.
diff --git a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/RecordElement.java b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/RecordElement.java
@@ -11,6 +11,7 @@
  */
 
 import com.boozallen.aiops.mda.ManualActionNotificationService;
+import com.boozallen.aiops.mda.metamodel.AissembleModelInstanceRepository;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
@@ -20,6 +21,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * Represents a record instance.
@@ -91,6 +93,49 @@ public List<RecordField> getFields() {
         return fields;
     }
 
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public List<String> getFieldIds(AissembleModelInstanceRepository metadataRepo) {
+        List<String> fieldIds = new ArrayList<>();
+        for(RecordField recordField: this.getFields()) {
+            fieldIds.add(recordField.getName());
+        }
+
+        List<String> relatedFieldIds = getFieldIdsFromRelations(metadataRepo);
+        fieldIds.addAll(relatedFieldIds);
+
+        return fieldIds;
+    }
+
+    /***
+     * This method gets the fully qualified field ids from relations (dot notation).
+     * For example: FieldBParent.FieldB
+     * @param metadataRepo the configured metadataRepop
+     * @return a list of fully qualified field names
+     */
+    private List<String> getFieldIdsFromRelations(AissembleModelInstanceRepository metadataRepo) {
+        List<String> fieldIds = new ArrayList<>();
+        List<Relation> relations =  getRelations();
+        for(Relation relation: relations) {
+            String relationPackage = relation.getPackage();
+            String relationName = relation.getName();
+            Record relatedRecord = metadataRepo.getRecord(relationPackage, relationName);
+
+            if(relatedRecord != null) {
+
+                List<String> updatedList = relatedRecord.getFieldIds(metadataRepo).stream()
+                        .map(s -> relation.getColumn() + "." + s) // Prepend the prefix
+                        .collect(Collectors.toList());
+
+                fieldIds.addAll(updatedList);
+            }
+        }
+
+        return fieldIds;
+    }
+
     /**
      * Adds a field to this instance.
      * 
diff --git a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/RelationElement.java b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/RelationElement.java
@@ -92,6 +92,15 @@ public void setDocumentation(String documentation) {
         this.documentation = documentation;
     }
 
+    /***
+     * Sets the Column name.
+     * @param column the column name
+     */
+    public void setColumn(String column) {
+        // This method is used in testing. i.e. not normally set from the RelationElement directly
+        this.column = column;
+    }
+
     /**
      * Sets the multiplicity value.
      * 
diff --git a/foundation/foundation-mda/src/main/resources/templates/data-delivery-pyspark/encryption.py.vm b/foundation/foundation-mda/src/main/resources/templates/data-delivery-pyspark/encryption.py.vm
@@ -23,19 +23,23 @@
         policy_manager = DataEncryptionPolicyManager.getInstance()
         retrieved_policies = policy_manager.policies
 
+        #if (${step.hasInboundNativeCollectionTypeAndRelations()})
+        if(len(retrieved_policies.items()) > 0):
+            raise NotImplementedError("Encryption of records that contain relations is not yet supported.")
+        #else
         for key, encrypt_policy in retrieved_policies.items():
             # Encryption policies have a property called encryptPhase.
             # If that property is missing then we should ignore the policy.
             if encrypt_policy.encryptPhase:
                 if self.step_phase.lower() == encrypt_policy.encryptPhase.lower():
-
                     encrypt_fields = encrypt_policy.encryptFields
                     input_fields = self.get_fields_list(inbound)
                     field_intersection = list(set(encrypt_fields) & set(input_fields))
 
                     return_payload = self.apply_encryption_to_dataset(inbound, field_intersection, encrypt_policy.encryptAlgorithm)
                 else:
                     ${step.capitalizedName}Base.logger.info('Encryption policy does not apply to this phase: ' + self.step_phase)
+        #end
 
         return return_payload
        #end
diff --git a/foundation/foundation-mda/src/main/resources/templates/data-delivery-spark/encryption.java.vm b/foundation/foundation-mda/src/main/resources/templates/data-delivery-spark/encryption.java.vm
@@ -24,6 +24,9 @@ ${step.encryptionSignature} {
         Map<String, EncryptionPolicy> policies = encryptionPolicyManager.getEncryptPolicies();
 
         if(!policies.isEmpty()) {
+        #if (${step.hasInboundNativeCollectionTypeAndRelations()})
+            throw new UnsupportedOperationException("Encryption of records that contain relations is not yet supported.");
+        #else
             for(EncryptionPolicy encryptionPolicy: policies.values()) {
                 if(stepPhase.equalsIgnoreCase(encryptionPolicy.getEncryptPhase())){
                     List<String> encryptFields = encryptionPolicy.getEncryptFields();
@@ -103,6 +106,7 @@ ${step.encryptionSignature} {
                     }
                 }
             }
+        #end
         }
     }
     #if ($step.hasMessagingInbound())
diff --git a/foundation/foundation-mda/src/test/java/com/boozallen/aiops/mda/metamodel/element/PipelineSteps.java b/foundation/foundation-mda/src/test/java/com/boozallen/aiops/mda/metamodel/element/PipelineSteps.java
@@ -13,6 +13,7 @@
 import com.boozallen.aiops.mda.generator.common.DataFlowStrategy;
 import com.boozallen.aiops.mda.generator.common.MachineLearningStrategy;
 import com.boozallen.aiops.mda.generator.common.PipelineContext;
+import com.boozallen.aiops.mda.metamodel.AissembleModelInstanceRepository;
 import com.boozallen.aiops.mda.util.TestMetamodelUtil;
 import com.boozallen.aiops.mda.metamodel.json.AissembleMdaJsonUtils;
 import io.cucumber.java.After;
@@ -37,10 +38,14 @@
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import org.technologybrewery.fermenter.mda.metamodel.ModelInstanceRepositoryManager;
+import org.technologybrewery.fermenter.mda.metamodel.ModelRepositoryConfiguration;
+import org.technologybrewery.fermenter.mda.metamodel.ModelInstanceUrl;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -56,6 +61,7 @@ public class PipelineSteps extends AbstractModelInstanceSteps {
     public static final String DO_MODIFY_REGEX = "DO\\s+MODIFY";
     public static final String ABSTRACT_DATA_ACTION_IMPL_INHERIT_REGEX = "AbstractDataActionImpl\\(AbstractDataAction\\):";
     public static final String ABSTRACT_PIPELINE_STEP_INHERIT_REGEX = "AbstractPipelineStep\\(AbstractDataActionImpl\\)";
+    public static final String TEST_RECORD_RELATIONS = "test.record.relations";
 
     protected Pipeline pipeline;
     protected Pipeline dataFLowPipeline;
@@ -967,4 +973,110 @@ private List<Step> createStepForPySparkRDMSDataFlowPipeline() {
         return steps;
     }
 
+
+    @Given("a valid data delivery pipeline with native inbound type")
+    public void a_valid_data_delivery_pipeline_with_native_inbound_type() throws IOException {
+        RecordElement newRecordB = createNewRecordWithNameAndPackage("RecordB", RELATION_PACKAGE);
+        saveRecordToFile(newRecordB);
+
+        RecordElement nativeRecord = createNewRecordWithNameAndPackage("RecordA", TEST_RECORD_RELATIONS);
+
+        // Create relation from RecordA to RecordB
+        RelationElement recordRelation = new RelationElement();
+        recordRelation.setName("RecordB");
+        recordRelation.setPackage(RELATION_PACKAGE);
+        recordRelation.setColumn("FieldBParent");
+        recordRelation.setDocumentation("Some Documentation");
+        recordRelation.setMultiplicity("1-1");
+
+        nativeRecord.addRelation(recordRelation);
+        saveRecordToFile(nativeRecord);
+
+        PipelineElement newPipeline = TestMetamodelUtil.createPipelineWithType("nativeDataDelivery", "com.boozallen.aiops.test", "data-flow", "versioned-streaming-spark-java");
+
+        // Add steps to pipeline
+        StepElement step = new StepElement();
+        step.setName("inbound-step");
+        step.setType("synchronous");
+
+        PersistElement persist = new PersistElement();
+        persist.setType("delta-lake");
+        step.setPersist(persist);
+
+        StepDataBindingElement inbound = new StepDataBindingElement();
+        inbound.setType("native");
+        inbound.setChannelName("unit-test-inbound");
+        inbound.setChannelType("topic");
+
+        // Add a record type to the inbound
+        StepDataRecordTypeElement stepDataRecordTypeElement = new StepDataRecordTypeElement();
+        stepDataRecordTypeElement.setName("RecordA");
+        stepDataRecordTypeElement.setPackage(TEST_RECORD_RELATIONS);
+        inbound.setRecordType(stepDataRecordTypeElement);
+        step.setInbound(inbound);
+
+        StepDataBindingElement outbound = new StepDataBindingElement();
+        outbound.setType("messaging");
+        outbound.setChannelName("unit-test-outbound");
+        outbound.setChannelType("queue");
+        step.setOutbound(outbound);
+
+        for (int j = 0; j < RandomUtils.insecure().randomInt(0, 4); j++) {
+            ConfigurationItemElement configurationItem = new ConfigurationItemElement();
+            configurationItem.setKey(RandomStringUtils.insecure().nextAlphanumeric(3));
+            configurationItem.setValue(RandomStringUtils.insecure().nextAlphanumeric(10));
+            step.addConfigurationItem(configurationItem);
+        }
+
+        newPipeline.addStep(step);
+
+        pipelineFile = savePipelineToFile(newPipeline);
+    }
+
+    private RecordElement createNewRecordWithNameAndPackage(String name, String packageName) {
+        RecordElement newRecord = new RecordElement();
+        if (StringUtils.isNotBlank(name)) {
+            newRecord.setName(name);
+        }
+
+        if (StringUtils.isNotBlank(packageName)) {
+            newRecord.setPackage(packageName);
+        }
+
+        return newRecord;
+    }
+
+    @Then("the native inbound pipeline can be read with the associated record and relations")
+    public void the_native_inbound_pipeline_can_be_read() {
+        primeRecordRepo("example");
+
+        pipeline = JsonUtils.readAndValidateJson(pipelineFile, PipelineElement.class);
+        List<? extends Step> mySteps = pipeline.getSteps();
+
+        for(Step mystep: mySteps) {
+            StepDataBinding myInbound = mystep.getInbound();
+            StepDataRecordType stepDataRecordType = myInbound.getRecordType();
+            Record myRecord = stepDataRecordType.getRecordType();
+
+            assertTrue("Record Relations was empty.", myRecord.getRelations() != null && !myRecord.getRelations().isEmpty());
+        }
+    }
+
+    /***
+     * This method loads the metadataRepo for records.  The default metadataRepo does
+     * not set the correct ModelInstanceUrl, so calling this will prime the repo with the correct path.
+     * @param artifactId The name of the project
+     */
+    protected void primeRecordRepo(String artifactId) {
+        ModelRepositoryConfiguration config = new ModelRepositoryConfiguration();
+        config.setArtifactId(artifactId);
+        config.setBasePackage(BOOZ_ALLEN_PACKAGE);
+        Map<String, ModelInstanceUrl> metadataUrlMap = config.getMetamodelInstanceLocations();
+        metadataUrlMap.put(artifactId,
+                new ModelInstanceUrl(artifactId, recordsDirectory.getParentFile().toURI().toString()));
+
+        metadataRepo = new AissembleModelInstanceRepository(config);
+        ModelInstanceRepositoryManager.setRepository(metadataRepo);
+        metadataRepo.load();
+    }
 }
diff --git a/foundation/foundation-mda/src/test/java/com/boozallen/aiops/mda/metamodel/element/RecordSteps.java b/foundation/foundation-mda/src/test/java/com/boozallen/aiops/mda/metamodel/element/RecordSteps.java
@@ -35,8 +35,6 @@
 import io.cucumber.java.en.Given;
 import io.cucumber.java.en.Then;
 import io.cucumber.java.en.When;
-//import org.apache.commons.lang3.RandomStringUtils;
-//import org.technologybrewery.fermenter.mda.metamodel.element.FieldElement;
 
 public class RecordSteps extends AbstractModelInstanceSteps {
 
@@ -519,11 +517,11 @@ public void aRecordWithARelationThatDoesNotDefineRequired() {
 
     @Then("the relation has the correct default values")
     public void theRelationHasTheCorrectDefaultValues() {
-        Record parentRecord = this.metadataRepo.getRecord(TEST_RECORD_RELATIONS,"ParentRecord");
+        Record parentRecord = this.metadataRepo.getRecord(TEST_RECORD_RELATIONS, "ParentRecord");
         assertNotNull("Parent Record with relation was not created successfully", parentRecord);
         assertNotNull("Parent record does not have the appropriate relation",
                 parentRecord.getRelations());
-        for(Relation childRelation : parentRecord.getRelations()) {
+        for (Relation childRelation : parentRecord.getRelations()) {
             assertNull("Child relation should not be required", childRelation.isRequired());
             assertNull("Child relation should not have a description", childRelation.getDocumentation());
             assertNull("Child relation should not have a column", childRelation.getColumn());
diff --git a/foundation/foundation-mda/src/test/resources/specifications/pipeline-model.feature b/foundation/foundation-mda/src/test/resources/specifications/pipeline-model.feature
@@ -193,4 +193,10 @@ Feature: Specify pipelines to be generated
   Scenario: Model lineage is invalid for a data delivery pipeline
     Given an otherwise valid data delivery pipeline with a step that has model lineage defined
     When pipelines are read
-    Then there are validation error messages
+    Then there are validation error messages
+
+  Scenario: Pipeline with native inbound and record relations can be parsed
+    Given a valid data delivery pipeline with native inbound type
+    When pipelines are read
+    Then the native inbound pipeline can be read with the associated record and relations
+
diff --git a/foundation/foundation-mda/src/test/resources/specifications/record-model.feature b/foundation/foundation-mda/src/test/resources/specifications/record-model.feature
@@ -203,7 +203,7 @@ Feature: Specify record of semantically defined types
 
   Scenario: A record can reference another record as a field
     Given record B
-    Given record A has a relation to record B
+    And record A has a relation to record B
     When records are read
     Then the records are successfully created
     And you can reference record B from record A

Original file line number	Diff line number	Diff line change
`@@ -24,6 +24,9 @@ ${step.encryptionSignature} {`
`24`	`24`	`Map<String, EncryptionPolicy> policies = encryptionPolicyManager.getEncryptPolicies();`
`25`	`25`
`26`	`26`	`if(!policies.isEmpty()) {`
	`27`	`+ #if (${step.hasInboundNativeCollectionTypeAndRelations()})`
	`28`	`+ throw new UnsupportedOperationException("Encryption of records that contain relations is not yet supported.");`
	`29`	`+ #else`
`27`	`30`	`for(EncryptionPolicy encryptionPolicy: policies.values()) {`
`28`	`31`	`if(stepPhase.equalsIgnoreCase(encryptionPolicy.getEncryptPhase())){`
`29`	`32`	`List<String> encryptFields = encryptionPolicy.getEncryptFields();`
`@@ -103,6 +106,7 @@ ${step.encryptionSignature} {`
`103`	`106`	`}`
`104`	`107`	`}`
`105`	`108`	`}`
	`109`	`+ #end`
`106`	`110`	`}`
`107`	`111`	`}`
`108`	`112`	`#if ($step.hasMessagingInbound())`