Merge pull request #582 from boozallen/577-policy-location-undefined

#577 the policies location property can be undefined for encryption policies

Author: carter-cundiff

foundation/aissemble-foundation-core-python/src/policy_manager/configuration/policy_configuration.py

@@ -25,4 +25,7 @@ def policiesLocation(self) -> str:
         Configures the location and file name of the file that contains the
         policies.
         """
-        return self.properties["policies-location"]
+        try:
+            return self.properties["policies-location"]
+        except TypeError:
+            return None

foundation/foundation-mda/src/main/resources/templates/data-delivery-pyspark/encryption.py.vm

@@ -20,26 +20,27 @@
 
         directory = PolicyConfiguration().policiesLocation()
 
-        policy_manager = DataEncryptionPolicyManager.getInstance()
-        retrieved_policies = policy_manager.policies
-
-        #if (${step.hasInboundNativeCollectionTypeAndRelations()})
-        if(len(retrieved_policies.items()) > 0):
-            raise NotImplementedError("Encryption of records that contain relations is not yet supported.")
-        #else
-        for key, encrypt_policy in retrieved_policies.items():
-            # Encryption policies have a property called encryptPhase.
-            # If that property is missing then we should ignore the policy.
-            if encrypt_policy.encryptPhase:
-                if self.step_phase.lower() == encrypt_policy.encryptPhase.lower():
-                    encrypt_fields = encrypt_policy.encryptFields
-                    input_fields = self.get_fields_list(inbound)
-                    field_intersection = list(set(encrypt_fields) & set(input_fields))
-
-                    return_payload = self.apply_encryption_to_dataset(inbound, field_intersection, encrypt_policy.encryptAlgorithm)
-                else:
-                    ${step.capitalizedName}Base.logger.info('Encryption policy does not apply to this phase: ' + self.step_phase)
-        #end
+        if directory is not None and os.path.isdir(directory):
+            policy_manager = DataEncryptionPolicyManager.getInstance()
+            retrieved_policies = policy_manager.policies
+
+            #if (${step.hasInboundNativeCollectionTypeAndRelations()})
+            if(len(retrieved_policies.items()) > 0):
+                raise NotImplementedError("Encryption of records that contain relations is not yet supported.")
+            #else
+            for key, encrypt_policy in retrieved_policies.items():
+                # Encryption policies have a property called encryptPhase.
+                # If that property is missing then we should ignore the policy.
+                if encrypt_policy.encryptPhase:
+                    if self.step_phase.lower() == encrypt_policy.encryptPhase.lower():
+                        encrypt_fields = encrypt_policy.encryptFields
+                        input_fields = self.get_fields_list(inbound)
+                        field_intersection = list(set(encrypt_fields) & set(input_fields))
+
+                        return_payload = self.apply_encryption_to_dataset(inbound, field_intersection, encrypt_policy.encryptAlgorithm)
+                    else:
+                        ${step.capitalizedName}Base.logger.info('Encryption policy does not apply to this phase: ' + self.step_phase)
+            #end
 
         return return_payload
        #end

foundation/foundation-mda/src/main/resources/templates/data-delivery-spark/encryption.java.vm

@@ -20,7 +20,7 @@ ${step.encryptionSignature} {
     EncryptionPolicyManager encryptionPolicyManager = EncryptionPolicyManager.getInstance();
     String filePath = encryptionPolicyManager.getPoliciesLocation();
 
-    if(Files.isDirectory(Paths.get(filePath))) {
+    if(filePath != null && Files.isDirectory(Paths.get(filePath))) {
         Map<String, EncryptionPolicy> policies = encryptionPolicyManager.getEncryptPolicies();
 
         if(!policies.isEmpty()) {

test/test-mda-models/aissemble-test-data-delivery-pyspark-model/tests/features/pyspark_data_encryption.feature

@@ -22,3 +22,9 @@ Feature: Data encryption
   Given a pipeline with native inbound collection and inbound record type
   When AES encryption is requested
   Then the correct AES algorithm is applied to the data set
+
+  Scenario: The policies location property can be undefined for encryption policies
+  Given a pipeline with an inbound data type
+  And the policies location property is not defined
+  When the check and apply encryption method is called
+  Then the method completes without applying encryption

test/test-mda-models/aissemble-test-data-delivery-pyspark-model/tests/features/steps/pyspark_data_encryption_steps.py

@@ -17,6 +17,7 @@
 
 from behave import given, when, then  # pylint: disable=no-name-in-module
 import nose.tools as nt
+import os
 from aissemble_test_data_delivery_pyspark_model.step.native_inbound_with_custom_types import (
     NativeInboundWithCustomTypes,
 )
@@ -26,6 +27,9 @@
 from aissemble_test_data_delivery_pyspark_model.step.native_inbound_with_custom_collection_type import (
     NativeInboundWithCustomCollectionType,
 )
+from aissemble_test_data_delivery_pyspark_model.step.native_inbound_and_outbound import (
+    NativeInboundAndOutbound,
+)
 from aissemble_test_data_delivery_pyspark_model.record.custom_data import CustomData
 from krausening.logging import LogManager
 
@@ -83,6 +87,19 @@ def step_impl(context):
     )
 
 
+@given("a pipeline with an inbound data type")
+def step_impl(context):
+    context.pipeline = NativeInboundAndOutbound()
+
+
+@given("the policies location property is not defined")
+def step_impl(context):
+    # Get the current krausening base dir for restoring after the test
+    context.default_krausening_base = os.environ.get("KRAUSENING_BASE")
+
+    os.environ["KRAUSENING_BASE"] = "invalid/path"
+
+
 @when("encryption is called on the inbound record")
 def step_impl(context):
     context.encrypted_dataset = context.pipeline.apply_encryption_to_dataset(
@@ -101,6 +118,15 @@ def step_impl(context):
     logger.info(context.encrypted_dataset)
 
 
+@when("the check and apply encryption method is called")
+def step_impl(context):
+    context.exception = None
+    try:
+        context.pipeline.check_and_apply_encryption_policy(None)
+    except Exception as e:
+        context.exception = e
+
+
 @then("the correct fields are encrypted")
 def step_impl(context):
     for record in context.encrypted_dataset:
@@ -227,3 +253,14 @@ def step_impl(context):
         logger.info(
             "Set[(dataframe)] - The Vault encrypted value is: " + encrypted_field
         )
+
+
+@then("the method completes without applying encryption")
+def step_impl(context):
+    nt.ok_(
+        context.exception is None,
+        "An exception was thrown",
+    )
+
+    # Restore the krausening base dir to the value from before the test
+    os.environ["KRAUSENING_BASE"] = context.default_krausening_base

test/test-mda-models/test-data-delivery-spark-model/pom.xml

@@ -36,12 +36,14 @@
                                 <exclude>test/java/com/boozallen/aiops/mda/pattern/RelationTest.java</exclude>
                                 <exclude>test/java/com/boozallen/aiops/mda/pattern/RelationValidationTest.java</exclude>
                                 <exclude>test/java/com/boozallen/aiops/mda/pattern/SparkSchemaTest.java</exclude>
+                                <exclude>test/java/com/boozallen/aiops/mda/pattern/EncryptionTest.java</exclude>
                                 <exclude>test/resources/config/</exclude>
                                 <exclude>test/resources/specifications/record.feature</exclude>
                                 <exclude>test/resources/specifications/lineage.feature</exclude>
                                 <exclude>test/resources/specifications/relation.feature</exclude>
                                 <exclude>test/resources/specifications/relation-validation.feature</exclude>
                                 <exclude>test/resources/specifications/sparkSchema.feature</exclude>
+                                <exclude>test/resources/specifications/encryption.feature</exclude>
                                 <exclude>test/resources/krausening/base/</exclude>
                                 <exclude>test/resources/apps</exclude>
                             </excludes>

test/test-mda-models/test-data-delivery-spark-model/src/test/java/com/boozallen/aiops/mda/pattern/EncryptionTest.java

@@ -0,0 +1,62 @@
+package com.boozallen.aiops.mda.pattern;
+
+/*-
+ * #%L
+ * aiSSEMBLE::Test::MDA::Data Delivery Spark
+ * %%
+ * Copyright (C) 2021 Booz Allen
+ * %%
+ * This software package is licensed under the Booz Allen Public License. All Rights Reserved.
+ * #L%
+ */
+
+import io.cucumber.java.en.Given;
+import io.cucumber.java.en.Then;
+import io.cucumber.java.en.When;
+
+import org.technologybrewery.krausening.Krausening;
+
+import static org.junit.Assert.assertNull;
+
+/**
+ * Implementation steps for encryption.feature
+ */
+public class EncryptionTest {
+    private String defaultKrauseningBaseDir;
+    private NativeInboundAndOutbound pipeline;
+    private Exception exception;
+    private Krausening krausening = Krausening.getInstance();
+
+    @Given("a pipeline with an inbound data type")
+    public void a_pipeline_with_an_inbound_data_type() {
+        this.pipeline = new NativeInboundAndOutbound();
+    }
+
+    @Given("the policies location property is not defined")
+    public void the_policies_location_property_is_not_defined() {
+        // Get the current krausening base dir for restoring after the test
+        this.defaultKrauseningBaseDir = System.getProperty("KRAUSENING_BASE");
+
+        // Force krausening to reload with the new base dir
+        System.setProperty("KRAUSENING_BASE", "invalid/path");
+        this.krausening.loadProperties();
+    }
+
+    @When("the check and apply encryption method is called")
+    public void the_check_and_apply_encryption_method_is_called() {
+        try {
+            this.pipeline.checkAndApplyEncryptionPolicy(null);
+        } catch (Exception e) {
+            this.exception = e;
+        }
+    }
+
+    @Then("the method completes without applying encryption")
+    public void the_method_completes_without_applying_encryption() {
+        assertNull("An exception was thrown", this.exception);
+
+        // Restore the krausening base dir to the value from before the test
+        System.setProperty("KRAUSENING_BASE", this.defaultKrauseningBaseDir);
+        this.krausening.loadProperties();
+    }
+}

test/test-mda-models/test-data-delivery-spark-model/src/test/resources/specifications/encryption.feature

@@ -0,0 +1,8 @@
+@encryption
+Feature: Step encryption are generated correctly and function as expected
+  
+  Scenario: The policies location property can be undefined for encryption policies
+    Given a pipeline with an inbound data type
+    And the policies location property is not defined
+    When the check and apply encryption method is called
+    Then the method completes without applying encryption