glue-workflow.yaml

template:
  path: glue-workflow.j2
stack_name: "{{ stack_group_config.namespace }}-glue-workflow"
dependencies:
  - develop/namespaced/glue-tables.yaml
  - develop/namespaced/glue-job-S3ToJsonS3.yaml
  - develop/namespaced/glue-job-JSONToParquet.yaml
  - develop/namespaced/glue-job-compare-parquet.yaml
  - develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml
  - develop/glue-job-role.yaml
  - develop/s3-cloudformation-bucket.yaml
parameters:
  Namespace: {{ stack_group_config.namespace }}
  JsonBucketName: {{ stack_group_config.intermediate_bucket_name }}
  ParquetBucketName: {{ stack_group_config.processed_data_bucket_name }}
  GlueDatabase: !stack_output_external "{{ stack_group_config.namespace }}-glue-tables::DatabaseName"
  CrawlerRole: !stack_output_external glue-job-role::RoleArn
  S3ToJsonJobName: !stack_output_external "{{ stack_group_config.namespace }}-glue-job-S3ToJsonS3::JobName"
  CompareParquetStagingNamespace: {{ stack_group_config.namespace }}
  CompareParquetMainNamespace: "main"
  S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
  CloudformationBucketName: {{ stack_group_config.template_bucket_name }}
stack_tags:
  {{ stack_group_config.default_stack_tags }}
sceptre_user_data:
  dataset_schemas: !file src/glue/resources/table_columns.yaml
  data_values_expectations: !file src/glue/resources/data_values_expectations.json