feat: add dbt project

.envrc

@@ -0,0 +1,11 @@
+[[ ! -d py_venv ]] && python3 -m venv py_venv || true
+pip install -r requirements.txt --quiet
+source ./py_venv/bin/activate
+
+if [[ -f ./demo.env ]]; then
+    export $(xargs < ./demo.env)
+else
+    echo "No demo.env file found"
+    echo "See demo-template.env for an example"
+fi
+

.gitignore

@@ -6,4 +6,7 @@ public
 target
 build
 index.adoc
-
+.idea
+.vscode
+py_venv
+demo.env

burger_factory/.dbt/.user.yml

@@ -0,0 +1 @@
+id: 6a1fd886-aa39-410c-ae37-7ef78b4a28d1

burger_factory/.dbt/profiles.yml

@@ -0,0 +1,14 @@
+burger_factory:
+  outputs:
+    dev:
+      account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}"
+      user: "{{ env_var('SNOWFLAKE_USER') }}"
+      private_key_path: "{{ env_var('SNOWFLAKE_PRIVATE_KEY_PATH') }}"
+      private_key_passphrase: "{{ env_var('PRIVATE_KEY_PASSPHRASE') }}"
+      database: "{{ env_var('SNOWFLAKE_DATABASE') }}"
+      schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}"
+      role: "{{ env_var('SNOWFLAKE_ROLE') }}"
+      warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}"
+      threads: 1
+      type: snowflake
+  target: dev

burger_factory/.gitignore

@@ -0,0 +1,4 @@
+
+target/
+dbt_packages/
+logs/

burger_factory/README.md

@@ -0,0 +1,15 @@
+Welcome to your new dbt project!
+
+### Using the starter project
+
+Try running the following commands:
+- dbt run
+- dbt test
+
+
+### Resources:
+- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
+- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
+- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
+- Find [dbt events](https://events.getdbt.com) near you
+- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

burger_factory/dbt_project.yml

@@ -0,0 +1,17 @@
+name: 'burger_factory'
+version: '1.0.0'
+profile: 'burger_factory'
+
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:
+  - "target"
+  - "dbt_packages"
+
+models:
+  burger_factory:

burger_factory/ddl.sql

@@ -0,0 +1,23 @@
+CREATE OR REPLACE DATABASE DEV_KARATE_DATA_DB;
+
+USE DATABASE DEV_KARATE_DATA_DB;
+
+CREATE OR REPLACE SCHEMA BURGER_INPUT;
+CREATE OR REPLACE SCHEMA BURGER_OUTPUT;
+
+CREATE OR REPLACE TABLE BURGER_INPUT.BREAD (
+    CLIENT_ID STRING,
+    VALUE STRING
+);
+CREATE OR REPLACE TABLE BURGER_INPUT.VEGETABLE (
+    CLIENT_ID STRING,
+    VALUE STRING
+);
+CREATE OR REPLACE TABLE BURGER_INPUT.MEAT (
+    CLIENT_ID STRING,
+    VALUE STRING
+);
+CREATE OR REPLACE TABLE BURGER_OUTPUT.BURGER (
+    CLIENT_ID STRING,
+    VALUE STRING
+);

burger_factory/it/features/burger-factory-clone-schemas.feature

@@ -0,0 +1,33 @@
+Feature: Demo - Clone Schemas
+  Background:
+    * json cliConfig = snowflake.cliConfigFromEnv
+    * string jwtToken = snowflake.cli.generateJwtToken(cliConfig)
+    * json restConfig = ({...cliConfig, snowflakeConfig: snowflakeConfigs.BREAD, jwtToken: jwtToken})
+    * json cloneResult = cloneSnowflakeConfigs(restConfig)
+    * configure afterScenario = function(){ dropSnowflakeConfigs(restConfig, cloneResult.snowflakeConfigs) }
+
+  Scenario Outline: Burger Factory - <bread> + <vegetable> + <meat> = <output>
+    Given string clientId = "😋_"+lectra.uuid()
+    And table inserts
+      | table       | value         |
+      | "BREAD"     | "<bread>"     |
+      | "VEGETABLE" | "<vegetable>" |
+      | "MEAT"      | "<meat>"      |
+    And def genStatement = (row) => "INSERT INTO "+row.table+"(CLIENT_ID, VALUE) VALUES ('"+clientId+"','"+row.value+"')"
+    And json responses = karate.map(inserts, (row) => snowflake.rest.runSql({...cliConfig, jwtToken: jwtToken, snowflakeConfig: cloneResult.snowflakeConfigs[row.table], statement: genStatement(row)}).status)
+    And match each responses == "OK"
+
+    * string cmd = cloneResult.dbtPrefix+" dbt run"
+    When string dbtConsoleOutput = karate.exec("bash -c '"+cmd+"'")
+    And match dbtConsoleOutput contains "Completed successfully"
+
+    Then string selectStatement = "SELECT VALUE FROM BURGER WHERE CLIENT_ID='"+clientId+"'"
+    And json response = snowflake.rest.runSql({...cliConfig, snowflakeConfig: cloneResult.snowflakeConfigs.BURGER, statement: selectStatement })
+    And match response.data == [ { "VALUE" : "<output>" } ]
+
+    Examples:
+      | bread | vegetable | meat | output       |
+      | 🍞    | 🍅        | 🥩   | 🍔           |
+      | 🍞    | 🍅        | 🍗   | 🍔           |
+      | 🍞    | 🍅        | 🐟   | 🍔           |
+      | 🍞    | 🥕        | 🥩   | 🍞 + 🥕 + 🥩 |

burger_factory/it/features/burger-factory.feature

@@ -0,0 +1,30 @@
+@ignore
+Feature: Demo
+  Background:
+    * json cliConfig = snowflake.cliConfigFromEnv
+    * string jwtToken = snowflake.cli.generateJwtToken(cliConfig)
+
+  Scenario Outline: Burger Factory - <bread> + <vegetable> + <meat> = <output>
+    Given string clientId = "😋_"+lectra.uuid()
+    And table inserts
+      | table       | value         |
+      | "BREAD"     | "<bread>"     |
+      | "VEGETABLE" | "<vegetable>" |
+      | "MEAT"      | "<meat>"      |
+    And def genStatement = (row) => "INSERT INTO "+row.table+"(CLIENT_ID, VALUE) VALUES ('"+clientId+"','"+row.value+"')"
+    And json responses = karate.map(inserts, (row) => snowflake.rest.runSql({...cliConfig, jwtToken: jwtToken, snowflakeConfig: snowflakeConfigs[row.table], statement: genStatement(row)}).status)
+    And match each responses == "OK"
+
+    When string dbtConsoleOutput = karate.exec("dbt run")
+    And match dbtConsoleOutput contains "Completed successfully"
+
+    Then string selectStatement = "SELECT VALUE FROM BURGER WHERE CLIENT_ID='"+clientId+"'"
+    And json response = snowflake.rest.runSql({...cliConfig, snowflakeConfig: snowflakeConfigs.BURGER, statement: selectStatement })
+    And match response.data == [ { "VALUE" : "<output>" } ]
+
+    Examples:
+      | bread | vegetable | meat | output       |
+      | 🍞    | 🍅        | 🥩   | 🍔           |
+      | 🍞    | 🍅        | 🍗   | 🍔           |
+      | 🍞    | 🍅        | 🐟   | 🍔           |
+      | 🍞    | 🥕        | 🥩   | 🍞 + 🥕 + 🥩 |

burger_factory/it/karate-config.js

@@ -0,0 +1,41 @@
+function fn() {
+
+
+    const snowflakeConfig = snowflake.snowflakeConfigFromEnv;
+    const sourceSchema = java.lang.System.getenv("SNOWFLAKE_SCHEMA_SOURCE");
+    const schema = java.lang.System.getenv("SNOWFLAKE_SCHEMA");
+    const snowflakeConfigSource = {...snowflake.snowflakeConfigFromEnv, schema: sourceSchema};
+    const genSnowflakeConfigs = (postfix) => {
+        const toAdd = postfix !== undefined ? "_"+postfix : "";
+        return { 
+        "BREAD": {...snowflakeConfigSource, schema: snowflakeConfigSource.schema+toAdd},
+        "VEGETABLE": {...snowflakeConfigSource, schema: snowflakeConfigSource.schema+toAdd},
+        "MEAT": {...snowflakeConfigSource, schema: snowflakeConfigSource.schema+toAdd},
+        "BURGER": {...snowflakeConfig, schema: snowflakeConfig.schema+toAdd}
+        };
+    };
+
+    const cloneSnowflakeConfigs = (restConfig) => {
+        const postfix = lectra.uuid().toUpperCase().replaceAll("-","_");
+        const clone1 = snowflake.rest.cloneSchema({...restConfig, "schemaToClone": snowflakeConfigSource.schema, "schemaToCreate": snowflakeConfigSource.schema+"_"+postfix}).status;
+        const clone2 = snowflake.rest.cloneSchema({...restConfig, "schemaToClone": snowflakeConfig.schema, "schemaToCreate": snowflakeConfig.schema+"_"+postfix}).status;
+        if (clone1 !== "OK" || clone2 !== "OK") {
+            karate.fail("cloneSnowflakeConfigs failed");
+        }
+        return { snowflakeConfigs: genSnowflakeConfigs(postfix), dbtPrefix: "SNOWFLAKE_SCHEMA_SOURCE="+sourceSchema+"_"+postfix+" SNOWFLAKE_SCHEMA="+schema+"_"+postfix};
+    };
+
+    const dropSnowflakeConfigs = (restConfig, snowflakeConfigs) => {
+        const dropResults = karate.map(snowflakeConfigs, (config) => snowflake.rest.dropSchema({...restConfig, "schemaToDrop": config.schema}).status);
+        if (dropResults.some(result => result.status !== "OK")) {
+            karate.fail("dropSnowflakeConfigs failed");
+        }
+    };
+
+    return {
+        "projectName": "burger_factory",
+        "snowflakeConfigs": genSnowflakeConfigs(),
+        "cloneSnowflakeConfigs": cloneSnowflakeConfigs,
+        "dropSnowflakeConfigs": dropSnowflakeConfigs
+    }
+}

burger_factory/models/burger.sql

@@ -0,0 +1,30 @@
+
+{{ config(
+    materialized='incremental',
+    unique_key='CLIENT_ID',
+    incremental_strategy='delete+insert'
+    ) 
+}}
+
+WITH SOURCE_DATA AS (
+    SELECT B.CLIENT_ID, B.VALUE AS BREAD_VALUE, V.VALUE AS VEGETABLE_VALUE, M.VALUE AS MEAT_VALUE
+    FROM {{ source('burger_input', 'bread') }} AS B
+    INNER JOIN {{ source('burger_input', 'vegetable') }} AS V ON V.CLIENT_ID = B.CLIENT_ID
+    INNER JOIN {{ source('burger_input', 'meat') }} AS M ON M.CLIENT_ID = B.CLIENT_ID
+),
+BURGER_DATA AS (
+    SELECT CLIENT_ID, '🍔' AS VALUE
+    FROM SOURCE_DATA
+    WHERE BREAD_VALUE = '🍞' 
+    AND VEGETABLE_VALUE = '🍅' 
+    AND (MEAT_VALUE = '🥩' OR MEAT_VALUE = '🍗' OR MEAT_VALUE = '🐟')
+),
+OTHER_DATA AS (
+    SELECT CLIENT_ID, BREAD_VALUE || ' + ' || VEGETABLE_VALUE || ' + ' || MEAT_VALUE AS VALUE
+    FROM SOURCE_DATA
+    WHERE NOT EXISTS (SELECT 1 FROM BURGER_DATA WHERE BURGER_DATA.CLIENT_ID = SOURCE_DATA.CLIENT_ID)
+)
+
+SELECT * FROM BURGER_DATA
+UNION
+SELECT * FROM OTHER_DATA

burger_factory/models/burger.yml

@@ -0,0 +1,5 @@
+version: 2
+
+models:
+ - name: burger
+

burger_factory/models/sources.yml

@@ -0,0 +1,13 @@
+version: 2
+
+sources:
+  - name: burger_input
+    schema: "{{ env_var('SNOWFLAKE_SCHEMA_SOURCE') }}"
+    tables:
+      - name: bread
+        identifier: BREAD
+      - name: vegetable
+        identifier: VEGETABLE
+      - name: meat
+        identifier: MEAT
+

burger_factory/models/unit_tests/burger.yml

@@ -0,0 +1,33 @@
+unit_tests:
+
+  - name: create_a_burger
+    model: burger
+    given:
+      - input: source("burger_input", "bread")
+        rows:
+          - { CLIENT_ID: "😋", VALUE: "🍞" }
+      - input: source("burger_input", "vegetable")
+        rows:
+          - { CLIENT_ID: "😋", VALUE: "🍅" }
+      - input: source("burger_input", "meat")
+        rows:
+          - { CLIENT_ID: "😋", VALUE: "🥩" }
+    expect:
+      rows:
+        - { CLIENT_ID: "😋", VALUE: "🍔" }
+
+  - name: create_a_non_burger
+    model: burger
+    given:
+      - input: source("burger_input", "bread")
+        rows:
+          - { CLIENT_ID: "😋", VALUE: "🍞" }
+      - input: source("burger_input", "vegetable")
+        rows:
+          - { CLIENT_ID: "😋", VALUE: "🥕" }
+      - input: source("burger_input", "meat")
+        rows:
+          - { CLIENT_ID: "😋", VALUE: "🥩" }
+    expect:
+      rows:
+        - { CLIENT_ID: "😋", VALUE: "🍞 + 🥕 + 🥩" }      

demo-template.env

@@ -0,0 +1,13 @@
+SNOWFLAKE_ACCOUNT=xxx.west-europe.azure
+SNOWFLAKE_USER=MY_USER
+SNOWFLAKE_PRIVATE_KEY_PATH=/my-path/private-key.pem
+PRIVATE_KEY_PASSPHRASE=my-passphrase
+
+SNOWFLAKE_DATABASE=DEV_KARATE_DATA_DB
+SNOWFLAKE_SCHEMA=BURGER_OUTPUT
+SNOWFLAKE_SCHEMA_SOURCE=BURGER_INPUT
+SNOWFLAKE_ROLE=MY_ROLE
+SNOWFLAKE_WAREHOUSE=MY_WH
+
+DBT_PROJECT_DIR=burger_factory
+DBT_PROFILES_DIR=burger_factory/.dbt

demo.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+dbt_unit_tests() {
+  dbt test
+}
+
+dbt_run() {
+  dbt run 
+}
+
+karate_jar() {
+  echo "TODO"
+}
+
+karate_docker() {
+  docker run --rm \
+    -v $(pwd)/burger_factory/it/features:/features \
+    -v $(pwd)/burger_factory/it/karate-config.js:/karate-config.js \
+    -v $(pwd)/target:/target \
+    -v ${SNOWFLAKE_PRIVATE_KEY_PATH}:/${SNOWFLAKE_PRIVATE_KEY_PATH} \
+    -v $(pwd)/burger_factory:/burger_factory \
+    --env-file ./demo.env \
+    -e KARATE_EXTENSIONS=snowflake \
+    docker.docker-registry.lectra.com/karate:1.5.1.0 features --threads 8
+}
+
+
+$1

diagrams/common.d2

@@ -52,6 +52,12 @@ classes: {
             font-size: ${component-font-size}
         }
     }
+    sql_table: {
+        shape: sql_table
+        style: {
+            font-size: ${component-font-size}
+        }
+    }
     title: {
         near: top-left
         shape: text

diagrams/ddl.d2

@@ -0,0 +1,33 @@
+...@common
+
+BURGER_INPUT: {
+    class: component
+    bread: { 
+        label: BREAD
+        class: sql_table
+        CLIENT_ID: STRING
+        VALUE: STRING
+    }
+    vegetable: { 
+        label: VEGETABLE
+        class: sql_table
+        CLIENT_ID: STRING
+        VALUE: STRING
+    }
+    meat: { 
+        label: MEAT
+        class: sql_table
+        CLIENT_ID: STRING
+        VALUE: STRING
+    }
+}
+BURGER_OUTPUT: {
+    class: component
+    near: center-right
+    burger: { 
+        label: BURGER
+        class: sql_table
+        CLIENT_ID: STRING
+        VALUE: STRING
+    }
+}