diff --git a/models/users/feedback.sql b/models/users/feedback.sql index ed7ff7fc..4408d826 100644 --- a/models/users/feedback.sql +++ b/models/users/feedback.sql @@ -16,7 +16,7 @@ SELECT document_metadata.uuid as uuid, document_metadata.saved_timestamp, - doc#>>'{meta,source}' AS SOURCE, + doc#>>'{meta,source}' AS source, doc#>>'{meta,url}' AS url, doc#>>'{meta,user,name}' AS user_name, doc#>>'{meta,time}' AS period_start, diff --git a/models/users/telemetry.sql b/models/users/telemetry.sql index d75124f5..9301fb54 100644 --- a/models/users/telemetry.sql +++ b/models/users/telemetry.sql @@ -34,9 +34,9 @@ SELECT THEN doc#>>'{metadata,day}' ELSE - '1' + '1970-01-01' END - )::date AS period_start, + )::timestamptz AS period_start, doc#>>'{metadata,user}' AS user_name, doc#>>'{metadata,versions,app}' AS app_version, doc#>>'{metrics,boot_time,min}' AS boot_time_min, diff --git a/models/users/telemetry_devices.sql b/models/users/telemetry_devices.sql index 57cea442..38f396d8 100644 --- a/models/users/telemetry_devices.sql +++ b/models/users/telemetry_devices.sql @@ -19,7 +19,6 @@ SELECT telemetry.period_start, doc #>> '{device,deviceInfo,hardware,manufacturer}' AS device_manufacturer, doc #>> '{device,deviceInfo,hardware,model}' AS device_model, - doc #>> '{dbInfo,doc_count}' AS doc_count, doc #>> '{device,userAgent}' AS user_agent, doc #>> '{device,deviceInfo,app,version}' AS cht_android_version, doc #>> '{device,deviceInfo,software,androidVersion}' AS android_version, diff --git a/models/users/tests/feedback.yml b/models/users/tests/feedback.yml new file mode 100644 index 00000000..0fb0f188 --- /dev/null +++ b/models/users/tests/feedback.yml @@ -0,0 +1,19 @@ +unit_tests: + - name: test_feedback_model_transformation_and_data_integrity + description: | + This unit test validates the transformation logic in the `feedback` model and ensures data integrity. + It uses fixture data for both `document_metadata` and `source_table` to test the complete logic. + model: feedback + overrides: + macros: + is_incremental: false + given: + - input: ref('document_metadata') + format: csv + fixture: user_document_metadata_initial + - input: source('couchdb', "{{ env_var('POSTGRES_TABLE') }}") + format: csv + fixture: user_source_table_initial + expect: + format: csv + fixture: feedback_initial_expected diff --git a/models/users/tests/telemetry.yml b/models/users/tests/telemetry.yml new file mode 100644 index 00000000..cd8116c9 --- /dev/null +++ b/models/users/tests/telemetry.yml @@ -0,0 +1,19 @@ +unit_tests: + - name: test_telemetry_model_transformation_and_data_integrity + description: | + This unit test validates the transformation logic in the `telemetry` model and ensures data integrity. + It uses fixture data for both `document_metadata` and `source_table` to test the complete logic. + model: telemetry + overrides: + macros: + is_incremental: false + given: + - input: ref('document_metadata') + format: csv + fixture: user_document_metadata_initial + - input: source('couchdb', "{{ env_var('POSTGRES_TABLE') }}") + format: csv + fixture: user_source_table_initial + expect: + format: csv + fixture: telemetry_initial_expected diff --git a/models/users/tests/telemetry_devices.yml b/models/users/tests/telemetry_devices.yml new file mode 100644 index 00000000..3e3c12d1 --- /dev/null +++ b/models/users/tests/telemetry_devices.yml @@ -0,0 +1,19 @@ +unit_tests: + - name: test_telemetry_devices_model_transformation_and_data_integrity + description: | + This unit test validates the transformation logic in the `telemetry_devices` model and ensures data integrity. + It uses fixture data for both `telemtry` and `source_table` to test the complete logic. + model: telemetry_devices + overrides: + macros: + is_incremental: false + given: + - input: ref('telemetry') + format: csv + fixture: telemetry_initial_expected + - input: source('couchdb', "{{ env_var('POSTGRES_TABLE') }}") + format: csv + fixture: user_source_table_initial + expect: + format: csv + fixture: telemetry_devices_initial_expected diff --git a/models/users/user.yml b/models/users/user.yml index f6d49e5d..eb133a65 100644 --- a/models/users/user.yml +++ b/models/users/user.yml @@ -32,3 +32,109 @@ models: data_type: string - name: roles data_type: string + - name: feedback + config: + contract: + enforced: true + columns: + - name: uuid + data_type: string + constraints: + - type: unique + - type: foreign_key + expression: "{{ env_var('POSTGRES_SCHEMA') }}.document_metadata (uuid) ON DELETE CASCADE" + data_tests: + - not_null + - relationships: + to: ref('document_metadata') + field: uuid + - name: saved_timestamp + data_type: timestamp + data_tests: + - not_null + - name: source + data_type: string + - name: url + data_type: string + - name: user_name + data_type: string + - name: period_start + data_type: string + - name: cause + data_type: string + - name: message + data_type: string + - name: telemetry + config: + contract: + enforced: true + columns: + - name: uuid + data_type: string + constraints: + - type: unique + - type: foreign_key + expression: "{{ env_var('POSTGRES_SCHEMA') }}.document_metadata (uuid) ON DELETE CASCADE" + data_tests: + - not_null + - relationships: + to: ref('document_metadata') + field: uuid + - name: saved_timestamp + data_type: timestamp + data_tests: + - not_null + - name: period_start + data_type: timestamp with time zone + - name: user_name + data_type: string + - name: app_version + data_type: string + - name: boot_time_min + data_type: string + - name: boot_time_max + data_type: string + - name: boot_time_count + data_type: string + - name: doc_count_on_local_db + data_type: string + - name: telemetry_devices + config: + contract: + enforced: true + columns: + - name: uuid + data_type: string + constraints: + - type: unique + - type: foreign_key + expression: "{{ env_var('POSTGRES_SCHEMA') }}.document_metadata (uuid) ON DELETE CASCADE" + data_tests: + - not_null + - relationships: + to: ref('document_metadata') + field: uuid + - name: saved_timestamp + data_type: timestamp + data_tests: + - not_null + - name: period_start + data_type: timestamp with time zone + - name: device_manufacturer + data_type: string + - name: device_model + data_type: string + - name: user_agent + data_type: string + - name: cht_android_version + data_type: string + - name: android_version + data_type: string + - name: storage_free + data_type: string + - name: storage_total + data_type: string + - name: network_up_speed + data_type: string + - name: network_down_speed + data_type: string diff --git a/tests/fixtures/user/feedback_initial_expected.csv b/tests/fixtures/user/feedback_initial_expected.csv new file mode 100644 index 00000000..2f5d78b2 --- /dev/null +++ b/tests/fixtures/user/feedback_initial_expected.csv @@ -0,0 +1,2 @@ +uuid,saved_timestamp,source,url,user_name,period_start,cause,message +6,2024-08-02 00:00:00,automatic,http://example.com,admin,2024-08-02 00:00:00,bug,this is a bug diff --git a/tests/fixtures/user/telemetry_devices_initial_expected.csv b/tests/fixtures/user/telemetry_devices_initial_expected.csv new file mode 100644 index 00000000..24aa1944 --- /dev/null +++ b/tests/fixtures/user/telemetry_devices_initial_expected.csv @@ -0,0 +1,2 @@ +uuid,saved_timestamp,period_start,device_manufacturer,device_model,user_agent,cht_android_version,android_version,storage_free,storage_total,network_up_speed,network_down_speed +8,2024-08-02 00:00:00,2024-09-11,Google,Pixel 3,Chrome Mobile,4.15.0,10,100,200,100,200 diff --git a/tests/fixtures/user/telemetry_initial_expected.csv b/tests/fixtures/user/telemetry_initial_expected.csv new file mode 100644 index 00000000..18d3429c --- /dev/null +++ b/tests/fixtures/user/telemetry_initial_expected.csv @@ -0,0 +1,2 @@ +uuid,saved_timestamp,period_start,user_name,app_version,boot_time_min,boot_time_max,boot_time_count,doc_count_on_local_db +8,2024-08-02 00:00:00,2024-09-11,admin,4.15.0,0.5,1.5,2,20 diff --git a/tests/fixtures/user/user_document_metadata_initial.csv b/tests/fixtures/user/user_document_metadata_initial.csv index 46d2dcc1..404dd5e1 100644 --- a/tests/fixtures/user/user_document_metadata_initial.csv +++ b/tests/fixtures/user/user_document_metadata_initial.csv @@ -4,4 +4,7 @@ uuid,_deleted,saved_timestamp,doc_type 3,false,2024-08-02 00:00:00,user-settings 4,false,2024-08-02 00:00:00,user-settings 5,true,2024-08-02 00:00:00,other-type - +6,false,2024-08-02 00:00:00,feedback +7,true,2024-08-02 00:00:00,feedback +8,false,2024-08-02 00:00:00,telemetry +9,true,2024-08-02 00:00:00,telemetry diff --git a/tests/fixtures/user/user_source_table_initial.csv b/tests/fixtures/user/user_source_table_initial.csv index 346efbe8..eadc1273 100644 --- a/tests/fixtures/user/user_source_table_initial.csv +++ b/tests/fixtures/user/user_source_table_initial.csv @@ -4,4 +4,7 @@ saved_timestamp,_id,_deleted,doc 2024-08-02 00:00:00,3,false,"{""type"": ""user-settings"", ""contact_id"": ""1003"", ""language"": ""es"", ""roles"": ""guest""}" 2024-08-02 00:00:00,4,false,"{""type"": ""user-settings"", ""contact_id"": ""1004"", ""language"": ""de"", ""roles"": ""user""}" 2024-08-02 00:00:00,5,true,"{""type"": ""other-type"", ""contact_id"": ""1005"", ""language"": ""it"", ""roles"": ""admin""}" - +2024-08-02 00:00:00,6,false,"{""type"": ""feedback"", ""meta"": {""source"": ""automatic"", ""url"": ""http://example.com"", ""time"": ""2024-08-02 00:00:00"", ""user"": {""name"": ""admin""}}, ""info"": {""cause"": ""bug"", ""message"": ""this is a bug""}}" +2024-08-02 00:00:00,7,true,"{""type"": ""feedback"", ""meta"": {""source"": ""automatic"", ""url"": ""http://example.com"", ""time"": ""2024-08-02 00:00:00"", ""user"": {""name"": ""admin""}}, ""info"": {""cause"": ""bug"", ""message"": ""this is a bug""}}" +2024-08-02 00:00:00,8,false,"{""type"": ""telemetry"", ""metadata"": {""day"": ""2024-09-11"", ""user"": ""admin"", ""versions"": {""app"": ""4.15.0""}}, ""metrics"": {""boot_time"": {""min"": 0.5, ""max"": 1.5, ""count"": 2}}, ""dbInfo"": {""doc_count"": 20}, ""device"": {""userAgent"": ""Chrome Mobile"", ""deviceInfo"": {""hardware"": {""manufacturer"": ""Google"", ""model"": ""Pixel 3""}, ""app"": {""version"": ""4.15.0""}, ""software"": {""androidVersion"": ""10""}, ""os"": {""version"": ""10"", ""sdkInt"": 29}, ""storage"": {""free"": 100, ""total"": 200}, ""network"": {""upSpeed"": 100, ""downSpeed"": 200}}}}" +2024-08-02 00:00:00,9,true,"{""type"": ""telemetry"", ""metadata"": {""day"": ""2024-09-12"", ""user"": ""admin2"", ""versions"": {""app"": ""4.13.0""}}, ""metrics"": {""boot_time"": {""min"": 1, ""max"": 2.5, ""count"": 2}}, ""dbInfo"": {""doc_count"": 20}, ""device"": {""userAgent"": ""Chrome Mobile"", ""deviceInfo"": {""hardware"": {""manufacturer"": ""Google"", ""model"": ""Pixel 2""}, ""app"": {""version"": ""4.13.0""}, ""software"": {""androidVersion"": ""9""}, ""os"": {""version"": ""10"", ""sdkInt"": 29}, ""storage"": {""free"": 100, ""total"": 200}, ""network"": {""upSpeed"": 100, ""downSpeed"": 200}}}}"