From 06c5d6642968784ec0113dd45c17b1ac05bf4c54 Mon Sep 17 00:00:00 2001 From: Anna Scholtz Date: Fri, 13 Dec 2024 11:55:42 -0800 Subject: [PATCH 01/29] Revert "Un-schedule Bigeye metrics that get triggered via Airflow (#6662)" (#6668) This reverts commit 6ad812eaaf043d38377a33181f1667646e3e1590. --- sql/bigconfig.yml | 30 ++--- .../ssl_ratios_v1/bigconfig.yml | 120 ------------------ 2 files changed, 15 insertions(+), 135 deletions(-) diff --git a/sql/bigconfig.yml b/sql/bigconfig.yml index 0f03621892c..dd42b3f6de3 100644 --- a/sql/bigconfig.yml +++ b/sql/bigconfig.yml @@ -8,9 +8,9 @@ saved_metric_definitions: threshold: type: CONSTANT upper_bound: 0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 + metric_schedule: + named_schedule: + name: default lookback: lookback_type: DATA_TIME lookback_window: @@ -24,9 +24,9 @@ saved_metric_definitions: threshold: type: CONSTANT upper_bound: 0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 + metric_schedule: + named_schedule: + name: default rct_overrides: - submission_date - saved_metric_id: freshness_fail @@ -71,9 +71,9 @@ saved_metric_definitions: type: CONSTANT lower_bound: 1 upper_bound: 1 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 + metric_schedule: + named_schedule: + name: default lookback: lookback_type: DATA_TIME lookback_window: @@ -88,9 +88,9 @@ saved_metric_definitions: type: CONSTANT lower_bound: 2 upper_bound: 2 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 + metric_schedule: + named_schedule: + name: default lookback: lookback_type: DATA_TIME lookback_window: @@ -105,9 +105,9 @@ saved_metric_definitions: type: CONSTANT lower_bound: 1 upper_bound: 1 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 + metric_schedule: + named_schedule: + name: default lookback: lookback_type: DATA_TIME lookback_window: diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/ssl_ratios_v1/bigconfig.yml b/sql/moz-fx-data-shared-prod/telemetry_derived/ssl_ratios_v1/bigconfig.yml index 6a8bbcce1ae..60bf0f677db 100755 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/ssl_ratios_v1/bigconfig.yml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/ssl_ratios_v1/bigconfig.yml @@ -10,18 +10,12 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 1.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - metric_type: type: PREDEFINED predefined_metric: COUNT_DUPLICATES threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.os metrics: @@ -31,18 +25,12 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 1.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - metric_type: type: PREDEFINED predefined_metric: COUNT_DUPLICATES threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.* metrics: @@ -52,9 +40,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 1.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.non_ssl_loads metrics: @@ -65,9 +50,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.ssl_loads metrics: @@ -78,9 +60,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.reporting_ratio metrics: @@ -91,9 +70,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - deployments: - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.* @@ -120,18 +96,12 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 1.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - metric_type: type: PREDEFINED predefined_metric: COUNT_DUPLICATES threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.os metrics: @@ -141,18 +111,12 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 1.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - metric_type: type: PREDEFINED predefined_metric: COUNT_DUPLICATES threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.* metrics: @@ -162,9 +126,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 1.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.non_ssl_loads metrics: @@ -175,9 +136,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.ssl_loads metrics: @@ -188,9 +146,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.reporting_ratio metrics: @@ -201,9 +156,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - deployments: - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.submission_date @@ -214,18 +166,12 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 1.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - metric_type: type: PREDEFINED predefined_metric: COUNT_DUPLICATES threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.os metrics: @@ -235,18 +181,12 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 1.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - metric_type: type: PREDEFINED predefined_metric: COUNT_DUPLICATES threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.* metrics: @@ -256,9 +196,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 1.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.non_ssl_loads metrics: @@ -269,9 +206,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.ssl_loads metrics: @@ -282,9 +216,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.reporting_ratio metrics: @@ -295,9 +226,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - deployments: - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.submission_date @@ -308,18 +236,12 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 1.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - metric_type: type: PREDEFINED predefined_metric: COUNT_DUPLICATES threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.os metrics: @@ -329,18 +251,12 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 1.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - metric_type: type: PREDEFINED predefined_metric: COUNT_DUPLICATES threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.* metrics: @@ -350,9 +266,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 1.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.non_ssl_loads metrics: @@ -363,9 +276,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.ssl_loads metrics: @@ -376,9 +286,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.reporting_ratio metrics: @@ -389,9 +296,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - deployments: - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.submission_date @@ -402,18 +306,12 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 1.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - metric_type: type: PREDEFINED predefined_metric: COUNT_DUPLICATES threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.os metrics: @@ -423,18 +321,12 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 1.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - metric_type: type: PREDEFINED predefined_metric: COUNT_DUPLICATES threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.* metrics: @@ -444,9 +336,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 1.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.non_ssl_loads metrics: @@ -457,9 +346,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.ssl_loads metrics: @@ -470,9 +356,6 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.telemetry_derived.ssl_ratios_v1.reporting_ratio metrics: @@ -483,6 +366,3 @@ tag_deployments: threshold: type: CONSTANT lower_bound: 0.0 - schedule_frequency: - interval_type: MINUTES - interval_value: 0 From 32dfb5fe46fd0e5ddf5a53a9a56e2777a100e156 Mon Sep 17 00:00:00 2001 From: Marlene Hirose <92952117+Marlene-M-Hirose@users.noreply.github.com> Date: Fri, 13 Dec 2024 12:23:02 -0800 Subject: [PATCH 02/29] added table_type:aggregate to these adjust and microsoft tables (#6669) --- .../adjust_derived/adjust_cohort_v1/metadata.yaml | 1 + .../adjust_derived/adjust_deliverables_v1/metadata.yaml | 1 + .../microsoft_derived/app_acquisitions_v1/metadata.yaml | 1 + .../microsoft_derived/app_conversions_v1/metadata.yaml | 1 + .../microsoft_derived/app_installs_v1/metadata.yaml | 1 + 5 files changed, 5 insertions(+) diff --git a/sql/moz-fx-data-shared-prod/adjust_derived/adjust_cohort_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/adjust_derived/adjust_cohort_v1/metadata.yaml index 999abc5adc5..4507a6cfae3 100644 --- a/sql/moz-fx-data-shared-prod/adjust_derived/adjust_cohort_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/adjust_derived/adjust_cohort_v1/metadata.yaml @@ -7,6 +7,7 @@ owners: - mhirose@mozilla.com labels: incremental: false + table_type: aggregate scheduling: dag_name: bqetl_adjust date_partition_parameter: date diff --git a/sql/moz-fx-data-shared-prod/adjust_derived/adjust_deliverables_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/adjust_derived/adjust_deliverables_v1/metadata.yaml index af394cfaf41..afe2829b226 100644 --- a/sql/moz-fx-data-shared-prod/adjust_derived/adjust_deliverables_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/adjust_derived/adjust_deliverables_v1/metadata.yaml @@ -7,6 +7,7 @@ owners: - mhirose@mozilla.com labels: incremental: false + table_type: aggregate scheduling: dag_name: bqetl_adjust date_partition_parameter: date diff --git a/sql/moz-fx-data-shared-prod/microsoft_derived/app_acquisitions_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/microsoft_derived/app_acquisitions_v1/metadata.yaml index af457df8e60..575fcb3fb7b 100644 --- a/sql/moz-fx-data-shared-prod/microsoft_derived/app_acquisitions_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/microsoft_derived/app_acquisitions_v1/metadata.yaml @@ -7,6 +7,7 @@ owners: labels: incremental: false owner1: mhirose@mozilla.com + table_type: aggregate bigquery: time_partitioning: type: day diff --git a/sql/moz-fx-data-shared-prod/microsoft_derived/app_conversions_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/microsoft_derived/app_conversions_v1/metadata.yaml index 3fff257d8fd..a5d9e6b2cb2 100644 --- a/sql/moz-fx-data-shared-prod/microsoft_derived/app_conversions_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/microsoft_derived/app_conversions_v1/metadata.yaml @@ -7,6 +7,7 @@ owners: labels: incremental: false owner1: mhirose@mozilla.com + table_type: aggregate bigquery: time_partitioning: type: day diff --git a/sql/moz-fx-data-shared-prod/microsoft_derived/app_installs_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/microsoft_derived/app_installs_v1/metadata.yaml index 47dc184388c..d0accde38a4 100644 --- a/sql/moz-fx-data-shared-prod/microsoft_derived/app_installs_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/microsoft_derived/app_installs_v1/metadata.yaml @@ -7,6 +7,7 @@ owners: labels: incremental: false owner1: mhirose@mozilla.com + table_type: aggregate bigquery: time_partitioning: type: day From fce7d34beedcdc4ac1bbe12c2a7ca8cef2ba8f3d Mon Sep 17 00:00:00 2001 From: Katie Windau <153020235+kwindau@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:04:40 -0600 Subject: [PATCH 03/29] Create aggregate tables for Firefox Health Indicator dashboard (#6667) * Remove comment * Create new fx_health_ind_windows_versions_mau_per_os_v1 * Fix cluster column * Fix query.sql * Fix typo * Fix query * Update metadata.yaml * Add fx_health_ind_mau_per_tier1_country_v1 * Add aggregate table label --- .../view.sql | 7 ++ .../view.sql | 7 ++ .../fx_health_ind_mau_per_os_v1/query.sql | 2 +- .../metadata.yaml | 22 +++++++ .../query.sql | 55 ++++++++++++++++ .../schema.yaml | 25 ++++++++ .../metadata.yaml | 22 +++++++ .../query.sql | 64 +++++++++++++++++++ .../schema.yaml | 25 ++++++++ 9 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_mau_per_tier1_country/view.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_windows_versions_mau_per_os/view.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_tier1_country_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_tier1_country_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_tier1_country_v1/schema.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_windows_versions_mau_per_os_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_windows_versions_mau_per_os_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_windows_versions_mau_per_os_v1/schema.yaml diff --git a/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_mau_per_tier1_country/view.sql b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_mau_per_tier1_country/view.sql new file mode 100644 index 00000000000..c012e328770 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_mau_per_tier1_country/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.telemetry.fx_health_ind_mau_per_tier1_country` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.telemetry_derived.fx_health_ind_mau_per_tier1_country_v1` diff --git a/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_windows_versions_mau_per_os/view.sql b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_windows_versions_mau_per_os/view.sql new file mode 100644 index 00000000000..cca6201f76e --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_windows_versions_mau_per_os/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.telemetry.fx_health_ind_windows_versions_mau_per_os` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.telemetry_derived.fx_health_ind_windows_versions_mau_per_os_v1` diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_os_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_os_v1/query.sql index b07861713ae..afbda7f174f 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_os_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_os_v1/query.sql @@ -5,7 +5,7 @@ WITH sample_cte AS ( SUM(dau) AS tot_dau, SUM(mau) AS tot_mau FROM - `moz-fx-data-shared-prod.telemetry.active_users_aggregates` --telemetry.firefox_desktop_exact_mau28_by_client_count_dimensions + `moz-fx-data-shared-prod.telemetry.active_users_aggregates` WHERE app_name = 'Firefox Desktop' AND submission_date diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_tier1_country_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_tier1_country_v1/metadata.yaml new file mode 100644 index 00000000000..d8389b21316 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_tier1_country_v1/metadata.yaml @@ -0,0 +1,22 @@ +friendly_name: Firefox Health Indicators MAU Per Tier1 Country +description: |- + Aggregate of MAU per tier 1 country used in dashboard +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau@mozilla.com + table_type: aggregate +scheduling: + dag_name: bqetl_fx_health_ind_dashboard +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - country +references: {} diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_tier1_country_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_tier1_country_v1/query.sql new file mode 100644 index 00000000000..252df69a4ac --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_tier1_country_v1/query.sql @@ -0,0 +1,55 @@ +WITH sample_cte AS ( + SELECT + submission_date, + country, + SUM(dau) AS tot_dau, + SUM(mau) AS tot_mau, + FROM + `mozdata.telemetry.active_users_aggregates` + WHERE + app_name = 'Firefox Desktop' + AND submission_date + BETWEEN DATE_SUB(@submission_date, INTERVAL 6 DAY) + AND @submission_date + AND country IN ('US', 'DE', 'FR', 'CA', 'GB', 'CN', 'ID', 'IN', 'RU', 'PL', 'BR', 'ES') + GROUP BY + submission_date, + country + HAVING + SUM(mau) > 1000 +), +smoothed AS ( + SELECT + *, + AVG(tot_dau) OVER ( + PARTITION BY + country + ORDER BY + submission_date + ROWS BETWEEN + 6 PRECEDING + AND 0 FOLLOWING + ) AS smoothed_dau, + COUNT(1) OVER ( + PARTITION BY + country + ORDER BY + submission_date + ROWS BETWEEN + 6 PRECEDING + AND 0 FOLLOWING + ) AS nbr_days_included + FROM + sample_cte +) +SELECT + submission_date, + country, + tot_mau AS mau, + tot_dau AS dau, + smoothed_dau, + smoothed_dau / tot_mau AS ER +FROM + smoothed +WHERE + nbr_days_included = 7 --only include those with at least 1000 MAU on all 7 days diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_tier1_country_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_tier1_country_v1/schema.yaml new file mode 100644 index 00000000000..6b405c058da --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_tier1_country_v1/schema.yaml @@ -0,0 +1,25 @@ +fields: +- mode: NULLABLE + name: submission_date + type: DATE + description: Submission Date +- mode: NULLABLE + name: country + type: STRING + description: Country +- mode: NULLABLE + name: mau + type: INTEGER + description: MAU +- mode: NULLABLE + name: dau + type: INTEGER + description: DAU +- mode: NULLABLE + name: smoothed_dau + type: FLOAT + description: Smoothed DAU +- mode: NULLABLE + name: ER + type: FLOAT + description: ER - Smoothed DAU Divided by MAU diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_windows_versions_mau_per_os_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_windows_versions_mau_per_os_v1/metadata.yaml new file mode 100644 index 00000000000..10f9deef562 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_windows_versions_mau_per_os_v1/metadata.yaml @@ -0,0 +1,22 @@ +friendly_name: Fx Health Ind Windows Versions Mau Per Os +description: |- + Aggregate table that calculates MAU, DAU and ER for different Windows versions +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau@mozilla.com + table_type: aggregate +scheduling: + dag_name: bqetl_fx_health_ind_dashboard +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - windows_os_version +references: {} diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_windows_versions_mau_per_os_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_windows_versions_mau_per_os_v1/query.sql new file mode 100644 index 00000000000..77ff01bb427 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_windows_versions_mau_per_os_v1/query.sql @@ -0,0 +1,64 @@ +WITH sample_cte AS ( + SELECT + submission_date, + CASE + WHEN os_version IN ('Windows 10', 'Windows 11', "10.0") + THEN 'Win10 or Win11' + WHEN LOWER(os_version) LIKE "windows%" + THEN os_version + ELSE COALESCE(`mozfun.norm.windows_version_info`(os, os_version, NULL), "Unknown") + END AS os_version, + SUM(dau) AS tot_dau, + SUM(mau) AS tot_mau + FROM + `moz-fx-data-shared-prod.telemetry.active_users_aggregates` + WHERE + app_name = 'Firefox Desktop' + AND submission_date + BETWEEN DATE_SUB(@submission_date, INTERVAL 6 DAY) + AND @submission_date + AND LOWER(os) LIKE '%windows%' + AND os_version_major + (os_version_minor / 100) > 6 --filter Windows 7+ + GROUP BY + submission_date, + os_version + HAVING + SUM(mau) > 1000 +), +smoothed AS ( + SELECT + *, + AVG(tot_dau) OVER ( + PARTITION BY + os_version + ORDER BY + submission_date + ROWS BETWEEN + 6 PRECEDING + AND 0 FOLLOWING + ) AS smoothed_dau, + COUNT(1) OVER ( + PARTITION BY + os_version + ORDER BY + submission_date + ROWS BETWEEN + 6 PRECEDING + AND 0 FOLLOWING + ) AS nbr_days_included + FROM + sample_cte + WHERE + os_version <> "Unknown" +) +SELECT + submission_date, + os_version AS windows_os_version, + tot_mau AS mau, + tot_dau AS dau, + smoothed_dau, + smoothed_dau / tot_mau AS ER +FROM + smoothed +WHERE + nbr_days_included = 7 --only include those versions that have at least 1000 MAU on all 7 days diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_windows_versions_mau_per_os_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_windows_versions_mau_per_os_v1/schema.yaml new file mode 100644 index 00000000000..890991b96cf --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_windows_versions_mau_per_os_v1/schema.yaml @@ -0,0 +1,25 @@ +fields: +- mode: NULLABLE + name: submission_date + type: DATE + description: Submission Date +- mode: NULLABLE + name: windows_os_version + type: STRING + description: Windows Operating System Version +- mode: NULLABLE + name: mau + type: INTEGER + description: MAU +- mode: NULLABLE + name: dau + type: INTEGER + description: DAU +- mode: NULLABLE + name: smoothed_dau + type: FLOAT + description: Smoothed DAU +- mode: NULLABLE + name: ER + type: FLOAT + description: ER - Smoothed DAU Divided by MAU From 75e7b09620677de5f2ac15c173a706651b43b68c Mon Sep 17 00:00:00 2001 From: akkomar Date: Fri, 13 Dec 2024 22:15:38 +0100 Subject: [PATCH 04/29] Add aggregate label to funnel template (#6670) --- sql_generators/funnels/templates/metadata.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/sql_generators/funnels/templates/metadata.yaml b/sql_generators/funnels/templates/metadata.yaml index d582ff535ae..fdbdddfb9ee 100644 --- a/sql_generators/funnels/templates/metadata.yaml +++ b/sql_generators/funnels/templates/metadata.yaml @@ -4,6 +4,7 @@ owners: {{ owners }} {% endif %} labels: incremental: true + table_type: aggregate scheduling: dag_name: bqetl_generated_funnels From d5542e8f4962c390685824be078b0ade31be98fb Mon Sep 17 00:00:00 2001 From: Anna Scholtz Date: Fri, 13 Dec 2024 13:27:09 -0800 Subject: [PATCH 05/29] Fix newtab_desktop_daily_engagement_by_tile_id_position_v1 schema (#6671) --- .../schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_desktop_daily_engagement_by_tile_id_position_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_desktop_daily_engagement_by_tile_id_position_v1/schema.yaml index ff57a08e195..e1b6592f944 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_desktop_daily_engagement_by_tile_id_position_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_desktop_daily_engagement_by_tile_id_position_v1/schema.yaml @@ -1,6 +1,6 @@ fields: - name: submission_date - type: date + type: DATE mode: NULLABLE description: Client-side date in Firefox for when the indicated engagement happened. - name: recommendation_id From 01fc7a44827eb169a65ad440660f0929362fdaa4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 08:28:41 -0800 Subject: [PATCH 06/29] chore(deps): bump pytest-black from 0.3.12 to 0.6.0 (#6675) Bumps [pytest-black](https://github.com/coherent-oss/pytest-black) from 0.3.12 to 0.6.0. - [Commits](https://github.com/coherent-oss/pytest-black/compare/0.3.12...v0.6.0) --- updated-dependencies: - dependency-name: pytest-black dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.in | 2 +- requirements.txt | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements.in b/requirements.in index 39f488be06c..6b080d3d44d 100644 --- a/requirements.in +++ b/requirements.in @@ -29,7 +29,7 @@ pathos==0.3.3 pip-tools==7.4.1 pre-commit==4.0.1 pyarrow==18.1.0 -pytest-black==0.3.12 +pytest-black==0.6.0 pytest-flake8==1.3.0 pytest-isort==4.0.0 pytest-mypy==0.10.3 diff --git a/requirements.txt b/requirements.txt index 352e0066bf0..6d756f4e1fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1610,8 +1610,9 @@ pytest==7.4.3 \ # pytest-mypy # pytest-pydocstyle # pytest-xdist -pytest-black==0.3.12 \ - --hash=sha256:1d339b004f764d6cd0f06e690f6dd748df3d62e6fe1a692d6a5500ac2c5b75a5 +pytest-black==0.6.0 \ + --hash=sha256:7eb747f54b6c997497b5cbc66a988be114b92016dbfa66d210d1d1f9f6b2dc76 \ + --hash=sha256:ecb77455f379805cb4bd8f45a813a3754c3bbee3199adf1b3665c0dfd086b511 # via -r requirements.in pytest-flake8==1.3.0 \ --hash=sha256:88fb35562ce32d915c6ba41ef0d5e1cfcdd8ff884a32b7d46aa99fc77a3d1fe6 \ From 31744dc655ccde5ffd5055176cef7787f5572ab5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 08:41:11 -0800 Subject: [PATCH 07/29] chore(deps): bump attrs from 24.2.0 to 24.3.0 (#6674) Bumps [attrs](https://github.com/sponsors/hynek) from 24.2.0 to 24.3.0. - [Commits](https://github.com/sponsors/hynek/commits) --- updated-dependencies: - dependency-name: attrs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.in | 2 +- requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.in b/requirements.in index 6b080d3d44d..ed7c6073068 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,4 @@ -attrs==24.2.0 +attrs==24.3.0 authlib==1.3.2 bigeye-sdk==0.4.93 black==24.10.0 diff --git a/requirements.txt b/requirements.txt index 6d756f4e1fa..75e1e194070 100644 --- a/requirements.txt +++ b/requirements.txt @@ -109,9 +109,9 @@ annotated-types==0.7.0 \ --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 # via pydantic -attrs==24.2.0 \ - --hash=sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346 \ - --hash=sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2 +attrs==24.3.0 \ + --hash=sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff \ + --hash=sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308 # via # -r requirements.in # aiohttp From 8c20e68f4ada7494c0df61ef847a9b2aea999541 Mon Sep 17 00:00:00 2001 From: Katie Windau <153020235+kwindau@users.noreply.github.com> Date: Mon, 16 Dec 2024 11:16:53 -0600 Subject: [PATCH 08/29] Create clients daily aggregate for Firefox Health Indicators dashboard (#6672) * Initial commit, not done yet * Initial commit * Fix SQL formatting * Fix schema.yaml * Update table description * Fix typo in table description * Fix schema.yaml data type --- .../view.sql | 7 ++ .../metadata.yaml | 22 +++++ .../query.sql | 94 +++++++++++++++++++ .../schema.yaml | 21 +++++ 4 files changed, 144 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_clients_daily_by_os/view.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/schema.yaml diff --git a/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_clients_daily_by_os/view.sql b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_clients_daily_by_os/view.sql new file mode 100644 index 00000000000..9cbe6afd0a6 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_clients_daily_by_os/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.telemetry.fx_health_ind_clients_daily_by_os` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.telemetry_derived.fx_health_ind_clients_daily_by_os_v1` diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/metadata.yaml new file mode 100644 index 00000000000..079d6843d73 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/metadata.yaml @@ -0,0 +1,22 @@ +friendly_name: Firefox Health Indicator Clients Daily OS Aggregates +description: |- + Calculates active hrs, subsession hrs, and searches per user on a 1% client sample +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau@mozilla.com + table_type: aggregate +scheduling: + dag_name: bqetl_fx_health_ind_dashboard +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - os +references: {} diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/query.sql new file mode 100644 index 00000000000..a6058d0e40b --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/query.sql @@ -0,0 +1,94 @@ +WITH searches_per_user_by_os_and_date_staging AS ( + SELECT + submission_date_s3, + os, + SUM(search_count_all) AS searches, + COUNT(DISTINCT client_id) AS users, + FROM + `moz-fx-data-shared-prod.telemetry.clients_daily` + WHERE + submission_date_s3 = @submission_date + AND app_name = 'Firefox' + AND sample_id = 42 + AND search_count_all < 10000 + AND os IN ('Windows_NT', 'Darwin', 'Linux') + GROUP BY + submission_date_s3, + os +), +searches_per_user_by_os_and_date AS ( + SELECT + submission_date_s3, + os, + searches / users AS searches_per_user_ratio, + FROM + searches_per_user_by_os_and_date_staging +), +subsession_hours_per_user_staging AS ( + SELECT + submission_date_s3, + os, + SUM(subsession_hours_sum) AS `hours`, + COUNT(DISTINCT client_id) AS users, + FROM + `moz-fx-data-shared-prod.telemetry.clients_daily` + WHERE + submission_date_s3 = @submission_date + AND app_name = 'Firefox' + AND sample_id = 42 + AND subsession_hours_sum < 24 + AND os IN ('Windows_NT', 'Darwin', 'Linux') + GROUP BY + submission_date_s3, + os +), +subsession_hours_per_user AS ( + SELECT + submission_date_s3, + os, + `hours` / users AS subsession_hours_per_user_ratio + FROM + subsession_hours_per_user_staging +), +active_hours_per_user_staging AS ( + SELECT + submission_date_s3, + os, + SUM(ROUND(active_hours_sum)) AS `hours`, + COUNT(DISTINCT(client_id)) AS users, + FROM + `moz-fx-data-shared-prod.telemetry.clients_daily` + WHERE + submission_date_s3 = @submission_date + AND app_name = 'Firefox' + AND sample_id = 42 + AND os IN ('Windows_NT', 'Darwin', 'Linux') + GROUP BY + submission_date_s3, + os +), +active_hours_per_user AS ( + SELECT + submission_date_s3, + os, + `hours` / users AS active_hours_per_user_ratio + FROM + active_hours_per_user_staging +) +SELECT + COALESCE( + COALESCE(spu.submission_date_s3, sshpu.submission_date_s3), + ahpu.submission_date_s3 + ) AS submission_date, + COALESCE(COALESCE(spu.os, sshpu.os), ahpu.os) AS os, + spu.searches_per_user_ratio, + sshpu.subsession_hours_per_user_ratio, + ahpu.active_hours_per_user_ratio +FROM + searches_per_user_by_os_and_date AS spu +FULL OUTER JOIN + subsession_hours_per_user AS sshpu + ON spu.os = sshpu.os +FULL OUTER JOIN + active_hours_per_user AS ahpu + ON COALESCE(spu.os, sshpu.os) = ahpu.os diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/schema.yaml new file mode 100644 index 00000000000..01d518fa404 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/schema.yaml @@ -0,0 +1,21 @@ +fields: +- mode: NULLABLE + name: submission_date + type: DATE + description: Submission Date +- mode: NULLABLE + name: os + type: STRING + description: Operating System +- mode: NULLABLE + name: searches_per_user_ratio + type: FLOAT + description: Ratio of Searches per User +- mode: NULLABLE + name: subsession_hours_per_user_ratio + type: NUMERIC + description: Ratio of Subsession Hours per User +- mode: NULLABLE + name: active_hours_per_user_ratio + type: FLOAT + description: Ratio of Active Hours per User From ccf196389f01505f5aa05eaefaf94380fddaf86a Mon Sep 17 00:00:00 2001 From: Katie Windau <153020235+kwindau@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:12:42 -0600 Subject: [PATCH 09/29] Create new aggregate table for Firefox Health Indicator dashboard (#6676) * Create new table fx_health_ind_clients_daily_by_os_version * Update table description * Fix SQL formatting * Remove old comment * Remove partition requirement * Update metadata.yaml --- .../view.sql | 7 ++ .../metadata.yaml | 22 +++++ .../query.sql | 97 +++++++++++++++++++ .../schema.yaml | 21 ++++ 4 files changed, 147 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_clients_daily_by_os_version/view.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/schema.yaml diff --git a/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_clients_daily_by_os_version/view.sql b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_clients_daily_by_os_version/view.sql new file mode 100644 index 00000000000..74c82d557cd --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_clients_daily_by_os_version/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.telemetry.fx_health_ind_clients_daily_by_os_version` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.telemetry_derived.fx_health_ind_clients_daily_by_os_version_v1` diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/metadata.yaml new file mode 100644 index 00000000000..1106a646c1d --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/metadata.yaml @@ -0,0 +1,22 @@ +friendly_name: Firefox Health Indicator Clients Daily By OS Version +description: |- + Calculates active hrs, subsession hrs, & searches per user on a 1% client sample by OS version for Windows NT +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau@mozilla.com + table_type: aggregate +scheduling: + dag_name: bqetl_fx_health_ind_dashboard +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - os_version +references: {} diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/query.sql new file mode 100644 index 00000000000..9f9168e0514 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/query.sql @@ -0,0 +1,97 @@ +WITH searches_per_user_by_os_version_and_date_staging AS ( + SELECT + submission_date_s3, + os_version, + SUM(search_count_all) AS searches, + COUNT(DISTINCT client_id) AS users, + FROM + `moz-fx-data-shared-prod.telemetry.clients_daily` + WHERE + submission_date_s3 = @submission_date + AND app_name = 'Firefox' + AND sample_id = 42 + AND search_count_all < 10000 + AND os = 'Windows_NT' + AND os_version IN ('6.1', '6.2', '6.3', '10.0') + GROUP BY + submission_date_s3, + os_version +), +searches_per_user_by_os_and_date AS ( + SELECT + submission_date_s3, + os_version, + searches / users AS searches_per_user_ratio, + FROM + searches_per_user_by_os_version_and_date_staging +), +subsession_hours_per_user_staging AS ( + SELECT + submission_date_s3, + os_version, + SUM(subsession_hours_sum) AS `hours`, + COUNT(DISTINCT client_id) AS users, + FROM + `moz-fx-data-shared-prod.telemetry.clients_daily` + WHERE + submission_date_s3 = @submission_date + AND app_name = 'Firefox' + AND sample_id = 42 + AND subsession_hours_sum < 24 + AND os = 'Windows_NT' + AND os_version IN ('6.1', '6.2', '6.3', '10.0') + GROUP BY + submission_date_s3, + os_version +), +subsession_hours_per_user AS ( + SELECT + submission_date_s3, + os_version, + `hours` / users AS subsession_hours_per_user_ratio + FROM + subsession_hours_per_user_staging +), +active_hours_per_user_staging AS ( + SELECT + submission_date_s3, + os_version, + SUM(active_hours_sum) AS `hours`, + COUNT(DISTINCT(client_id)) AS users, + FROM + `moz-fx-data-shared-prod.telemetry.clients_daily` + WHERE + submission_date_s3 = @submission_date + AND app_name = 'Firefox' + AND sample_id = 42 + AND os = 'Windows_NT' + AND os_version IN ('6.1', '6.2', '6.3', '10.0') + GROUP BY + submission_date_s3, + os_version +), +active_hours_per_user AS ( + SELECT + submission_date_s3, + os_version, + `hours` / users AS active_hours_per_user_ratio + FROM + active_hours_per_user_staging +) +SELECT + COALESCE( + COALESCE(spu.submission_date_s3, sshpu.submission_date_s3), + ahpu.submission_date_s3 + ) AS submission_date, + COALESCE(COALESCE(spu.os_version, sshpu.os_version), ahpu.os_version) AS os_version, + spu.searches_per_user_ratio, + sshpu.subsession_hours_per_user_ratio, + ahpu.active_hours_per_user_ratio +FROM + searches_per_user_by_os_and_date AS spu +FULL OUTER JOIN + subsession_hours_per_user AS sshpu + ON spu.os_version = sshpu.os_version +FULL OUTER JOIN + active_hours_per_user AS ahpu + ON COALESCE(spu.os_version, sshpu.os_version) = ahpu.os_version diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/schema.yaml new file mode 100644 index 00000000000..06f3a69f3e7 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/schema.yaml @@ -0,0 +1,21 @@ +fields: +- mode: NULLABLE + name: submission_date + type: DATE + description: Submission Date +- mode: NULLABLE + name: os_version + type: STRING + description: Operating System Version +- mode: NULLABLE + name: searches_per_user_ratio + type: FLOAT + description: Ratio of Searches per User +- mode: NULLABLE + name: subsession_hours_per_user_ratio + type: NUMERIC + description: Ratio of Subsession Hours per User +- mode: NULLABLE + name: active_hours_per_user_ratio + type: FLOAT + description: Ratio of Active Hours per User From cfd1a670241e5c2a60bed30cecda02ef77775640 Mon Sep 17 00:00:00 2001 From: Katie Windau <153020235+kwindau@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:31:35 -0600 Subject: [PATCH 10/29] Create new aggregate table for Firefox Health Indicators dashboard (#6678) * Create new aggregate table for Firefox Health Indicators dashboard * Add aggregate table label and update table description * Do not require partition filter --- .../view.sql | 7 ++ .../metadata.yaml | 22 +++++ .../query.sql | 92 +++++++++++++++++++ .../schema.yaml | 21 +++++ 4 files changed, 142 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_clients_daily_by_country/view.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/schema.yaml diff --git a/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_clients_daily_by_country/view.sql b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_clients_daily_by_country/view.sql new file mode 100644 index 00000000000..007790993d2 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_clients_daily_by_country/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.telemetry.fx_health_ind_clients_daily_by_country` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.telemetry_derived.fx_health_ind_clients_daily_by_country_v1` diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/metadata.yaml new file mode 100644 index 00000000000..5509e6d6032 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/metadata.yaml @@ -0,0 +1,22 @@ +friendly_name: Fx Health Ind Clients Daily By Country +description: |- + Calculates active hrs, subsession hrs, and searches per user by country on a 1% client sample +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau@mozilla.com + table_type: aggregate +scheduling: + dag_name: bqetl_fx_health_ind_dashboard +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - country +references: {} diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/query.sql new file mode 100644 index 00000000000..a9034486c38 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/query.sql @@ -0,0 +1,92 @@ +WITH searches_per_user_by_country_and_date_staging AS ( + SELECT + submission_date_s3, + country, + SUM(search_count_all) AS searches, + COUNT(DISTINCT client_id) AS users, + FROM + `moz-fx-data-shared-prod.telemetry.clients_daily` + WHERE + submission_date_s3 = @submission_date + AND app_name = 'Firefox' + AND sample_id = 42 + AND search_count_all < 10000 + GROUP BY + submission_date_s3, + country +), +searches_per_user_by_country_and_date AS ( + SELECT + submission_date_s3, + country, + searches / users AS searches_per_user_ratio, + FROM + searches_per_user_by_country_and_date_staging +), +subsession_hours_per_user_staging AS ( + SELECT + submission_date_s3, + country, + SUM(subsession_hours_sum) AS `hours`, + COUNT(DISTINCT client_id) AS users, + FROM + `moz-fx-data-shared-prod.telemetry.clients_daily` + WHERE + submission_date_s3 = @submission_date + AND app_name = 'Firefox' + AND sample_id = 42 + AND subsession_hours_sum < 24 + GROUP BY + submission_date_s3, + country +), +subsession_hours_per_user AS ( + SELECT + submission_date_s3, + country, + `hours` / users AS subsession_hours_per_user_ratio + FROM + subsession_hours_per_user_staging +), +active_hours_per_user_staging AS ( + SELECT + submission_date_s3, + country, + SUM(active_hours_sum) AS `hours`, + COUNT(DISTINCT(client_id)) AS users, + FROM + `moz-fx-data-shared-prod.telemetry.clients_daily` + WHERE + submission_date_s3 = @submission_date + AND app_name = 'Firefox' + AND sample_id = 42 + AND active_hours_sum < 24 + GROUP BY + submission_date_s3, + country +), +active_hours_per_user AS ( + SELECT + submission_date_s3, + country, + `hours` / users AS active_hours_per_user_ratio + FROM + active_hours_per_user_staging +) +SELECT + COALESCE( + COALESCE(spu.submission_date_s3, sshpu.submission_date_s3), + ahpu.submission_date_s3 + ) AS submission_date, + COALESCE(COALESCE(spu.country, sshpu.country), ahpu.country) AS country, + spu.searches_per_user_ratio, + sshpu.subsession_hours_per_user_ratio, + ahpu.active_hours_per_user_ratio +FROM + searches_per_user_by_country_and_date AS spu +FULL OUTER JOIN + subsession_hours_per_user AS sshpu + ON spu.country = sshpu.country +FULL OUTER JOIN + active_hours_per_user AS ahpu + ON COALESCE(spu.country, sshpu.country) = ahpu.country diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/schema.yaml new file mode 100644 index 00000000000..a729a1695db --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/schema.yaml @@ -0,0 +1,21 @@ +fields: +- mode: NULLABLE + name: submission_date + type: DATE + description: Submission Date +- mode: NULLABLE + name: country + type: STRING + description: Country +- mode: NULLABLE + name: searches_per_user_ratio + type: FLOAT + description: Ratio of Searches per User +- mode: NULLABLE + name: subsession_hours_per_user_ratio + type: NUMERIC + description: Ratio of Subsession Hours per User +- mode: NULLABLE + name: active_hours_per_user_ratio + type: FLOAT + description: Ratio of Active Hours per User From 86f0dc21103fb1cff26690f136281c3f1171ee4b Mon Sep 17 00:00:00 2001 From: Eduardo Filho Date: Tue, 17 Dec 2024 08:54:36 -0500 Subject: [PATCH 11/29] fix(glam): update beta and nightly refresh aggregates upstream task_id (#6677) * fix(glam): update beta and nightly refresh aggregates upstream task_id * same for fenix * update upstream task name --- .../glam_etl/glam_fenix_beta_aggregates_v1/metadata.yaml | 2 +- .../glam_etl/glam_fenix_nightly_aggregates_v1/metadata.yaml | 2 +- .../glam_etl/glam_fenix_release_aggregates_v1/metadata.yaml | 2 +- .../glam_etl/glam_fog_beta_aggregates_v1/metadata.yaml | 2 +- .../glam_etl/glam_fog_nightly_aggregates_v1/metadata.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_beta_aggregates_v1/metadata.yaml b/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_beta_aggregates_v1/metadata.yaml index acda357ba2f..bdc43fabe71 100644 --- a/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_beta_aggregates_v1/metadata.yaml +++ b/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_beta_aggregates_v1/metadata.yaml @@ -10,6 +10,6 @@ scheduling: dag_name: bqetl_glam_refresh_aggregates date_partition_parameter: null depends_on: - - task_id: export_org_mozilla_fenix_glam_beta + - task_id: query_org_mozilla_fenix_glam_beta__extract_probe_counts_v1 dag_name: glam_fenix execution_delta: 6h diff --git a/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_nightly_aggregates_v1/metadata.yaml b/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_nightly_aggregates_v1/metadata.yaml index 4c987b2df88..6e65df15687 100644 --- a/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_nightly_aggregates_v1/metadata.yaml +++ b/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_nightly_aggregates_v1/metadata.yaml @@ -10,6 +10,6 @@ scheduling: dag_name: bqetl_glam_refresh_aggregates date_partition_parameter: null depends_on: - - task_id: export_org_mozilla_fenix_glam_nightly + - task_id: query_org_mozilla_fenix_glam_nightly__extract_probe_counts_v1 dag_name: glam_fenix execution_delta: 6h diff --git a/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_release_aggregates_v1/metadata.yaml b/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_release_aggregates_v1/metadata.yaml index 0d65c10f773..ae240d4b46e 100644 --- a/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_release_aggregates_v1/metadata.yaml +++ b/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_release_aggregates_v1/metadata.yaml @@ -10,6 +10,6 @@ scheduling: dag_name: bqetl_glam_refresh_aggregates date_partition_parameter: null depends_on: - - task_id: export_org_mozilla_fenix_glam_release + - task_id: query_org_mozilla_fenix_glam_release__extract_probe_counts_v1 dag_name: glam_fenix execution_delta: 6h diff --git a/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fog_beta_aggregates_v1/metadata.yaml b/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fog_beta_aggregates_v1/metadata.yaml index 4bb8e1693a6..6c4afa7c47f 100644 --- a/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fog_beta_aggregates_v1/metadata.yaml +++ b/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fog_beta_aggregates_v1/metadata.yaml @@ -10,6 +10,6 @@ scheduling: dag_name: bqetl_glam_refresh_aggregates date_partition_parameter: null depends_on: - - task_id: export_firefox_desktop_glam_beta + - task_id: firefox_desktop_glam_beta_done dag_name: glam_fog execution_delta: 6h diff --git a/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fog_nightly_aggregates_v1/metadata.yaml b/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fog_nightly_aggregates_v1/metadata.yaml index 5c0ab5d0a6a..ab84eae8173 100644 --- a/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fog_nightly_aggregates_v1/metadata.yaml +++ b/sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fog_nightly_aggregates_v1/metadata.yaml @@ -10,6 +10,6 @@ scheduling: dag_name: bqetl_glam_refresh_aggregates date_partition_parameter: null depends_on: - - task_id: export_firefox_desktop_glam_nightly + - task_id: firefox_desktop_glam_nightly_done dag_name: glam_fog execution_delta: 6h From bf97a38684853c6777cac1b9d1b6f5cefd532d7b Mon Sep 17 00:00:00 2001 From: kik-kik <42538694+kik-kik@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:50:18 +0100 Subject: [PATCH 12/29] feat: introduce bigconfig to android_app_campaign_stats_v1 and ios_app_campaign_stats_v1 (#6683) --- .../ios_app_campaign_stats_v1/bigconfig.yml | 55 +++++++++++++---- .../ios_app_campaign_stats_v1/metadata.yaml | 2 + .../bigconfig.yml | 61 +++++++++++++++++++ .../metadata.yaml | 2 + 4 files changed, 107 insertions(+), 13 deletions(-) create mode 100644 sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/bigconfig.yml diff --git a/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/bigconfig.yml b/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/bigconfig.yml index 79bc10d5c9f..8f5a019c27f 100644 --- a/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/bigconfig.yml +++ b/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/bigconfig.yml @@ -2,32 +2,61 @@ type: BIGCONFIG_FILE tag_deployments: - collection: - name: Growth Program + name: Operational Checks notification_channels: - - slack: '#ds-bigeye-triage' + - slack: '#de-bigeye-triage' deployments: - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.date - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign_id - - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign_country_code - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.ad_group_name - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.ad_group_id + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.impressions + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.clicks + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.downloads + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.new_profiles + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.activated_profiles + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.repeat_users + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.week_4_retained_users + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.spend + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.lifetime_value metrics: - saved_metric_id: is_not_null + lookback: + lookback_type: DATA_TIME + lookback_window: + interval_type: DAYS + interval_value: 28 + rct_overrides: + - date - column_selectors: - - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.* + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign_country_code metrics: - - saved_metric_id: freshness - - saved_metric_id: volume - - - collection: - name: Operational Checks - notification_channels: - - slack: '#de-bigeye-triage' - deployments: + - saved_metric_id: is_2_char_len + lookback: + lookback_type: DATA_TIME + lookback_window: + interval_type: DAYS + interval_value: 28 + rct_overrides: + - date - column_selectors: - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.* metrics: - - saved_metric_id: freshness - saved_metric_id: volume + lookback: + lookback_type: DATA_TIME + lookback_window: + interval_type: DAYS + interval_value: 28 + rct_overrides: + - date + - saved_metric_id: freshness + lookback: + lookback_type: DATA_TIME + lookback_window: + interval_type: DAYS + interval_value: 28 + rct_overrides: + - date diff --git a/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/metadata.yaml index 73798114d57..83eb94e00c4 100644 --- a/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/metadata.yaml @@ -23,3 +23,5 @@ bigquery: clustering: fields: - campaign +monitoring: + enabled: true diff --git a/sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/bigconfig.yml b/sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/bigconfig.yml new file mode 100644 index 00000000000..ab05a91c60e --- /dev/null +++ b/sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/bigconfig.yml @@ -0,0 +1,61 @@ +type: BIGCONFIG_FILE + +tag_deployments: + - collection: + name: Operational Checks + notification_channels: + - slack: '#de-bigeye-triage' + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_region + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_country_code + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_language + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.ad_group + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.impressions + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.clicks + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.new_profiles + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.activated_profiles + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.repeat_users + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.week_4_retained_users + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.spend + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.lifetime_value + metrics: + - saved_metric_id: is_not_null + lookback: + lookback_type: DATA_TIME + lookback_window: + interval_type: DAYS + interval_value: 28 + rct_overrides: + - date + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_country_code + metrics: + - saved_metric_id: is_2_char_len + lookback: + lookback_type: DATA_TIME + lookback_window: + interval_type: DAYS + interval_value: 28 + rct_overrides: + - date + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.* + metrics: + - saved_metric_id: volume + lookback: + lookback_type: DATA_TIME + lookback_window: + interval_type: DAYS + interval_value: 28 + rct_overrides: + - date + - saved_metric_id: freshness + lookback: + lookback_type: DATA_TIME + lookback_window: + interval_type: DAYS + interval_value: 28 + rct_overrides: + - date diff --git a/sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/metadata.yaml index d644f5a34d1..c0cdb60d3b5 100644 --- a/sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/metadata.yaml @@ -24,3 +24,5 @@ bigquery: type: day field: date require_partition_filter: false +monitoring: + enabled: true From 07c0a43036f59b9c131c8323c4f23e72cc016578 Mon Sep 17 00:00:00 2001 From: Katie Windau <153020235+kwindau@users.noreply.github.com> Date: Tue, 17 Dec 2024 09:55:01 -0600 Subject: [PATCH 13/29] Add additional columns to new fx_health_indicator aggregate tables (#6679) * Add new column * Add default_percent column to OS aggregate table * Add new column to OS aggregate table * Add average profile age column * work in progress * Add % age breakouts --- .../query.sql | 36 +++++++++++++++++- .../schema.yaml | 20 ++++++++++ .../query.sql | 37 +++++++++++++++++- .../schema.yaml | 20 ++++++++++ .../query.sql | 38 ++++++++++++++++++- .../schema.yaml | 20 ++++++++++ 6 files changed, 168 insertions(+), 3 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/query.sql index a9034486c38..ee171be29cd 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/query.sql @@ -72,6 +72,32 @@ active_hours_per_user AS ( `hours` / users AS active_hours_per_user_ratio FROM active_hours_per_user_staging +), +default_percent_and_avg_age_by_country AS ( + SELECT + submission_date_s3, + country, + ROUND(AVG(IF(is_default_browser, 1, 0)) * 100, 1) AS default_percent, + SUM(ROUND(profile_age_in_days)) / COUNT(DISTINCT(client_id)) AS average_profile_age, + COUNT(DISTINCT(client_id)) AS total_nbr_users, + COUNT( + DISTINCT(CASE WHEN profile_age_in_days <= 7 THEN client_id ELSE NULL END) + ) AS nbr_users_profile_age_less_than_7, + COUNT( + DISTINCT(CASE WHEN profile_age_in_days BETWEEN 8 AND 365 THEN client_id ELSE NULL END) + ) AS nbr_users_profile_age_between_8_and_365, + COUNT( + DISTINCT(CASE WHEN profile_age_in_days > 365 THEN client_id ELSE NULL END) + ) AS nbr_users_profile_age_over_365 + FROM + `moz-fx-data-shared-prod.telemetry.clients_daily` + WHERE + submission_date_s3 = @submission_date + AND app_name = 'Firefox' + AND sample_id = 42 + GROUP BY + submission_date_s3, + country ) SELECT COALESCE( @@ -81,7 +107,12 @@ SELECT COALESCE(COALESCE(spu.country, sshpu.country), ahpu.country) AS country, spu.searches_per_user_ratio, sshpu.subsession_hours_per_user_ratio, - ahpu.active_hours_per_user_ratio + ahpu.active_hours_per_user_ratio, + dflt.default_percent, + dflt.average_profile_age, + dflt.nbr_users_profile_age_less_than_7 / dflt.total_nbr_users AS pct_users_profile_age_less_than_7_days, + dflt.nbr_users_profile_age_between_8_and_365 / dflt.total_nbr_users AS pct_users_profile_age_8_to_365_days, + dflt.nbr_users_profile_age_over_365 / dflt.total_nbr_users AS pct_users_profile_age_over_365_days FROM searches_per_user_by_country_and_date AS spu FULL OUTER JOIN @@ -90,3 +121,6 @@ FULL OUTER JOIN FULL OUTER JOIN active_hours_per_user AS ahpu ON COALESCE(spu.country, sshpu.country) = ahpu.country +FULL OUTER JOIN + default_percent_and_avg_age_by_country AS dflt + ON COALESCE(COALESCE(spu.country, sshpu.country), ahpu.country) = dflt.country diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/schema.yaml index a729a1695db..264ff517151 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/schema.yaml @@ -19,3 +19,23 @@ fields: name: active_hours_per_user_ratio type: FLOAT description: Ratio of Active Hours per User +- mode: NULLABLE + name: default_percent + type: FLOAT + description: Percent of Users with Firefox as Default Browser +- mode: NULLABLE + name: average_profile_age + type: FLOAT + description: Average Profile Age +- mode: NULLABLE + name: pct_users_profile_age_less_than_7_days + type: FLOAT + description: Percent of Users with Profile Age Less than 7 Days +- mode: NULLABLE + name: pct_users_profile_age_8_to_365_days + type: FLOAT + description: Percent of Users with Profile Age Between 8 and 365 Days +- mode: NULLABLE + name: pct_users_profile_age_over_365_days + type: FLOAT + description: Percent of Users with Profile Age Over 365 Days diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/query.sql index a6058d0e40b..9765843d503 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/query.sql @@ -74,6 +74,33 @@ active_hours_per_user AS ( `hours` / users AS active_hours_per_user_ratio FROM active_hours_per_user_staging +), +default_percent_by_os AS ( + SELECT + submission_date_s3, + os, + ROUND(AVG(IF(is_default_browser, 1, 0)) * 100, 1) AS default_percent, + SUM(ROUND(profile_age_in_days)) / COUNT(DISTINCT(client_id)) AS average_profile_age, + COUNT(DISTINCT(client_id)) AS total_nbr_users, + COUNT( + DISTINCT(CASE WHEN profile_age_in_days <= 7 THEN client_id ELSE NULL END) + ) AS nbr_users_profile_age_less_than_7, + COUNT( + DISTINCT(CASE WHEN profile_age_in_days BETWEEN 8 AND 365 THEN client_id ELSE NULL END) + ) AS nbr_users_profile_age_between_8_and_365, + COUNT( + DISTINCT(CASE WHEN profile_age_in_days > 365 THEN client_id ELSE NULL END) + ) AS nbr_users_profile_age_over_365 + FROM + `moz-fx-data-shared-prod.telemetry.clients_daily` + WHERE + submission_date_s3 = @submission_date + AND app_name = 'Firefox' + AND os IN ('Windows_NT', 'Linux', 'Darwin') + AND sample_id = 42 + GROUP BY + submission_date_s3, + os ) SELECT COALESCE( @@ -83,7 +110,12 @@ SELECT COALESCE(COALESCE(spu.os, sshpu.os), ahpu.os) AS os, spu.searches_per_user_ratio, sshpu.subsession_hours_per_user_ratio, - ahpu.active_hours_per_user_ratio + ahpu.active_hours_per_user_ratio, + dflt.default_percent, + dflt.average_profile_age, + dflt.nbr_users_profile_age_less_than_7 / dflt.total_nbr_users AS pct_users_profile_age_less_than_7_days, + dflt.nbr_users_profile_age_between_8_and_365 / dflt.total_nbr_users AS pct_users_profile_age_8_to_365_days, + dflt.nbr_users_profile_age_over_365 / dflt.total_nbr_users AS pct_users_profile_age_over_365_days FROM searches_per_user_by_os_and_date AS spu FULL OUTER JOIN @@ -92,3 +124,6 @@ FULL OUTER JOIN FULL OUTER JOIN active_hours_per_user AS ahpu ON COALESCE(spu.os, sshpu.os) = ahpu.os +FULL OUTER JOIN + default_percent_by_os AS dflt + ON COALESCE(COALESCE(spu.os, sshpu.os), ahpu.os) = dflt.os diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/schema.yaml index 01d518fa404..a4990f15438 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/schema.yaml @@ -19,3 +19,23 @@ fields: name: active_hours_per_user_ratio type: FLOAT description: Ratio of Active Hours per User +- mode: NULLABLE + name: default_percent + type: FLOAT + description: Percent of Users with Firefox as Default Browser +- mode: NULLABLE + name: average_profile_age + type: FLOAT + description: Average Profile Age +- mode: NULLABLE + name: pct_users_profile_age_less_than_7_days + type: FLOAT + description: Percent of Users with Profile Age Less than 7 Days +- mode: NULLABLE + name: pct_users_profile_age_8_to_365_days + type: FLOAT + description: Percent of Users with Profile Age Between 8 and 365 Days +- mode: NULLABLE + name: pct_users_profile_age_over_365_days + type: FLOAT + description: Percent of Users with Profile Age Over 365 Days diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/query.sql index 9f9168e0514..5eedbb61358 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/query.sql @@ -77,6 +77,34 @@ active_hours_per_user AS ( `hours` / users AS active_hours_per_user_ratio FROM active_hours_per_user_staging +), +default_percent_by_os_version AS ( + SELECT + submission_date_s3, + os_version, + ROUND(AVG(IF(is_default_browser, 1, 0)) * 100, 1) AS default_percent, + SUM(ROUND(profile_age_in_days)) / COUNT(DISTINCT(client_id)) AS average_profile_age, + COUNT(DISTINCT(client_id)) AS total_nbr_users, + COUNT( + DISTINCT(CASE WHEN profile_age_in_days <= 7 THEN client_id ELSE NULL END) + ) AS nbr_users_profile_age_less_than_7, + COUNT( + DISTINCT(CASE WHEN profile_age_in_days BETWEEN 8 AND 365 THEN client_id ELSE NULL END) + ) AS nbr_users_profile_age_between_8_and_365, + COUNT( + DISTINCT(CASE WHEN profile_age_in_days > 365 THEN client_id ELSE NULL END) + ) AS nbr_users_profile_age_over_365 + FROM + `moz-fx-data-shared-prod.telemetry.clients_daily` + WHERE + submission_date_s3 = @submission_date + AND app_name = 'Firefox' + AND os = 'Windows_NT' + AND sample_id = 42 + AND os_version IN ('6.1', '6.2', '6.3', '10.0') + GROUP BY + submission_date_s3, + os_version ) SELECT COALESCE( @@ -86,7 +114,12 @@ SELECT COALESCE(COALESCE(spu.os_version, sshpu.os_version), ahpu.os_version) AS os_version, spu.searches_per_user_ratio, sshpu.subsession_hours_per_user_ratio, - ahpu.active_hours_per_user_ratio + ahpu.active_hours_per_user_ratio, + dflt.default_percent, + dflt.average_profile_age, + dflt.nbr_users_profile_age_less_than_7 / dflt.total_nbr_users AS pct_users_profile_age_less_than_7_days, + dflt.nbr_users_profile_age_between_8_and_365 / dflt.total_nbr_users AS pct_users_profile_age_8_to_365_days, + dflt.nbr_users_profile_age_over_365 / dflt.total_nbr_users AS pct_users_profile_age_over_365_days FROM searches_per_user_by_os_and_date AS spu FULL OUTER JOIN @@ -95,3 +128,6 @@ FULL OUTER JOIN FULL OUTER JOIN active_hours_per_user AS ahpu ON COALESCE(spu.os_version, sshpu.os_version) = ahpu.os_version +FULL OUTER JOIN + default_percent_by_os_version AS dflt + ON COALESCE(COALESCE(spu.os_version, sshpu.os_version), ahpu.os_version) = dflt.os_version diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/schema.yaml index 06f3a69f3e7..ca60ba9af1e 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/schema.yaml @@ -19,3 +19,23 @@ fields: name: active_hours_per_user_ratio type: FLOAT description: Ratio of Active Hours per User +- mode: NULLABLE + name: default_percent + type: FLOAT + description: Percent of Users with Firefox as Default Browser +- mode: NULLABLE + name: average_profile_age + type: FLOAT + description: Average Profile Age +- mode: NULLABLE + name: pct_users_profile_age_less_than_7_days + type: FLOAT + description: Percent of Users with Profile Age Less than 7 Days +- mode: NULLABLE + name: pct_users_profile_age_8_to_365_days + type: FLOAT + description: Percent of Users with Profile Age Between 8 and 365 Days +- mode: NULLABLE + name: pct_users_profile_age_over_365_days + type: FLOAT + description: Percent of Users with Profile Age Over 365 Days From 41b43399089fd64e4fc95346ba054e6203a138be Mon Sep 17 00:00:00 2001 From: akkomar Date: Tue, 17 Dec 2024 18:02:12 +0100 Subject: [PATCH 14/29] Delete metadata for public_data_report_hardware (#6686) Table was deleted on 2024-12-17 --- .../public_data_report_hardware/metadata.yaml | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/public_data_report_hardware/metadata.yaml diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/public_data_report_hardware/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/public_data_report_hardware/metadata.yaml deleted file mode 100644 index 8aa746d62b3..00000000000 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/public_data_report_hardware/metadata.yaml +++ /dev/null @@ -1,16 +0,0 @@ -friendly_name: Firefox Public Data Report -description: |- - Powers the public https://data.firefox.com/ dashboard. - - Source code is in the [firefox-public-data-report-etl repository] - (https://github.com/mozilla/firefox-public-data-report-etl). - - This table is to be replaced by telemetry_derived.public_data_report_hardware_v1. -owners: -- akomar@mozilla.com -labels: - owner1: akomar -bigquery: null -references: {} -deprecated: true -deletion_date: 2024-07-03 From db5d02dcf14dfeca9ec8a68bd7ee954f8eb9d37f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 09:42:00 -0800 Subject: [PATCH 15/29] chore(deps): bump mkdocs-material from 9.5.48 to 9.5.49 (#6682) Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 9.5.48 to 9.5.49. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.5.48...9.5.49) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.in | 2 +- requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.in b/requirements.in index ed7c6073068..285016da162 100644 --- a/requirements.in +++ b/requirements.in @@ -20,7 +20,7 @@ PyJWT==2.10.1 markdown-include==0.8.1 mdx_truly_sane_lists==1.3 mkdocs==1.6.1 -mkdocs-material==9.5.48 +mkdocs-material==9.5.49 mkdocs-awesome-pages-plugin==2.9.3 mozilla-metric-config-parser==2024.11.1 mozilla-schema-generator==0.5.1 diff --git a/requirements.txt b/requirements.txt index 75e1e194070..3f8b301322f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -934,9 +934,9 @@ mkdocs-get-deps==0.2.0 \ --hash=sha256:162b3d129c7fad9b19abfdcb9c1458a651628e4b1dea628ac68790fb3061c60c \ --hash=sha256:2bf11d0b133e77a0dd036abeeb06dec8775e46efa526dc70667d8863eefc6134 # via mkdocs -mkdocs-material==9.5.48 \ - --hash=sha256:a582531e8b34f4c7ed38c29d5c44763053832cf2a32f7409567e0c74749a47db \ - --hash=sha256:b695c998f4b939ce748adbc0d3bff73fa886a670ece948cf27818fa115dc16f8 +mkdocs-material==9.5.49 \ + --hash=sha256:3671bb282b4f53a1c72e08adbe04d2481a98f85fed392530051f80ff94a9621d \ + --hash=sha256:c3c2d8176b18198435d3a3e119011922f3e11424074645c24019c2dcf08a360e # via -r requirements.in mkdocs-material-extensions==1.3 \ --hash=sha256:0297cc48ba68a9fdd1ef3780a3b41b534b0d0df1d1181a44676fda5f464eeadc \ From b6184593f15f4ebbc93b2c8ec28fec51589c228d Mon Sep 17 00:00:00 2001 From: kik-kik <42538694+kik-kik@users.noreply.github.com> Date: Tue, 17 Dec 2024 19:48:55 +0100 Subject: [PATCH 16/29] feat(DENG-4975): add bigconfig to baseline clients daily and first seen (#6684) * feat: removed schedule from all metrics except fresness and volume * feat: add template for bigconfig for baseline_clients_first_seen * feat: add template for bigconfig for baseline_clients_daily * feat: add a metric to make sure client_id is unique in the first_seen table * revert sql/bigconfig.yml for metrics to use default schedule --- .../baseline_clients_daily_v1.bigconfig.yml | 35 +++++++++++++++++++ ...seline_clients_first_seen_v1.bigconfig.yml | 25 +++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 sql_generators/glean_usage/templates/baseline_clients_daily_v1.bigconfig.yml create mode 100644 sql_generators/glean_usage/templates/baseline_clients_first_seen_v1.bigconfig.yml diff --git a/sql_generators/glean_usage/templates/baseline_clients_daily_v1.bigconfig.yml b/sql_generators/glean_usage/templates/baseline_clients_daily_v1.bigconfig.yml new file mode 100644 index 00000000000..6dd7a837928 --- /dev/null +++ b/sql_generators/glean_usage/templates/baseline_clients_daily_v1.bigconfig.yml @@ -0,0 +1,35 @@ +{{ header_yaml }} +type: BIGCONFIG_FILE + +tag_deployments: + - collection: + name: Operational Checks + notification_channels: + - slack: '#de-bigeye-triage' + deployments: + - column_selectors: + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.submission_date + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.client_id + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.sample_id + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.first_seen_date + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.first_run_date + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.durations + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.normalized_os + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.normalized_os_version + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.locale + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.country + metrics: + - saved_metric_id: is_not_null + - column_selectors: + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.normalized_channel + metrics: + - saved_metric_id: is_valid_channel + - column_selectors: + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.country + metrics: + - saved_metric_id: is_2_char_len + - column_selectors: + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.* + metrics: + - saved_metric_id: freshness + - saved_metric_id: volume diff --git a/sql_generators/glean_usage/templates/baseline_clients_first_seen_v1.bigconfig.yml b/sql_generators/glean_usage/templates/baseline_clients_first_seen_v1.bigconfig.yml new file mode 100644 index 00000000000..27d3e2a6e30 --- /dev/null +++ b/sql_generators/glean_usage/templates/baseline_clients_first_seen_v1.bigconfig.yml @@ -0,0 +1,25 @@ +{{ header_yaml }} +type: BIGCONFIG_FILE + +tag_deployments: + - collection: + name: Operational Checks + notification_channels: + - slack: '#de-bigeye-triage' + deployments: + - column_selectors: + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.client_id + metrics: + - saved_metric_id: is_unique + - column_selectors: + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.submission_date + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.client_id + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.sample_id + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.first_seen_date + metrics: + - saved_metric_id: is_not_null + - column_selectors: + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.* + metrics: + - saved_metric_id: freshness + - saved_metric_id: volume From b9ba8f6324dd194ee07e479dcfa7bb6fd615a3c6 Mon Sep 17 00:00:00 2001 From: kik-kik <42538694+kik-kik@users.noreply.github.com> Date: Tue, 17 Dec 2024 20:07:19 +0100 Subject: [PATCH 17/29] feat(DENG-7000): Update Bigconfig collection used by mobile_kpi_support_metrics to Operational Checks (#6688) * feat: Update Bigconfig collection used by mobile_kpi_support_metrics to Operational Checks * chore: remove mention of ds alert channel since not needed --- .../mobile_kpi_support_metrics/__init__.py | 8 +++----- .../templates/attribution_clients.bigconfig.yml | 13 +------------ .../templates/engagement.bigconfig.yml | 13 +------------ .../templates/new_profiles.bigconfig.yml | 13 +------------ .../templates/retention.bigconfig.yml | 13 +------------ 5 files changed, 7 insertions(+), 53 deletions(-) diff --git a/sql_generators/mobile_kpi_support_metrics/__init__.py b/sql_generators/mobile_kpi_support_metrics/__init__.py index 7dd753ec711..a029e6cb3fb 100644 --- a/sql_generators/mobile_kpi_support_metrics/__init__.py +++ b/sql_generators/mobile_kpi_support_metrics/__init__.py @@ -33,9 +33,8 @@ ("AGGREGATE", "new_profiles.view.sql"), ("AGGREGATE", "new_profiles.query.sql"), ) -BIGEYE_COLLECTION = "Browser Metrics (non-KPI)" -BIGEYE_NOTIFICATION_SLACK_CHANNEL_DS = "#ds-bigeye-triage" -BIGEYE_NOTIFICATION_SLACK_CHANNEL_DE = "#de-bigeye-triage" +BIGEYE_COLLECTION = "Operational Checks" +BIGEYE_NOTIFICATION_SLACK_CHANNEL = "#de-bigeye-triage" class AttributionPings(Enum): @@ -304,8 +303,7 @@ def generate(target_project, output_dir, use_cloud_function): "version": VERSION, "project_id": target_project, "bigeye_collection": BIGEYE_COLLECTION, - "bigeye_notification_slack_channel_ds": BIGEYE_NOTIFICATION_SLACK_CHANNEL_DS, - "bigeye_notification_slack_channel_de": BIGEYE_NOTIFICATION_SLACK_CHANNEL_DE, + "bigeye_notification_slack_channel": BIGEYE_NOTIFICATION_SLACK_CHANNEL, } query_support_configs = ( diff --git a/sql_generators/mobile_kpi_support_metrics/templates/attribution_clients.bigconfig.yml b/sql_generators/mobile_kpi_support_metrics/templates/attribution_clients.bigconfig.yml index caa41d8b158..208bd77f713 100644 --- a/sql_generators/mobile_kpi_support_metrics/templates/attribution_clients.bigconfig.yml +++ b/sql_generators/mobile_kpi_support_metrics/templates/attribution_clients.bigconfig.yml @@ -4,7 +4,7 @@ tag_deployments: - collection: name: {{ bigeye_collection }} notification_channels: - - slack: '{{ bigeye_notification_slack_channel_ds }}' + - slack: '{{ bigeye_notification_slack_channel }}' deployments: - column_selectors: - name: {{ project_id }}.{{ project_id }}.{{ dataset }}_derived.{{ name }}_{{ version }}.client_id @@ -16,14 +16,3 @@ tag_deployments: metrics: - saved_metric_id: freshness - saved_metric_id: volume - - - collection: - name: Operational Checks - notification_channels: - - slack: '{{ bigeye_notification_slack_channel_de }}' - deployments: - - column_selectors: - - name: {{ project_id }}.{{ project_id }}.{{ dataset }}_derived.{{ name }}_{{ version }}.* - metrics: - - saved_metric_id: freshness - - saved_metric_id: volume diff --git a/sql_generators/mobile_kpi_support_metrics/templates/engagement.bigconfig.yml b/sql_generators/mobile_kpi_support_metrics/templates/engagement.bigconfig.yml index 4bf9be2d182..c814c83b57d 100644 --- a/sql_generators/mobile_kpi_support_metrics/templates/engagement.bigconfig.yml +++ b/sql_generators/mobile_kpi_support_metrics/templates/engagement.bigconfig.yml @@ -4,7 +4,7 @@ tag_deployments: - collection: name: {{ bigeye_collection }} notification_channels: - - slack: '{{ bigeye_notification_slack_channel_ds }}' + - slack: '{{ bigeye_notification_slack_channel }}' deployments: - column_selectors: - name: {{ project_id }}.{{ project_id }}.{{ dataset }}_derived.{{ name }}_{{ version }}.first_seen_date @@ -26,14 +26,3 @@ tag_deployments: metrics: - saved_metric_id: volume - saved_metric_id: freshness - - - collection: - name: Operational Checks - notification_channels: - - slack: '{{ bigeye_notification_slack_channel_de }}' - deployments: - - column_selectors: - - name: {{ project_id }}.{{ project_id }}.{{ dataset }}_derived.{{ name }}_{{ version }}.* - metrics: - - saved_metric_id: freshness - - saved_metric_id: volume diff --git a/sql_generators/mobile_kpi_support_metrics/templates/new_profiles.bigconfig.yml b/sql_generators/mobile_kpi_support_metrics/templates/new_profiles.bigconfig.yml index 4bf9be2d182..c814c83b57d 100644 --- a/sql_generators/mobile_kpi_support_metrics/templates/new_profiles.bigconfig.yml +++ b/sql_generators/mobile_kpi_support_metrics/templates/new_profiles.bigconfig.yml @@ -4,7 +4,7 @@ tag_deployments: - collection: name: {{ bigeye_collection }} notification_channels: - - slack: '{{ bigeye_notification_slack_channel_ds }}' + - slack: '{{ bigeye_notification_slack_channel }}' deployments: - column_selectors: - name: {{ project_id }}.{{ project_id }}.{{ dataset }}_derived.{{ name }}_{{ version }}.first_seen_date @@ -26,14 +26,3 @@ tag_deployments: metrics: - saved_metric_id: volume - saved_metric_id: freshness - - - collection: - name: Operational Checks - notification_channels: - - slack: '{{ bigeye_notification_slack_channel_de }}' - deployments: - - column_selectors: - - name: {{ project_id }}.{{ project_id }}.{{ dataset }}_derived.{{ name }}_{{ version }}.* - metrics: - - saved_metric_id: freshness - - saved_metric_id: volume diff --git a/sql_generators/mobile_kpi_support_metrics/templates/retention.bigconfig.yml b/sql_generators/mobile_kpi_support_metrics/templates/retention.bigconfig.yml index 4bf9be2d182..c814c83b57d 100644 --- a/sql_generators/mobile_kpi_support_metrics/templates/retention.bigconfig.yml +++ b/sql_generators/mobile_kpi_support_metrics/templates/retention.bigconfig.yml @@ -4,7 +4,7 @@ tag_deployments: - collection: name: {{ bigeye_collection }} notification_channels: - - slack: '{{ bigeye_notification_slack_channel_ds }}' + - slack: '{{ bigeye_notification_slack_channel }}' deployments: - column_selectors: - name: {{ project_id }}.{{ project_id }}.{{ dataset }}_derived.{{ name }}_{{ version }}.first_seen_date @@ -26,14 +26,3 @@ tag_deployments: metrics: - saved_metric_id: volume - saved_metric_id: freshness - - - collection: - name: Operational Checks - notification_channels: - - slack: '{{ bigeye_notification_slack_channel_de }}' - deployments: - - column_selectors: - - name: {{ project_id }}.{{ project_id }}.{{ dataset }}_derived.{{ name }}_{{ version }}.* - metrics: - - saved_metric_id: freshness - - saved_metric_id: volume From 9a67368126e636f19b8bc77112387942990a59f7 Mon Sep 17 00:00:00 2001 From: Chelsey Beck <64881557+chelseybeck@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:40:58 -0800 Subject: [PATCH 18/29] AD-517 fixing error in path (#6681) * fixing error in path * updating model to reflect usage * updating dag name * updating start date * updating tier * formatting --- dags.yaml | 6 +++--- .../metadata.yaml | 13 +++++++------ .../query.sql | 3 +-- .../schema.yaml | 0 4 files changed, 11 insertions(+), 11 deletions(-) rename sql/moz-fx-data-shared-prod/telemetry_derived/{newtab_desktop_daily_engagement_by_tile_id_position_v1 => newtab_interactions_hourly_v1}/metadata.yaml (58%) rename sql/moz-fx-data-shared-prod/telemetry_derived/{newtab_desktop_daily_engagement_by_tile_id_position_v1 => newtab_interactions_hourly_v1}/query.sql (97%) rename sql/moz-fx-data-shared-prod/telemetry_derived/{newtab_desktop_daily_engagement_by_tile_id_position_v1 => newtab_interactions_hourly_v1}/schema.yaml (100%) diff --git a/dags.yaml b/dags.yaml index 66d8940ba39..d36b572623f 100644 --- a/dags.yaml +++ b/dags.yaml @@ -1947,18 +1947,18 @@ bqetl_firefox_installer_aggregates: tags: - impact/tier_3 -bqetl_newtab_engagement_hourly: +bqetl_newtab_interactions_hourly: schedule_interval: hourly description: | This DAG aggregates Newtab engagement data - on an hourly cadence default_args: owner: cbeck@mozilla.com email: ["cbeck@mozilla.com"] - start_date: "2024-12-09" + start_date: "2024-12-17" retries: 2 retry_delay: 5m tags: - - impact/tier_2 + - impact/tier_1 bqetl_fx_health_ind_dashboard: description: | diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_desktop_daily_engagement_by_tile_id_position_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_interactions_hourly_v1/metadata.yaml similarity index 58% rename from sql/moz-fx-data-shared-prod/telemetry_derived/newtab_desktop_daily_engagement_by_tile_id_position_v1/metadata.yaml rename to sql/moz-fx-data-shared-prod/telemetry_derived/newtab_interactions_hourly_v1/metadata.yaml index c19626dedeb..2f4a352379b 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_desktop_daily_engagement_by_tile_id_position_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_interactions_hourly_v1/metadata.yaml @@ -1,19 +1,20 @@ -friendly_name: Newtab Desktop Daily Engagement By Tile Id & Position +friendly_name: Newtab Interactions Hourly description: |- - Daily Firefox New Tab engagement on desktop by tile ID & position, across all territories and release channels - This table is a temporary solution for supporting AdOps reporting. This table should eventually be merged with the consolidated ad metrics table(s) + Interactions on Newtab desktop across all territories and release channels. + + This job runs hourly and is aggregated by day, tile ID, and position. owners: - cbeck@mozilla.com labels: incremental: true owner1: cbeck@mozilla.com scheduling: - dag_name: bqetl_newtab_engagement_hourly + dag_name: bqetl_newtab_interactions_hourly date_partition_parameter: null # We reprocess the same day every hour up until 1:00 the following day, to give # the live data time to come in destination_table: >- - newtab_desktop_daily_engagement_by_tile_id_position_v1${{ + newtab_interactions_hourly_v1${{ (execution_date - macros.timedelta(hours=1)).strftime("%Y%m%d") }} parameters: @@ -24,7 +25,7 @@ scheduling: query_file_path: # explicit query file path is necessary because the destination table # includes a partition identifier that is not in the path - sql/moz-fx-data-shared-prod/ads_derived/newtab_desktop_daily_engagement_by_tile_id_position/query.sql + sql/moz-fx-data-shared-prod/telemetry_derived/newtab_interactions_hourly_v1/query.sql bigquery: time_partitioning: type: day diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_desktop_daily_engagement_by_tile_id_position_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_interactions_hourly_v1/query.sql similarity index 97% rename from sql/moz-fx-data-shared-prod/telemetry_derived/newtab_desktop_daily_engagement_by_tile_id_position_v1/query.sql rename to sql/moz-fx-data-shared-prod/telemetry_derived/newtab_interactions_hourly_v1/query.sql index ed3edd16e1c..342b6af3f0d 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_desktop_daily_engagement_by_tile_id_position_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_interactions_hourly_v1/query.sql @@ -91,8 +91,7 @@ glean_deduplicated_pings AS ( FROM `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` WHERE - DATE(submission_timestamp) >= '2024-11-15' - AND DATE(submission_timestamp) = @submission_date + DATE(submission_timestamp) = @submission_date QUALIFY ROW_NUMBER() OVER ( PARTITION BY diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_desktop_daily_engagement_by_tile_id_position_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_interactions_hourly_v1/schema.yaml similarity index 100% rename from sql/moz-fx-data-shared-prod/telemetry_derived/newtab_desktop_daily_engagement_by_tile_id_position_v1/schema.yaml rename to sql/moz-fx-data-shared-prod/telemetry_derived/newtab_interactions_hourly_v1/schema.yaml From b26d5334ccc4fb5c73c2664a5fba15f3eb81fd7a Mon Sep 17 00:00:00 2001 From: Marlene Hirose <92952117+Marlene-M-Hirose@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:14:44 -0800 Subject: [PATCH 19/29] Deng 6360 change upload to json update schema (#6673) * update query.py to use load_file_from_json, eliminate unnecessary functions, add comments to describe reasons, take out partitions; update schema.yaml to reflect the schema. * take out unnecessary csv functions, clean up comments and tatake out extraneous new lines * take out extraenous comma * take out partitioning from schema.yaml file * add schema applied policies field which is a list of dictionaries, sometimes empty * add in the schemas for entity record,typed_metadata record and applied_policies record * add in schema for entity.attributes record * add more fields to scheam from entity.attributes * add date to cinder downloaded data for partitioning, restore date partitioning to metadata.yaml, organize schema.yaml file to build table in a less haphazard way --- .../cinder_decisions_raw_v1/metadata.yaml | 5 +- .../cinder_decisions_raw_v1/query.py | 116 ++++---------- .../cinder_decisions_raw_v1/schema.yaml | 149 +++++++++++++++--- 3 files changed, 157 insertions(+), 113 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/addon_moderations_derived/cinder_decisions_raw_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/addon_moderations_derived/cinder_decisions_raw_v1/metadata.yaml index 0da1d1b4939..43466132df6 100644 --- a/sql/moz-fx-data-shared-prod/addon_moderations_derived/cinder_decisions_raw_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/addon_moderations_derived/cinder_decisions_raw_v1/metadata.yaml @@ -15,8 +15,5 @@ scheduling: bigquery: time_partitioning: type: day - field: 'date' + field: date require_partition_filter: false - expiration_days: null - range_partitioning: null -references: {} diff --git a/sql/moz-fx-data-shared-prod/addon_moderations_derived/cinder_decisions_raw_v1/query.py b/sql/moz-fx-data-shared-prod/addon_moderations_derived/cinder_decisions_raw_v1/query.py index d8f1f27d2aa..8174bd0f259 100644 --- a/sql/moz-fx-data-shared-prod/addon_moderations_derived/cinder_decisions_raw_v1/query.py +++ b/sql/moz-fx-data-shared-prod/addon_moderations_derived/cinder_decisions_raw_v1/query.py @@ -1,30 +1,14 @@ """Cinder API Addon Moderations Decisions - download decisions, clean and upload to BigQuery.""" -import csv import json import os import tempfile from argparse import ArgumentParser -from time import sleep import requests from google.cloud import bigquery -CSV_FIELDS = [ - "user", - "queue_slug", - "job_id", - "uuid", - "applied_policies", - "entity", - "entity_slug", - "entity_id", - "created_at", - "decision_type", - "job_assigned_at", - "typed_metadata", -] - +"""Get the bearer token for Cinder from the environment""" cinder_bearer_token = os.environ.get("CINDER_TOKEN") @@ -53,17 +37,8 @@ def read_json(filename: str) -> dict: return data -def write_dict_to_csv(json_data, filename): - """Write a dictionary to a csv.""" - with open(filename, "w") as out_file: - dict_writer = csv.DictWriter(out_file, CSV_FIELDS) - dict_writer.writeheader() - dict_writer.writerows(json_data) - - def cinder_addon_decisions_download(date, bearer_token): - """Download data from Cinder - bearer_token are called here.""" - # getting overview metrics for different kpis / Deliverables + """Download data from Cinder - bearer_token is called here.""" url = "https://stage.cinder.nonprod.webservices.mozgcp.net/api/v1/decisions/" headers = {"accept": "application/json", "authorization": f"Bearer {bearer_token}"} print(url) @@ -85,68 +60,44 @@ def check_json(cinder_addon_decisions_response_text): return query_export -def clean_json(query_export, date): - """Turn the json file into a list to be input into a CSV for bq upload.""" +def add_date_to_json(query_export_contents, date): + """Add a date to the entries so we can partition table by this date.""" fields_list = [] - for item in query_export["items"]: + for item in query_export_contents: field_dict = { + "date": date, + "decision_type": item["decision_type"], "user": item["user"], - "queue_slug": item["queue_slug"], - "job_id": item["job_id"], - "uuid": item["uuid"], - "applied_policies": item["applied_policies"], + "created_at": item["created_at"], + "entity_id": item["entity_id"], "entity": item["entity"], + "uuid": item["uuid"], "entity_slug": item["entity_slug"], - "entity_id": item["entity_id"], - "created_at": item["created_at"], - "decision_type": item["decision_type"], + "job_id": item["job_id"], "job_assigned_at": item["job_assigned_at"], "typed_metadata": item["typed_metadata"], + "applied_policies": item["applied_policies"], } fields_list.append(field_dict) return fields_list -def upload_to_bigquery(csv_data, project, dataset, table_name, date): +def upload_to_bigquery(data, project, dataset, table_name, date): """Upload the data to bigquery.""" date = date - print("writing json to csv") partition = f"{date}".replace("-", "") - print(partition) - with tempfile.NamedTemporaryFile() as tmp_csv: - with open(tmp_csv.name, "w+b") as f_csv: - write_dict_to_csv(csv_data, f_csv.name) - client = bigquery.Client(project) - job_config = bigquery.LoadJobConfig( - create_disposition="CREATE_IF_NEEDED", - write_disposition="WRITE_TRUNCATE", - time_partitioning=bigquery.TimePartitioning( - type_=bigquery.TimePartitioningType.DAY, - field="date", - ), - skip_leading_rows=1, - schema=[ - bigquery.SchemaField("date", "DATE"), - bigquery.SchemaField("user", "STRING"), - bigquery.SchemaField("queue_slug", "STRING"), - bigquery.SchemaField("job_id", "STRING"), - bigquery.SchemaField("uuid", "STRING"), - bigquery.SchemaField("applied_policies", "STRING"), - bigquery.SchemaField("entity", "STRING"), - bigquery.SchemaField("entity_slug", "STRING"), - bigquery.SchemaField("entity_id", "STRING"), - bigquery.SchemaField("created_at", "DATE"), - bigquery.SchemaField("decision_type", "STRING"), - bigquery.SchemaField("job_assigned_at", "STRING"), - bigquery.SchemaField("typed_metadata", "STRING"), - ], - ) - destination = f"{project}.{dataset}.{table_name}${partition}" - job = client.load_table_from_file(f_csv, destination, job_config=job_config) - print( - f"Writing Decisions data to {destination}. BigQuery job ID: {job.job_id}" - ) - job.result() + client = bigquery.Client(project) + job_config = bigquery.LoadJobConfig( + create_disposition="CREATE_IF_NEEDED", + write_disposition="WRITE_TRUNCATE", + time_partitioning=bigquery.TimePartitioning( + type_=bigquery.TimePartitioningType.DAY, + field="date", + ), + ) + destination = f"{project}.{dataset}.{table_name}${partition}" + job = client.load_table_from_json(data, destination, job_config=job_config) + job.result() def main(): @@ -165,20 +116,17 @@ def main(): date = args.date bearer_token = cinder_bearer_token - data = [] + cinder_data = [] json_file = cinder_addon_decisions_download(date, bearer_token) + """Data returns as a dictionary with a key called 'items' and the value being a list of data""" query_export = check_json(json_file.text) + """Add date to each element in query_export for partitioning""" + query_export_contents = query_export["items"] + cinder_data = add_date_to_json(query_export_contents, date) + """Pull out the list from query_export["items"] and put that data into the cinder_data list""" - if query_export is not None: - # This section writes the tmp json data into a temp CSV file which will then be put into a BigQuery table - cinder_addon_decisions_data = clean_json(query_export, date) - data.extend(cinder_addon_decisions_data) - else: - print("no data for today") - sleep(5) - - upload_to_bigquery(data, project, dataset, table_name, date) + upload_to_bigquery(cinder_data, project, dataset, table_name, date) if __name__ == "__main__": diff --git a/sql/moz-fx-data-shared-prod/addon_moderations_derived/cinder_decisions_raw_v1/schema.yaml b/sql/moz-fx-data-shared-prod/addon_moderations_derived/cinder_decisions_raw_v1/schema.yaml index 31d4bef9e99..64acb799696 100644 --- a/sql/moz-fx-data-shared-prod/addon_moderations_derived/cinder_decisions_raw_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/addon_moderations_derived/cinder_decisions_raw_v1/schema.yaml @@ -3,7 +3,7 @@ fields: - mode: NULLABLE name: date type: DATE - description: date when job run and field that the table is partitioned by + description: Date job is run - mode: NULLABLE name: user @@ -11,49 +11,130 @@ fields: description: User who submitted the report - mode: NULLABLE - name: queue_slug + name: uuid type: STRING - description: Queue_slug + description: ID of UU - mode: NULLABLE - name: job_id + name: created_at type: STRING - description: Job_id of Decision + description: Date decision made - mode: NULLABLE - name: uuid + name: decision_type type: STRING - description: ID of UU + description: The type of decision -- mode: NULLABLE +- fields: + - mode: NULLABLE + name: uuid + type: STRING + - mode: NULLABLE + name: name + type: STRING + description: The name of the policy applied + - mode: NULLABLE + name: is_illegal + type: BOOLEAN + description: Signifies if the addon illegal or not + - mode: NULLABLE + name: parent_uuid + type: STRING + description: Top level UUID + - mode: REPEATED + name: enforcement_actions + type: STRING + description: List of enforcement actions + mode: REPEATED name: applied_policies - type: STRING - description: Policies applied to moderate addon + type: RECORD + description: Applied policies includes parent_uuid, is_illegal field, name, enforcement_actions, uuid -- mode: NULLABLE +- fields: + - mode: NULLABLE + name: entity_type + type: STRING + description: Type of entity + - fields: + - mode: NULLABLE + name: id + type: STRING + description: Attribute ID + - mode: NULLABLE + name: slug + type: STRING + description: Slug + - mode: NULLABLE + name: guid + type: STRING + description: GU ID + - mode: NULLABLE + name: name + type: STRING + description: Name of attribute + - mode: NULLABLE + name: summary + type: STRING + description: Summary of the action taken + - mode: NULLABLE + name: description + type: STRING + description: Description of the issue + - mode: NULLABLE + name: release_notes + type: STRING + description: notes about the release + - mode: NULLABLE + name: average_daily_users + type: INT64 + description: Average number of daily users + - mode: NULLABLE + name: last_updated + type: TIMESTAMP + description: when the job was last updated + - mode: NULLABLE + name: version + type: STRING + description: Which version was this addon + - mode: NULLABLE + name: privacy_policy + type: STRING + description: Which privacy policy is used + - mode: NULLABLE + name: created + type: TIMESTAMP + description: When the attribute was created + - mode: NULLABLE + name: promoted + type: STRING + description: Promoted + mode: NULLABLE + name: attributes + type: RECORD + description: All the attributes + - mode: NULLABLE + name: classifier_scores + type: STRING + description: Top level UUID + mode: NULLABLE name: entity - type: STRING - description: Information about the entity - -- mode: NULLABLE - name: entity_slug - type: STRING - description: Entity Slug + type: RECORD + description: Entity includes classifier_scores, attributes(RECORD), entity_type - mode: NULLABLE name: entity_id type: STRING - description: Add on ID + description: Entity ID - mode: NULLABLE - name: created_at + name: entity_slug type: STRING - description: Date decision made + description: Entity Slug - mode: NULLABLE - name: decision_type + name: job_id type: STRING - description: type of decision + description: Job_id of Decision - mode: NULLABLE name: job_assigned_at @@ -61,6 +142,24 @@ fields: description: Date addon report was assigned to a moderator - mode: NULLABLE - name: typed_metadata + name: queue_slug type: STRING - description: Contains more data + description: Queue slug + +- fields: + - mode: NULLABLE + name: legacy_decisions_labels + type: STRING + description: Legacy decision label + - mode: NULLABLE + name: policy_map + type: STRING + description: Policy Map + - mode: NULLABLE + name: escalation_details + type: STRING + description: Details about the escalation + mode: NULLABLE + name: typed_metadata + type: RECORD + description: Typed Metadata includes escalation_details, policy_map, legacy_decision_labels From 8350f94e266b54bbd527099925bcc2659fb01694 Mon Sep 17 00:00:00 2001 From: Katie Windau <153020235+kwindau@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:57:36 -0600 Subject: [PATCH 20/29] Add additional columns to Firefox Health Indicator aggregate tables (#6689) * Add new column sessions per user to all 3 tables * Add uris_per_user to all 3 table * Add new_profile_ratio to all 3 tables * Add domains_per_user to all 3 tables --- .../query.sql | 12 ++++++++++-- .../schema.yaml | 16 ++++++++++++++++ .../query.sql | 12 ++++++++++-- .../schema.yaml | 16 ++++++++++++++++ .../query.sql | 12 ++++++++++-- .../schema.yaml | 16 ++++++++++++++++ 6 files changed, 78 insertions(+), 6 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/query.sql index ee171be29cd..835c9dd50af 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/query.sql @@ -79,6 +79,7 @@ default_percent_and_avg_age_by_country AS ( country, ROUND(AVG(IF(is_default_browser, 1, 0)) * 100, 1) AS default_percent, SUM(ROUND(profile_age_in_days)) / COUNT(DISTINCT(client_id)) AS average_profile_age, + SUM(sessions_started_on_this_day) AS nbr_sessions, COUNT(DISTINCT(client_id)) AS total_nbr_users, COUNT( DISTINCT(CASE WHEN profile_age_in_days <= 7 THEN client_id ELSE NULL END) @@ -88,7 +89,10 @@ default_percent_and_avg_age_by_country AS ( ) AS nbr_users_profile_age_between_8_and_365, COUNT( DISTINCT(CASE WHEN profile_age_in_days > 365 THEN client_id ELSE NULL END) - ) AS nbr_users_profile_age_over_365 + ) AS nbr_users_profile_age_over_365, + SUM(scalar_parent_browser_engagement_total_uri_count_sum) AS uris, + COUNTIF(profile_age_in_days = 0) AS new_profiles, + SUM(ROUND(scalar_parent_browser_engagement_unique_domains_count_mean)) AS domains FROM `moz-fx-data-shared-prod.telemetry.clients_daily` WHERE @@ -112,7 +116,11 @@ SELECT dflt.average_profile_age, dflt.nbr_users_profile_age_less_than_7 / dflt.total_nbr_users AS pct_users_profile_age_less_than_7_days, dflt.nbr_users_profile_age_between_8_and_365 / dflt.total_nbr_users AS pct_users_profile_age_8_to_365_days, - dflt.nbr_users_profile_age_over_365 / dflt.total_nbr_users AS pct_users_profile_age_over_365_days + dflt.nbr_users_profile_age_over_365 / dflt.total_nbr_users AS pct_users_profile_age_over_365_days, + dflt.nbr_sessions / dflt.total_nbr_users AS sessions_per_user, + dflt.uris / dflt.total_nbr_users AS uris_per_user, + dflt.new_profiles / dflt.total_nbr_users AS new_profile_ratio, + dflt.domains / dflt.total_nbr_users AS domains_per_user FROM searches_per_user_by_country_and_date AS spu FULL OUTER JOIN diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/schema.yaml index 264ff517151..9a68f7b8796 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_country_v1/schema.yaml @@ -39,3 +39,19 @@ fields: name: pct_users_profile_age_over_365_days type: FLOAT description: Percent of Users with Profile Age Over 365 Days +- mode: NULLABLE + name: sessions_per_user + type: FLOAT + description: Ratio of Sessions per User +- mode: NULLABLE + name: uris_per_user + type: FLOAT + description: Ratio of URIs per User +- mode: NULLABLE + name: new_profile_ratio + type: FLOAT + description: Ratio of New Profiles vs All Profiles +- mode: NULLABLE + name: domains_per_user + type: FLOAT + description: Domains per User diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/query.sql index 9765843d503..b17cf5785e4 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/query.sql @@ -81,6 +81,7 @@ default_percent_by_os AS ( os, ROUND(AVG(IF(is_default_browser, 1, 0)) * 100, 1) AS default_percent, SUM(ROUND(profile_age_in_days)) / COUNT(DISTINCT(client_id)) AS average_profile_age, + SUM(sessions_started_on_this_day) AS nbr_sessions, COUNT(DISTINCT(client_id)) AS total_nbr_users, COUNT( DISTINCT(CASE WHEN profile_age_in_days <= 7 THEN client_id ELSE NULL END) @@ -90,7 +91,10 @@ default_percent_by_os AS ( ) AS nbr_users_profile_age_between_8_and_365, COUNT( DISTINCT(CASE WHEN profile_age_in_days > 365 THEN client_id ELSE NULL END) - ) AS nbr_users_profile_age_over_365 + ) AS nbr_users_profile_age_over_365, + SUM(scalar_parent_browser_engagement_total_uri_count_sum) AS uris, + COUNTIF(profile_age_in_days = 0) AS new_profiles, + SUM(ROUND(scalar_parent_browser_engagement_unique_domains_count_mean)) AS domains FROM `moz-fx-data-shared-prod.telemetry.clients_daily` WHERE @@ -115,7 +119,11 @@ SELECT dflt.average_profile_age, dflt.nbr_users_profile_age_less_than_7 / dflt.total_nbr_users AS pct_users_profile_age_less_than_7_days, dflt.nbr_users_profile_age_between_8_and_365 / dflt.total_nbr_users AS pct_users_profile_age_8_to_365_days, - dflt.nbr_users_profile_age_over_365 / dflt.total_nbr_users AS pct_users_profile_age_over_365_days + dflt.nbr_users_profile_age_over_365 / dflt.total_nbr_users AS pct_users_profile_age_over_365_days, + dflt.nbr_sessions / dflt.total_nbr_users AS sessions_per_user, + dflt.uris / dflt.total_nbr_users AS uris_per_user, + dflt.new_profiles / dflt.total_nbr_users AS new_profile_ratio, + dflt.domains / dflt.total_nbr_users AS domains_per_user FROM searches_per_user_by_os_and_date AS spu FULL OUTER JOIN diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/schema.yaml index a4990f15438..8a11fdc407c 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_v1/schema.yaml @@ -39,3 +39,19 @@ fields: name: pct_users_profile_age_over_365_days type: FLOAT description: Percent of Users with Profile Age Over 365 Days +- mode: NULLABLE + name: sessions_per_user + type: FLOAT + description: Ratio of Sessions per User +- mode: NULLABLE + name: uris_per_user + type: FLOAT + description: Ratio of URIs per User +- mode: NULLABLE + name: new_profile_ratio + type: FLOAT + description: Ratio of New Profiles vs All Profiles +- mode: NULLABLE + name: domains_per_user + type: FLOAT + description: Domains per User diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/query.sql index 5eedbb61358..3dad936826e 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/query.sql @@ -84,6 +84,7 @@ default_percent_by_os_version AS ( os_version, ROUND(AVG(IF(is_default_browser, 1, 0)) * 100, 1) AS default_percent, SUM(ROUND(profile_age_in_days)) / COUNT(DISTINCT(client_id)) AS average_profile_age, + SUM(sessions_started_on_this_day) AS nbr_sessions, COUNT(DISTINCT(client_id)) AS total_nbr_users, COUNT( DISTINCT(CASE WHEN profile_age_in_days <= 7 THEN client_id ELSE NULL END) @@ -93,7 +94,10 @@ default_percent_by_os_version AS ( ) AS nbr_users_profile_age_between_8_and_365, COUNT( DISTINCT(CASE WHEN profile_age_in_days > 365 THEN client_id ELSE NULL END) - ) AS nbr_users_profile_age_over_365 + ) AS nbr_users_profile_age_over_365, + SUM(scalar_parent_browser_engagement_total_uri_count_sum) AS uris, + COUNTIF(profile_age_in_days = 0) AS new_profiles, + SUM(ROUND(scalar_parent_browser_engagement_unique_domains_count_mean)) AS domains FROM `moz-fx-data-shared-prod.telemetry.clients_daily` WHERE @@ -119,7 +123,11 @@ SELECT dflt.average_profile_age, dflt.nbr_users_profile_age_less_than_7 / dflt.total_nbr_users AS pct_users_profile_age_less_than_7_days, dflt.nbr_users_profile_age_between_8_and_365 / dflt.total_nbr_users AS pct_users_profile_age_8_to_365_days, - dflt.nbr_users_profile_age_over_365 / dflt.total_nbr_users AS pct_users_profile_age_over_365_days + dflt.nbr_users_profile_age_over_365 / dflt.total_nbr_users AS pct_users_profile_age_over_365_days, + dflt.nbr_sessions / dflt.total_nbr_users AS sessions_per_user, + dflt.uris / dflt.total_nbr_users AS uris_per_user, + dflt.new_profiles / dflt.total_nbr_users AS new_profile_ratio, + dflt.domains / dflt.total_nbr_users AS domains_per_user FROM searches_per_user_by_os_and_date AS spu FULL OUTER JOIN diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/schema.yaml index ca60ba9af1e..b28d94ef2ff 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_clients_daily_by_os_version_v1/schema.yaml @@ -39,3 +39,19 @@ fields: name: pct_users_profile_age_over_365_days type: FLOAT description: Percent of Users with Profile Age Over 365 Days +- mode: NULLABLE + name: sessions_per_user + type: FLOAT + description: Ratio of Sessions per User +- mode: NULLABLE + name: uris_per_user + type: FLOAT + description: Ratio of URIs per User +- mode: NULLABLE + name: new_profile_ratio + type: FLOAT + description: Ratio of New Profiles vs All Profiles +- mode: NULLABLE + name: domains_per_user + type: FLOAT + description: Domains per User From ff1bdccb9cd9f612af7aaca9cbcefd8a7f179d5c Mon Sep 17 00:00:00 2001 From: Marlene Hirose <92952117+Marlene-M-Hirose@users.noreply.github.com> Date: Tue, 17 Dec 2024 17:44:50 -0800 Subject: [PATCH 21/29] =?UTF-8?q?add=20query,=20metadata,=20schema=20files?= =?UTF-8?q?=20for=20clients=5Ffirst=5Fseen=5F28=5Fdays=5Flate=E2=80=A6=20(?= =?UTF-8?q?#6691)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add query, metadata, schema files for clients_first_seen_28_days_later_v3 * take out extraneous space in schema.yaml, add 'v3' to friendly name in metadata.yaml file * reformat sql file * replace string dates with @submission_date * add in platform_version to query --- .../metadata.yaml | 24 ++ .../query.sql | 124 ++++++ .../schema.yaml | 363 ++++++++++++++++++ 3 files changed, 511 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/schema.yaml diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/metadata.yaml new file mode 100644 index 00000000000..fc4fd85732f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/metadata.yaml @@ -0,0 +1,24 @@ +friendly_name: Clients First Seen 28 Days Later v3 +description: |- + Client cohorts 28 days after their first seen date with relevant retention indicators. +owners: +- mhirose@mozilla.com +- shong@mozilla.com +labels: + incremental: true + owner1: example +scheduling: + dag_name: bqetl_analytics_tables + date_partition_offset: -27 + date_partition_parameter: null + parameters: + - submission_date:DATE:{{ds}} +bigquery: + time_partitioning: + type: day + field: first_seen_date + require_partition_filter: true + expiration_days: null + clustering: + fields: + - sample_id diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/query.sql new file mode 100644 index 00000000000..754376bf54f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/query.sql @@ -0,0 +1,124 @@ +WITH clients_first_seen_28_days_ago AS ( + SELECT + client_id, + sample_id, + first_seen_date, + second_seen_date, + architecture, + app_build_id, + app_name, + locale, + platform_version, + vendor, + app_version, + xpcom_abi, + document_id, + distribution_id, + partner_distribution_version, + partner_distributor, + partner_distributor_channel, + partner_id, + attribution_campaign, + attribution_content, + attribution_dltoken, + attribution_dlsource, + attribution_experiment, + attribution_medium, + attribution_source, + attribution_ua, + attribution_variation, + engine_data_load_path, + engine_data_name, + engine_data_origin, + engine_data_submission_url, + apple_model_id, + city, + db_version, + subdivision1, + isp_name, + normalized_channel, + country, + normalized_os, + normalized_os_version, + startup_profile_selection_reason, + installation_first_seen_admin_user, + installation_first_seen_default_path, + installation_first_seen_failure_reason, + installation_first_seen_from_msi, + installation_first_seen_install_existed, + installation_first_seen_installer_type, + installation_first_seen_other_inst, + installation_first_seen_other_msix_inst, + installation_first_seen_profdir_existed, + installation_first_seen_silent, + installation_first_seen_version, + os, + os_version, + windows_build_number, + windows_version, + metadata, + profile_group_id + FROM + `moz-fx-data-shared-prod.telemetry_derived.clients_first_seen_v3` + WHERE + first_seen_date = DATE_SUB(@submission_date, INTERVAL 27 day) +), +clients_first_seen_28_days_ago_with_days_seen AS ( + SELECT + clients_first_seen_28_days_ago.*, + cls.days_seen_bits, + cls.days_visited_1_uri_bits, + cls.days_interacted_bits, + FROM + clients_first_seen_28_days_ago + LEFT JOIN + `moz-fx-data-shared-prod.telemetry.clients_last_seen` cls + ON clients_first_seen_28_days_ago.client_id = cls.client_id + AND cls.submission_date = @submission_date +) +SELECT + * REPLACE ( + COALESCE( + days_seen_bits, + mozfun.bits28.from_string('0000000000000000000000000000') + ) AS days_seen_bits, + COALESCE( + days_visited_1_uri_bits, + mozfun.bits28.from_string('0000000000000000000000000000') + ) AS days_visited_1_uri_bits, + COALESCE( + days_interacted_bits, + mozfun.bits28.from_string('0000000000000000000000000000') + ) AS days_interacted_bits + ), + COALESCE( + BIT_COUNT(mozfun.bits28.from_string('1111111000000000000000000000') & days_seen_bits) >= 5, + FALSE + ) AS activated, + COALESCE( + BIT_COUNT(mozfun.bits28.from_string('0111111111111111111111111111') & days_seen_bits) > 0, + FALSE + ) AS returned_second_day, + COALESCE( + BIT_COUNT( + mozfun.bits28.from_string( + '0111111111111111111111111111' + ) & days_visited_1_uri_bits & days_interacted_bits + ) > 0, + FALSE + ) AS qualified_second_day, + COALESCE( + BIT_COUNT(mozfun.bits28.from_string('0000000000000000000001111111') & days_seen_bits) > 0, + FALSE + ) AS retained_week4, + COALESCE( + BIT_COUNT( + mozfun.bits28.from_string( + '0000000000000000000001111111' + ) & days_visited_1_uri_bits & days_interacted_bits + ) > 0, + FALSE + ) AS qualified_week4, + @submission_date AS submission_date +FROM + clients_first_seen_28_days_ago_with_days_seen diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/schema.yaml new file mode 100644 index 00000000000..115f06b8ed8 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3/schema.yaml @@ -0,0 +1,363 @@ +fields: +- description: Unique ID for the client installation. + name: client_id + type: STRING + mode: NULLABLE + +- description: Sample ID to limit query results during an analysis. + name: sample_id + type: INTEGER + mode: NULLABLE + +- description: Date when the server first received either of the 3 pings from this client. + name: first_seen_date + type: DATE + mode: NULLABLE + +- description: Second date when the server received either of the 3 pings from this client. + name: second_seen_date + type: DATE + mode: NULLABLE + +- description: The application architecture reported by the client. + name: architecture + type: STRING + mode: NULLABLE + +- description: The application build reported by the client. + name: app_build_id + type: STRING + mode: NULLABLE + +- description: The name of the installed app/browser. + name: app_name + type: STRING + mode: NULLABLE + +- description: The best locale that the application should be localized to. + name: locale + type: STRING + mode: NULLABLE + +- description: The appliction platform version reported by the client. + name: platform_version + type: STRING + mode: NULLABLE + +- description: The application vendor. + name: vendor + type: STRING + mode: NULLABLE + +- description: The application version. + name: app_version + type: STRING + mode: NULLABLE + +- description: 'A string tag identifying the binary ABI of the current processor and compiler vtable, as taken from the TARGET_XPCOM_ABI configure variable. + It may not be available on all platforms, especially unusual processor or compiler combinations. The result takes the form -, + eg. x86-msvc, ppc-gcc3. This value should almost always be used in combination with the OS.' + name: xpcom_abi + type: STRING + mode: NULLABLE + +- description: The document ID specified in the URI when the client sent this message. + name: document_id + type: STRING + mode: NULLABLE + +- description: The value of the `distribution.id` preference that identifies the Firefox distribution. + name: distribution_id + type: STRING + mode: NULLABLE + +- description: The value selected for the `distribution.version` preference in the Partner Distribution Configuration File. + name: partner_distribution_version + type: STRING + mode: NULLABLE + +- description: The value of the `app.distributor` preference in the Partner Distribution Configuration File. + name: partner_distributor + type: STRING + mode: NULLABLE + +- description: The value of the `app.distributor.channel` preference in the Partner Distribution Configuration File. + name: partner_distributor_channel + type: STRING + mode: NULLABLE + +- description: The value of the `mozilla.partner.id` preference in the Partner Distribution Configuration File. + name: partner_id + type: STRING + mode: NULLABLE + +- description: Identifier of the particular campaign that led to the download of the product. + name: attribution_campaign + type: STRING + mode: NULLABLE + +- description: Identifier to indicate the particular link within a campaign. + name: attribution_content + type: STRING + mode: NULLABLE + +- description: Unique token created at Firefox download time. + name: attribution_dltoken + type: STRING + mode: NULLABLE + +- description: Identifier that indicates where installations of Firefox originated. + name: attribution_dlsource + type: STRING + mode: NULLABLE + +- description: Funnel experiment parameters. + name: attribution_experiment + type: STRING + mode: NULLABLE + +- description: Category of the source, such as 'organic' for a search engine. + name: attribution_medium + type: STRING + mode: NULLABLE + +- description: Referring partner domain, when install happens via a known partner. + name: attribution_source + type: STRING + mode: NULLABLE + +- description: Client's user agent, which corresponds to the web browser used to download the Firefox installer. + name: attribution_ua + type: STRING + mode: NULLABLE + +- description: Variation of the identifier that indicates where installations of Firefox originated. + name: attribution_variation + type: STRING + mode: NULLABLE + +- description: The anonymized path of the engine xml file. For details on the components refer to the metadata for telemetry.new_profile. + name: engine_data_load_path + type: STRING + mode: NULLABLE + +- description: The name of the default search engine. + name: engine_data_name + type: STRING + mode: NULLABLE + +- description: 'The origin of the search engine. The value will be default for engines that are built-in or from distribution partners, + verified for user-installed engines with valid verification hashes, unverified for non-default engines without verification hash, + and invalid for engines with broken verification hashes.' + name: engine_data_origin + type: STRING + mode: NULLABLE + +- description: The HTTP url we would use to search. For privacy, we don’t record this for user-installed engines. + name: engine_data_submission_url + type: STRING + mode: NULLABLE + +- description: The model IDs for Apple desktop devices. Applies to Mac only. + name: apple_model_id + type: STRING + mode: NULLABLE + +- description: City retrieved as a result of a geographic lookup based on the client's IP address. + name: city + type: STRING + mode: NULLABLE + +- description: The specific geo database version used. + name: db_version + type: STRING + mode: NULLABLE + +- description: First major country subdivision, typically a state, province, or county. + name: subdivision1 + type: STRING + mode: NULLABLE + +- description: The name of the ISP. + name: isp_name + type: STRING + mode: NULLABLE + +- description: The Firefox channel, set to Other for unrecognized channel names. + name: normalized_channel + type: STRING + mode: NULLABLE + +- description: The ISO 3166-1 alpha-2 country code. + name: country + type: STRING + mode: NULLABLE + +- description: The OS name, set to Other for unrecognized OS names. + name: normalized_os + type: STRING + mode: NULLABLE + +- description: The OS version. + name: normalized_os_version + type: STRING + mode: NULLABLE + +- description: 'How the profile was selected during startup. Possible reasons are: + unknown: Generally should not happen, set as a default in case no other reason occurred. profile-manager: The profile was selected by the profile manager. + profile-reset: The profile was selected for reset, normally this would mean a restart. restart: The user restarted the application, + the same profile as previous will be used. argument-profile: The profile was selected by the --profile command line argument. + argument-p: The profile was selected by the -p command line argument. firstrun-claimed-default: A first run of a dedicated profiles build chose the old + default profile to be the default for this install. firstrun-skipped-default: A first run of a dedicated profiles build skipped over the old default profile + and created a new profile. restart-claimed-default: A first run of a dedicated profiles build after a restart chose the old default.' + name: startup_profile_selection_reason + type: STRING + mode: NULLABLE + +- description: First seen admin user at installation + name: installation_first_seen_admin_user + type: BOOLEAN + mode: NULLABLE + +- description: First seen default path at installation + name: installation_first_seen_default_path + type: BOOLEAN + mode: NULLABLE + +- description: First seen failure reason at installation + name: installation_first_seen_failure_reason + type: STRING + mode: NULLABLE + +- description: First seen from MSI at installation + name: installation_first_seen_from_msi + type: BOOLEAN + mode: NULLABLE + +- description: First seen install exists at installation + name: installation_first_seen_install_existed + type: BOOLEAN + mode: NULLABLE + +- description: First seen installer type at installation + name: installation_first_seen_installer_type + type: STRING + mode: NULLABLE + +- description: First seen other inst at installation + name: installation_first_seen_other_inst + type: BOOLEAN + mode: NULLABLE + +- description: First seen other MSIX inst at installation + name: installation_first_seen_other_msix_inst + type: BOOLEAN + mode: NULLABLE + +- description: First seen profdir existed at installation + name: installation_first_seen_profdir_existed + type: BOOLEAN + mode: NULLABLE + +- description: First seen silent at installation + name: installation_first_seen_silent + type: BOOLEAN + mode: NULLABLE + +- description: First seen version at installation + name: installation_first_seen_version + type: STRING + mode: NULLABLE + +- description: OS name - not normalized_os + name: os + type: STRING + mode: NULLABLE + +- description: OS version - not normalized + name: os_version + type: STRING + mode: NULLABLE + +- description: Windows build number + name: windows_build_number + type: FLOAT64 + mode: NULLABLE + +- description: Combo of os, os_version, windows_build_number, NULL if not windows + name: windows_version + type: STRING + mode: NULLABLE + +- description: A collection of pings - which ping is reported in the table as well as a record of all other + pings in the data for this one particular client + name: metadata + type: RECORD + mode: NULLABLE + fields: + - description: Ping that reported the first seen date (main, first_shutdown or new_profile). + name: first_seen_date_source_ping + type: STRING + mode: NULLABLE + - description: Indicates wether the client ever reported a main ping. + name: reported_main_ping + type: BOOLEAN + mode: NULLABLE + - description: Indicates wether the client ever reported a new profile ping. + name: reported_new_profile_ping + type: BOOLEAN + mode: NULLABLE + - description: Indicates wether the client ever reported a first shutdown ping. + name: reported_shutdown_ping + type: BOOLEAN + mode: NULLABLE + +- name: profile_group_id + type: STRING + mode: NULLABLE + description: A UUID identifying the profile's group on a single device and allowing user-oriented correlation of data + +- description: Days seen in bits + name: days_seen_bits + type: INTEGER + mode: NULLABLE + +- description: Days visited 1 in uri bits + name: days_visited_1_uri_bits + type: INTEGER + mode: NULLABLE + +- description: Days interacted in bits + name: days_interacted_bits + type: INTEGER + mode: NULLABLE + +- description: Is client activated, true or false + name: activated + type: BOOLEAN + mode: NULLABLE + +- description: Did client return second day + name: returned_second_day + type: BOOLEAN + mode: NULLABLE + +- description: Is client qualified second day + name: qualified_second_day + type: BOOLEAN + mode: NULLABLE + +- description: Is client retained week 4 + name: retained_week4 + type: BOOLEAN + mode: NULLABLE + +- description: Is client qualified week 4 + name: qualified_week4 + type: BOOLEAN + mode: NULLABLE + +- description: Submission date + name: submission_date + type: DATE + mode: NULLABLE From aafdf6bffc130239f2c2408ededd880fd0e50ad9 Mon Sep 17 00:00:00 2001 From: Katie Windau <153020235+kwindau@users.noreply.github.com> Date: Tue, 17 Dec 2024 20:20:21 -0600 Subject: [PATCH 22/29] Create new event_aggregates_v1 table (#6690) --- dags.yaml | 19 +++++ .../telemetry/event_aggregates/view.sql | 7 ++ .../event_aggregates_v1/metadata.yaml | 23 +++++ .../event_aggregates_v1/query.sql | 72 ++++++++++++++++ .../event_aggregates_v1/schema.yaml | 85 +++++++++++++++++++ 5 files changed, 206 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/telemetry/event_aggregates/view.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/schema.yaml diff --git a/dags.yaml b/dags.yaml index d36b572623f..711ad0bc404 100644 --- a/dags.yaml +++ b/dags.yaml @@ -1978,3 +1978,22 @@ bqetl_fx_health_ind_dashboard: - impact/tier_2 repo: bigquery-etl schedule_interval: 0 16 * * * + +bqetl_event_aggregates: + description: | + This DAG builds the event_aggregates table + default_args: + depends_on_past: false + owner: kwindau@mozilla.com + email: + - telemetry-alerts@mozilla.com + - kwindau@mozilla.com + email_on_failure: true + email_on_retry: false + start_date: "2024-12-19" + retries: 2 + retry_delay: 5m + tags: + - impact/tier_3 + repo: bigquery-etl + schedule_interval: 40 16 * * * diff --git a/sql/moz-fx-data-shared-prod/telemetry/event_aggregates/view.sql b/sql/moz-fx-data-shared-prod/telemetry/event_aggregates/view.sql new file mode 100644 index 00000000000..3074aa74a4d --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry/event_aggregates/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.telemetry.event_aggregates` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.telemetry_derived.event_aggregates_v1` diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/metadata.yaml new file mode 100644 index 00000000000..6fdad15f423 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/metadata.yaml @@ -0,0 +1,23 @@ +friendly_name: Event Aggregates +description: |- + Aggregate table based off of telemetry.events +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau@mozilla.com + table_type: aggregate +scheduling: + dag_name: bqetl_event_aggregates +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - normalized_channel + - country +references: {} diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/query.sql new file mode 100644 index 00000000000..41ef01f63d6 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/query.sql @@ -0,0 +1,72 @@ +SELECT + submission_date, + event_category, + event_object, + event_method, + event_string_value, + country, + normalized_channel, + os, + os_version, + -- Installation events + COUNTIF(event_category = 'installation') AS installation_cnt, + -- Security UI Protections Popup events + COUNTIF( + event_category = 'security.ui.protectionspopup' + AND event_object = 'etp_toggle_off' + ) AS etp_toggle_off_cnt, + COUNTIF( + event_category = 'security.ui.protectionspopup' + AND event_object = 'etp_toggle_on' + ) AS etp_toggle_on_cnt, + COUNTIF( + event_category = 'security.ui.protectionspopup' + AND event_object = 'sitenotworking_link' + ) AS sitenotworking_link_cnt, + COUNTIF( + event_category = 'security.ui.protectionspopup' + AND event_object = 'send_report_link' + ) AS send_report_link_cnt, + COUNTIF( + event_category = 'security.ui.protectionspopup' + AND event_object = 'send_report_submit' + ) AS send_report_submit_cnt, + -- Security UI Certerror events + COUNTIF(event_category = 'security.ui.certerror') AS certerror_cnt, + -- Certerror aboutcerterror load events + COUNTIF( + event_category = 'security.ui.certerror' + AND event_object = 'aboutcerterror' + AND event_method = 'load' + ) AS certerror_aboutcerterror_load_cnt, + -- Certerror aboutcerterror load events with event_string_value set + COUNTIF( + event_category = 'security.ui.certerror' + AND event_object = 'aboutcerterror' + AND event_method = 'load' + AND event_string_value IS NOT NULL + ) AS certerror_aboutcerterror_load_value_cnt, + -- Intl Browser Language Events + COUNTIF(event_category = 'intl.ui.browserLanguage') AS browser_language_cnt, + COUNTIF( + event_category = 'intl.ui.browserLanguage' + AND event_method = 'click' + ) AS browser_language_click_cnt, + COUNTIF( + event_category = 'intl.ui.browserLanguage' + AND event_object = 'language_item' + ) AS browser_language_language_item_cnt +FROM + `moz-fx-data-shared-prod.telemetry.events` +WHERE + submission_date = @submission_date +GROUP BY + submission_date, + event_category, + event_object, + event_method, + event_string_value, + country, + normalized_channel, + os, + os_version diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/schema.yaml new file mode 100644 index 00000000000..012b2c7765b --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/schema.yaml @@ -0,0 +1,85 @@ +fields: +- name: submission_date + type: DATE + mode: NULLABLE + description: Submission Date +- name: event_category + type: STRING + mode: NULLABLE + description: Event Category +- name: event_object + type: STRING + mode: NULLABLE + description: Event Object +- name: event_method + type: STRING + mode: NULLABLE + description: Event Method +- name: event_string_value + type: STRING + mode: NULLABLE + description: Event String Value +- name: country + type: STRING + mode: NULLABLE + description: Country +- name: normalized_channel + type: STRING + mode: NULLABLE + description: Normalized Channel +- name: os + type: STRING + mode: NULLABLE + description: Operating System +- name: os_version + type: STRING + mode: NULLABLE + description: Operating System Version +- name: installation_cnt + type: INTEGER + mode: NULLABLE + description: Installation Count +- name: etp_toggle_off_cnt + type: INTEGER + mode: NULLABLE + description: ETP Toggle Off Count +- name: etp_toggle_on_cnt + type: INTEGER + mode: NULLABLE + description: ETP Toggle On Count +- name: sitenotworking_link_cnt + type: INTEGER + mode: NULLABLE + description: Site Not Working Link Count +- name: send_report_link_cnt + type: INTEGER + mode: NULLABLE + description: Send Report Link Count +- name: send_report_submit_cnt + type: INTEGER + mode: NULLABLE + description: Send Report Submit Count +- name: certerror_cnt + type: INTEGER + mode: NULLABLE + description: Certificate Error Count +- name: certerror_aboutcerterror_load_cnt + type: INTEGER + mode: NULLABLE + description: Certificate Error - About Certificate Error - Load Count +- name: certerror_aboutcerterror_load_value_cnt + type: INTEGER + mode: NULLABLE + description: Certificate Error - About Certificate Error - Load Value Count +- name: browser_language_cnt + type: INTEGER + mode: NULLABLE + description: Browser Language Count +- name: browser_language_click_cnt + type: INTEGER + mode: NULLABLE + description: Browser Language Click Count +- name: browser_language_language_item_cnt + type: INTEGER + mode: NULLABLE + description: Browser Language Language Item Count From a2419fd64f70bdb940eec562b1097d42303b162d Mon Sep 17 00:00:00 2001 From: Katie Windau <153020235+kwindau@users.noreply.github.com> Date: Wed, 18 Dec 2024 09:10:46 -0600 Subject: [PATCH 23/29] DENG-6892 - backfill new event_aggregates_v1 table (#6695) * DENG-6892 - backfill new event_aggregates_v1 table * Make start date 1/1/19 instead of 1/1/20 --- .../telemetry_derived/event_aggregates_v1/backfill.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/backfill.yaml diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/backfill.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/backfill.yaml new file mode 100644 index 00000000000..b3f99353fbb --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/backfill.yaml @@ -0,0 +1,9 @@ +2024-12-18: + start_date: 2019-01-01 + end_date: 2024-12-16 + reason: https://mozilla-hub.atlassian.net/browse/DENG-6892 + watchers: + - kwindau@mozilla.com + status: Initiate + shredder_mitigation: false + override_retention_limit: true From cfd7af5866a881cea08c947fb924ead7b7ba731c Mon Sep 17 00:00:00 2001 From: kik-kik <42538694+kik-kik@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:20:09 +0100 Subject: [PATCH 24/29] feat: add bigconfig to metrics_clients_daily and last_seen templates (#6687) * feat: add bigconfig to metrics_clients_daily and last_seen templates * feat: update glean_usage/common to also copy bigconfig for metrics_clients --- sql_generators/glean_usage/common.py | 13 +++++++++ .../metrics_clients_daily.bigconfig.yml | 27 +++++++++++++++++++ .../metrics_clients_daily.metadata.yaml | 2 ++ .../metrics_clients_last_seen.bigconfig.yml | 27 +++++++++++++++++++ .../metrics_clients_last_seen.metadata.yaml | 2 ++ 5 files changed, 71 insertions(+) create mode 100644 sql_generators/glean_usage/templates/metrics_clients_daily.bigconfig.yml create mode 100644 sql_generators/glean_usage/templates/metrics_clients_last_seen.bigconfig.yml diff --git a/sql_generators/glean_usage/common.py b/sql_generators/glean_usage/common.py index e4048ebac14..db4f1ceeb8b 100644 --- a/sql_generators/glean_usage/common.py +++ b/sql_generators/glean_usage/common.py @@ -460,6 +460,7 @@ def generate_per_app( format=False, **render_kwargs, ) + table = f"{project_id}.{target_dataset}_derived.{self.target_table_id}" view = f"{project_id}.{target_dataset}.{target_view_name}" @@ -477,6 +478,18 @@ def generate_per_app( Artifact(view, "view.sql", view_sql), ] + if self.target_table_id.startswith("metrics_clients_"): + bigconfig_contents = render( + f"{self.target_table_id[:-3]}.bigconfig.yml", + format=False, + template_folder=PATH / "templates", + **render_kwargs, + ) + + artifacts.append( + Artifact(table, "bigconfig.yml", bigconfig_contents) + ) + for artifact in artifacts: destination = ( get_table_dir(output_dir, artifact.table_id) / artifact.basename diff --git a/sql_generators/glean_usage/templates/metrics_clients_daily.bigconfig.yml b/sql_generators/glean_usage/templates/metrics_clients_daily.bigconfig.yml new file mode 100644 index 00000000000..90ce29cd612 --- /dev/null +++ b/sql_generators/glean_usage/templates/metrics_clients_daily.bigconfig.yml @@ -0,0 +1,27 @@ +{{ header_yaml }} +type: BIGCONFIG_FILE + +tag_deployments: + - collection: + name: Operational Checks + notification_channels: + - slack: '#de-bigeye-triage' + deployments: + - column_selectors: + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.submission_date + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.client_id + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.sample_id + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.normalized_channel + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.n_metrics_ping + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.days_sent_metrics_ping_bits + metrics: + - saved_metric_id: is_not_null + - column_selectors: + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.normalized_channel + metrics: + - saved_metric_id: is_valid_channel + - column_selectors: + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.* + metrics: + - saved_metric_id: freshness + - saved_metric_id: volume diff --git a/sql_generators/glean_usage/templates/metrics_clients_daily.metadata.yaml b/sql_generators/glean_usage/templates/metrics_clients_daily.metadata.yaml index 3679b0e9b31..39b68f1575c 100644 --- a/sql_generators/glean_usage/templates/metrics_clients_daily.metadata.yaml +++ b/sql_generators/glean_usage/templates/metrics_clients_daily.metadata.yaml @@ -19,3 +19,5 @@ bigquery: fields: - normalized_channel - sample_id +monitoring: + enabled: true diff --git a/sql_generators/glean_usage/templates/metrics_clients_last_seen.bigconfig.yml b/sql_generators/glean_usage/templates/metrics_clients_last_seen.bigconfig.yml new file mode 100644 index 00000000000..90ce29cd612 --- /dev/null +++ b/sql_generators/glean_usage/templates/metrics_clients_last_seen.bigconfig.yml @@ -0,0 +1,27 @@ +{{ header_yaml }} +type: BIGCONFIG_FILE + +tag_deployments: + - collection: + name: Operational Checks + notification_channels: + - slack: '#de-bigeye-triage' + deployments: + - column_selectors: + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.submission_date + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.client_id + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.sample_id + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.normalized_channel + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.n_metrics_ping + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.days_sent_metrics_ping_bits + metrics: + - saved_metric_id: is_not_null + - column_selectors: + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.normalized_channel + metrics: + - saved_metric_id: is_valid_channel + - column_selectors: + - name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}.* + metrics: + - saved_metric_id: freshness + - saved_metric_id: volume diff --git a/sql_generators/glean_usage/templates/metrics_clients_last_seen.metadata.yaml b/sql_generators/glean_usage/templates/metrics_clients_last_seen.metadata.yaml index 796cd422b5b..ab77c078f82 100644 --- a/sql_generators/glean_usage/templates/metrics_clients_last_seen.metadata.yaml +++ b/sql_generators/glean_usage/templates/metrics_clients_last_seen.metadata.yaml @@ -24,3 +24,5 @@ bigquery: fields: - normalized_channel - sample_id +monitoring: + enabled: true From 237b648484c7e5c40ca18371f8f69dc309099ab9 Mon Sep 17 00:00:00 2001 From: kik-kik <42538694+kik-kik@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:30:24 +0100 Subject: [PATCH 25/29] feat: make sure campaign_stats aggregations have a value if null. (#6694) * feat: remove some null metrics for ios_app_campaign_statS_v1 and android_app_campaign_stats_v1 * feat: make sure aggregated fields are set to 0 if null * feat: add back null checks as now the value should be set to 0 instead of null via the query * feat: fix typo in the query --- .../ios_app_campaign_stats_v1/query.sql | 18 +++++++++--------- .../android_app_campaign_stats_v1/query.sql | 16 ++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/query.sql b/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/query.sql index 541152fe61d..63fe949ca60 100644 --- a/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/query.sql @@ -77,15 +77,15 @@ SELECT daily_stats.campaign_country_code, daily_stats.ad_group_name, daily_stats.ad_group_id, - SUM(impressions) AS impressions, - SUM(clicks) AS clicks, - SUM(daily_stats.total_downloads) AS downloads, - SUM(new_profiles) AS new_profiles, - SUM(activated) AS activated_profiles, - SUM(repeat_users) AS repeat_users, - SUM(retained_week_4) week_4_retained_users, - SUM(spend) AS spend, - SUM(lifetime_value) AS lifetime_value, + COALESCE(SUM(impressions), 0) AS impressions, + COALESCE(SUM(clicks), 0) AS clicks, + COALESCE(SUM(daily_stats.total_downloads), 0) AS downloads, + COALESCE(SUM(new_profiles), 0) AS new_profiles, + COALESCE(SUM(activated), 0) AS activated_profiles, + COALESCE(SUM(repeat_users), 0) AS repeat_users, + COALESCE(SUM(retained_week_4), 0) week_4_retained_users, + COALESCE(SUM(spend), 0) AS spend, + COALESCE(SUM(lifetime_value), 0) AS lifetime_value, FROM daily_stats LEFT JOIN diff --git a/sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/query.sql b/sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/query.sql index 61950813f12..bf9fd54129a 100644 --- a/sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/query.sql @@ -92,14 +92,14 @@ SELECT campaigns_v2.campaign_segments, ad_groups_v1.ad_group_name AS ad_group, ad_groups_v1.ad_group_segments, - impressions, - clicks, - new_profiles, - activated_profiles, - repeat_users, - week_4_retained_users, - spend, - lifetime_value, + COALESCE(impressions, 0) AS impressions, + COALESCE(clicks, 0) AS clicks, + COALESCE(new_profiles, 0) AS new_profiles, + COALESCE(activated_profiles, 0) AS activated_profiles, + COALESCE(repeat_users, 0) AS repeat_users, + COALESCE(week_4_retained_users, 0) AS week_4_retained_users, + COALESCE(spend, 0) AS spend, + COALESCE(lifetime_value, 0) AS lifetime_value, FROM by_ad_group_id JOIN From bf8b006204236d4274186ab53d958a08a7a2cc4c Mon Sep 17 00:00:00 2001 From: kik-kik <42538694+kik-kik@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:32:56 +0100 Subject: [PATCH 26/29] feat: bump up version of bigeye-sdk to 0.4.94 (#6693) --- requirements.in | 2 +- requirements.txt | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/requirements.in b/requirements.in index 285016da162..80578856660 100644 --- a/requirements.in +++ b/requirements.in @@ -1,6 +1,6 @@ attrs==24.3.0 authlib==1.3.2 -bigeye-sdk==0.4.93 +bigeye-sdk==0.4.94 black==24.10.0 cattrs==24.1.2 click==8.1.7 diff --git a/requirements.txt b/requirements.txt index 3f8b301322f..ed4fb7ebc32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -132,9 +132,9 @@ babel==2.12.1 \ betterproto[compiler]==1.2.5 \ --hash=sha256:74a3ab34646054f674d236d1229ba8182dc2eae86feb249b8590ef496ce9803d # via bigeye-sdk -bigeye-sdk==0.4.93 \ - --hash=sha256:29b14882f31f67b4888e58c0e4efe6a3b152abbcc09af131bc38f91b33e93450 \ - --hash=sha256:77a0333a4c98888d511230503c9e30c0e16441ed455b2c74f9740f7628a6b7e7 +bigeye-sdk==0.4.94 \ + --hash=sha256:d17e50629ecf3273bc7ca8e4ff9883d8fb59e1111776a02ef9cdc07399e1306c \ + --hash=sha256:ff230eb0fcfdfd72932ec83975ede59c7f1c2d2fed89286fb70463f3752fcef7 # via -r requirements.in black==24.10.0 \ --hash=sha256:14b3502784f09ce2443830e3133dacf2c0110d45191ed470ecb04d0f5f6fcb0f \ @@ -764,9 +764,7 @@ jaraco-classes==3.4.0 \ jeepney==0.8.0 \ --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ --hash=sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755 - # via - # keyring - # secretstorage + # via secretstorage jinja2==3.1.4 \ --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \ --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d @@ -2128,9 +2126,7 @@ s3transfer==0.10.2 \ secretstorage==3.3.3 \ --hash=sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77 \ --hash=sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99 - # via - # bigeye-sdk - # keyring + # via bigeye-sdk shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de From 93f99d37cef3c80d0f7020227204a0904c68cfe8 Mon Sep 17 00:00:00 2001 From: Chelsey Beck <64881557+chelseybeck@users.noreply.github.com> Date: Wed, 18 Dec 2024 09:02:34 -0800 Subject: [PATCH 27/29] fixing error in group by causing abundance of records (#6697) --- .../telemetry_derived/newtab_interactions_hourly_v1/query.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_interactions_hourly_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_interactions_hourly_v1/query.sql index 342b6af3f0d..d829eaee137 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_interactions_hourly_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_interactions_hourly_v1/query.sql @@ -140,7 +140,7 @@ glean_summary AS ( -- exclude suspicious activity NOT (user_event_count > 50) GROUP BY - submission_timestamp, + submission_date, recommendation_id, tile_id, position From 6f23929d0a50b995c28ec5d73ec7381ef88dbea7 Mon Sep 17 00:00:00 2001 From: Katie Windau <153020235+kwindau@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:12:44 -0600 Subject: [PATCH 28/29] Create aggregate tables for Firefox Health Indicators dashboard (#6699) * Create new aggregate table for Firefox Health Indicator dashboard * Create new video playback by OS version table * Create agg table at country level * Fix SQL formatting --- .../view.sql | 7 ++++++ .../fx_health_ind_vid_plybck_by_os/view.sql | 7 ++++++ .../view.sql | 7 ++++++ .../metadata.yaml | 22 +++++++++++++++++++ .../query.sql | 15 +++++++++++++ .../schema.yaml | 13 +++++++++++ .../metadata.yaml | 22 +++++++++++++++++++ .../query.sql | 17 ++++++++++++++ .../schema.yaml | 13 +++++++++++ .../metadata.yaml | 22 +++++++++++++++++++ .../query.sql | 16 ++++++++++++++ .../schema.yaml | 13 +++++++++++ 12 files changed, 174 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_vid_plybck_by_country/view.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_vid_plybck_by_os/view.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_vid_plybck_by_os_version/view.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_country_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_country_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_country_v1/schema.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_v1/schema.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_version_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_version_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_version_v1/schema.yaml diff --git a/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_vid_plybck_by_country/view.sql b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_vid_plybck_by_country/view.sql new file mode 100644 index 00000000000..97cd78d79f9 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_vid_plybck_by_country/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.telemetry.fx_health_ind_vid_plybck_by_country` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.telemetry_derived.fx_health_ind_vid_plybck_by_country_v1` diff --git a/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_vid_plybck_by_os/view.sql b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_vid_plybck_by_os/view.sql new file mode 100644 index 00000000000..48794eccf3d --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_vid_plybck_by_os/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.telemetry.fx_health_ind_vid_plybck_by_os` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.telemetry_derived.fx_health_ind_vid_plybck_by_os_v1` diff --git a/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_vid_plybck_by_os_version/view.sql b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_vid_plybck_by_os_version/view.sql new file mode 100644 index 00000000000..ba68d171b2f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_vid_plybck_by_os_version/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.telemetry.fx_health_ind_vid_plybck_by_os_version` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.telemetry_derived.fx_health_ind_vid_plybck_by_os_version_v1` diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_country_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_country_v1/metadata.yaml new file mode 100644 index 00000000000..e3a61332a9b --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_country_v1/metadata.yaml @@ -0,0 +1,22 @@ +friendly_name: Fx Health Ind Vid Plybck By Os +description: |- + Aggregate table of video playback minutes per user per OS +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau@mozilla.com + table_type: aggregate +scheduling: + dag_name: bqetl_fx_health_ind_dashboard +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - normalized_country_code +references: {} diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_country_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_country_v1/query.sql new file mode 100644 index 00000000000..51eb11f00ba --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_country_v1/query.sql @@ -0,0 +1,15 @@ +SELECT + DATE(submission_timestamp) AS submission_date, + normalized_country_code, + SUM( + `moz-fx-data-shared-prod.udf.histogram_max_key_with_nonzero_value`( + payload.processes.content.histograms.video_play_time_ms + ) + ) / COUNT(DISTINCT client_id) / 60000 AS play_time_ratio +FROM + `moz-fx-data-shared-prod.telemetry.main_1pct` +WHERE + DATE(submission_timestamp) = @submission_date +GROUP BY + DATE(submission_timestamp), + normalized_country_code diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_country_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_country_v1/schema.yaml new file mode 100644 index 00000000000..a933bd3d024 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_country_v1/schema.yaml @@ -0,0 +1,13 @@ +fields: +- mode: NULLABLE + name: submission_date + type: DATE + description: Submission Date +- mode: NULLABLE + name: normalized_country_code + type: STRING + description: Normalized Country Code +- mode: NULLABLE + name: play_time_ratio + type: FLOAT + description: Play Time Ratio diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_v1/metadata.yaml new file mode 100644 index 00000000000..1b718992a0a --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_v1/metadata.yaml @@ -0,0 +1,22 @@ +friendly_name: Fx Health Ind Vid Plybck By Os +description: |- + Aggregate table of video playback minutes per user per OS +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau@mozilla.com + table_type: aggregate +scheduling: + dag_name: bqetl_fx_health_ind_dashboard +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - normalized_os +references: {} diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_v1/query.sql new file mode 100644 index 00000000000..6e971b8b7e6 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_v1/query.sql @@ -0,0 +1,17 @@ +SELECT + DATE(submission_timestamp) AS submission_date, + normalized_os, + SUM( + `moz-fx-data-shared-prod.udf.histogram_max_key_with_nonzero_value`( + payload.processes.content.histograms.video_play_time_ms + ) + ) / COUNT(DISTINCT client_id) / 60000 AS play_time_ratio +FROM + `moz-fx-data-shared-prod.telemetry.main` +WHERE + DATE(submission_timestamp) = @submission_date + AND normalized_os IN ('Windows', 'Linux', 'Mac') + AND sample_id = 42 +GROUP BY + DATE(submission_timestamp), + normalized_os diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_v1/schema.yaml new file mode 100644 index 00000000000..40df47c3b1a --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_v1/schema.yaml @@ -0,0 +1,13 @@ +fields: +- mode: NULLABLE + name: submission_date + type: DATE + description: Submission Date +- mode: NULLABLE + name: normalized_os + type: STRING + description: Normalized Operating System +- mode: NULLABLE + name: play_time_ratio + type: FLOAT + description: Play Time Ratio diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_version_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_version_v1/metadata.yaml new file mode 100644 index 00000000000..cbe84fc7d9d --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_version_v1/metadata.yaml @@ -0,0 +1,22 @@ +friendly_name: Fx Health Ind Vid Plybck By Os +description: |- + Aggregate table of video playback minutes per user per OS +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau@mozilla.com + table_type: aggregate +scheduling: + dag_name: bqetl_fx_health_ind_dashboard +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - normalized_os_version +references: {} diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_version_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_version_v1/query.sql new file mode 100644 index 00000000000..e033daaa29f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_version_v1/query.sql @@ -0,0 +1,16 @@ +SELECT + DATE(submission_timestamp) AS submission_date, + normalized_os_version, + SUM( + `moz-fx-data-shared-prod.udf.histogram_max_key_with_nonzero_value`( + payload.processes.content.histograms.video_play_time_ms + ) + ) / COUNT(DISTINCT client_id) / 60000 AS play_time_ratio +FROM + `moz-fx-data-shared-prod.telemetry.main_1pct` +WHERE + DATE(submission_timestamp) = @submission_date + AND normalized_os = 'Windows' +GROUP BY + DATE(submission_timestamp), + normalized_os_version diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_version_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_version_v1/schema.yaml new file mode 100644 index 00000000000..e5a0999122a --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_vid_plybck_by_os_version_v1/schema.yaml @@ -0,0 +1,13 @@ +fields: +- mode: NULLABLE + name: submission_date + type: DATE + description: Submission Date +- mode: NULLABLE + name: normalized_os_version + type: STRING + description: Normalized Operating System Version +- mode: NULLABLE + name: play_time_ratio + type: FLOAT + description: Play Time Ratio From facf8ea9184d518a25f8bc8519970c735283174c Mon Sep 17 00:00:00 2001 From: Katie Windau <153020235+kwindau@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:41:11 -0600 Subject: [PATCH 29/29] Mark managed backfill as complete (#6696) --- .../telemetry_derived/event_aggregates_v1/backfill.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/backfill.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/backfill.yaml index b3f99353fbb..bd0bf93774e 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/backfill.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/event_aggregates_v1/backfill.yaml @@ -4,6 +4,6 @@ reason: https://mozilla-hub.atlassian.net/browse/DENG-6892 watchers: - kwindau@mozilla.com - status: Initiate + status: Complete shredder_mitigation: false override_retention_limit: true