Merge branch 'main' into shredder_mitigation_kpi_search

dags.yaml

@@ -1947,18 +1947,18 @@ bqetl_firefox_installer_aggregates:
   tags:
     - impact/tier_3
 
-bqetl_newtab_engagement_hourly:
+bqetl_newtab_interactions_hourly:
   schedule_interval: hourly
   description: |
     This DAG aggregates Newtab engagement data - on an hourly cadence
   default_args:
     owner: cbeck@mozilla.com
     email: ["cbeck@mozilla.com"]
-    start_date: "2024-12-09"
+    start_date: "2024-12-17"
     retries: 2
     retry_delay: 5m
   tags:
-    - impact/tier_2
+    - impact/tier_1
 
 bqetl_fx_health_ind_dashboard:
   description: |
@@ -1978,3 +1978,22 @@ bqetl_fx_health_ind_dashboard:
     - impact/tier_2
   repo: bigquery-etl
   schedule_interval: 0 16 * * *
+
+bqetl_event_aggregates:
+  description: |
+    This DAG builds the event_aggregates table
+  default_args:
+    depends_on_past: false
+    owner: kwindau@mozilla.com
+    email:
+      - telemetry-alerts@mozilla.com
+      - kwindau@mozilla.com
+    email_on_failure: true
+    email_on_retry: false
+    start_date: "2024-12-19"
+    retries: 2
+    retry_delay: 5m
+  tags:
+    - impact/tier_3
+  repo: bigquery-etl
+  schedule_interval: 40 16 * * *

requirements.in

@@ -1,6 +1,6 @@
-attrs==24.2.0
+attrs==24.3.0
 authlib==1.3.2
-bigeye-sdk==0.4.93
+bigeye-sdk==0.4.94
 black==24.10.0
 cattrs==24.1.2
 click==8.1.7
@@ -20,7 +20,7 @@ PyJWT==2.10.1
 markdown-include==0.8.1
 mdx_truly_sane_lists==1.3
 mkdocs==1.6.1
-mkdocs-material==9.5.48
+mkdocs-material==9.5.49
 mkdocs-awesome-pages-plugin==2.9.3
 mozilla-metric-config-parser==2024.11.1
 mozilla-schema-generator==0.5.1
@@ -29,7 +29,7 @@ pathos==0.3.3
 pip-tools==7.4.1
 pre-commit==4.0.1
 pyarrow==18.1.0
-pytest-black==0.3.12
+pytest-black==0.6.0
 pytest-flake8==1.3.0
 pytest-isort==4.0.0
 pytest-mypy==0.10.3

requirements.txt

@@ -109,9 +109,9 @@ annotated-types==0.7.0 \
     --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \
     --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89
     # via pydantic
-attrs==24.2.0 \
-    --hash=sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346 \
-    --hash=sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2
+attrs==24.3.0 \
+    --hash=sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff \
+    --hash=sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308
     # via
     #   -r requirements.in
     #   aiohttp
@@ -132,9 +132,9 @@ babel==2.12.1 \
 betterproto[compiler]==1.2.5 \
     --hash=sha256:74a3ab34646054f674d236d1229ba8182dc2eae86feb249b8590ef496ce9803d
     # via bigeye-sdk
-bigeye-sdk==0.4.93 \
-    --hash=sha256:29b14882f31f67b4888e58c0e4efe6a3b152abbcc09af131bc38f91b33e93450 \
-    --hash=sha256:77a0333a4c98888d511230503c9e30c0e16441ed455b2c74f9740f7628a6b7e7
+bigeye-sdk==0.4.94 \
+    --hash=sha256:d17e50629ecf3273bc7ca8e4ff9883d8fb59e1111776a02ef9cdc07399e1306c \
+    --hash=sha256:ff230eb0fcfdfd72932ec83975ede59c7f1c2d2fed89286fb70463f3752fcef7
     # via -r requirements.in
 black==24.10.0 \
     --hash=sha256:14b3502784f09ce2443830e3133dacf2c0110d45191ed470ecb04d0f5f6fcb0f \
@@ -764,9 +764,7 @@ jaraco-classes==3.4.0 \
 jeepney==0.8.0 \
     --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \
     --hash=sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755
-    # via
-    #   keyring
-    #   secretstorage
+    # via secretstorage
 jinja2==3.1.4 \
     --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \
     --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d
@@ -934,9 +932,9 @@ mkdocs-get-deps==0.2.0 \
     --hash=sha256:162b3d129c7fad9b19abfdcb9c1458a651628e4b1dea628ac68790fb3061c60c \
     --hash=sha256:2bf11d0b133e77a0dd036abeeb06dec8775e46efa526dc70667d8863eefc6134
     # via mkdocs
-mkdocs-material==9.5.48 \
-    --hash=sha256:a582531e8b34f4c7ed38c29d5c44763053832cf2a32f7409567e0c74749a47db \
-    --hash=sha256:b695c998f4b939ce748adbc0d3bff73fa886a670ece948cf27818fa115dc16f8
+mkdocs-material==9.5.49 \
+    --hash=sha256:3671bb282b4f53a1c72e08adbe04d2481a98f85fed392530051f80ff94a9621d \
+    --hash=sha256:c3c2d8176b18198435d3a3e119011922f3e11424074645c24019c2dcf08a360e
     # via -r requirements.in
 mkdocs-material-extensions==1.3 \
     --hash=sha256:0297cc48ba68a9fdd1ef3780a3b41b534b0d0df1d1181a44676fda5f464eeadc \
@@ -1610,8 +1608,9 @@ pytest==7.4.3 \
     #   pytest-mypy
     #   pytest-pydocstyle
     #   pytest-xdist
-pytest-black==0.3.12 \
-    --hash=sha256:1d339b004f764d6cd0f06e690f6dd748df3d62e6fe1a692d6a5500ac2c5b75a5
+pytest-black==0.6.0 \
+    --hash=sha256:7eb747f54b6c997497b5cbc66a988be114b92016dbfa66d210d1d1f9f6b2dc76 \
+    --hash=sha256:ecb77455f379805cb4bd8f45a813a3754c3bbee3199adf1b3665c0dfd086b511
     # via -r requirements.in
 pytest-flake8==1.3.0 \
     --hash=sha256:88fb35562ce32d915c6ba41ef0d5e1cfcdd8ff884a32b7d46aa99fc77a3d1fe6 \
@@ -2127,9 +2126,7 @@ s3transfer==0.10.2 \
 secretstorage==3.3.3 \
     --hash=sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77 \
     --hash=sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99
-    # via
-    #   bigeye-sdk
-    #   keyring
+    # via bigeye-sdk
 shellingham==1.5.4 \
     --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \
     --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de

sql/bigconfig.yml

@@ -8,9 +8,9 @@ saved_metric_definitions:
       threshold:
         type: CONSTANT
         upper_bound: 0
-      schedule_frequency:
-        interval_type: MINUTES
-        interval_value: 0
+      metric_schedule:
+        named_schedule:
+          name: default
       lookback:
         lookback_type: DATA_TIME
         lookback_window:
@@ -24,9 +24,9 @@ saved_metric_definitions:
       threshold:
         type: CONSTANT
         upper_bound: 0
-      schedule_frequency:
-        interval_type: MINUTES
-        interval_value: 0
+      metric_schedule:
+        named_schedule:
+          name: default
       rct_overrides:
         - submission_date
     - saved_metric_id: freshness_fail
@@ -71,9 +71,9 @@ saved_metric_definitions:
         type: CONSTANT
         lower_bound: 1
         upper_bound: 1
-      schedule_frequency:
-        interval_type: MINUTES
-        interval_value: 0
+      metric_schedule:
+        named_schedule:
+          name: default
       lookback:
         lookback_type: DATA_TIME
         lookback_window:
@@ -88,9 +88,9 @@ saved_metric_definitions:
         type: CONSTANT
         lower_bound: 2
         upper_bound: 2
-      schedule_frequency:
-        interval_type: MINUTES
-        interval_value: 0
+      metric_schedule:
+        named_schedule:
+          name: default
       lookback:
         lookback_type: DATA_TIME
         lookback_window:
@@ -105,9 +105,9 @@ saved_metric_definitions:
         type: CONSTANT
         lower_bound: 1
         upper_bound: 1
-      schedule_frequency:
-        interval_type: MINUTES
-        interval_value: 0
+      metric_schedule:
+        named_schedule:
+          name: default
       lookback:
         lookback_type: DATA_TIME
         lookback_window:

sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_beta_aggregates_v1/metadata.yaml

@@ -10,6 +10,6 @@ scheduling:
   dag_name: bqetl_glam_refresh_aggregates
   date_partition_parameter: null
   depends_on:
-  - task_id: export_org_mozilla_fenix_glam_beta
+  - task_id: query_org_mozilla_fenix_glam_beta__extract_probe_counts_v1
     dag_name: glam_fenix
     execution_delta: 6h

sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_nightly_aggregates_v1/metadata.yaml

@@ -10,6 +10,6 @@ scheduling:
   dag_name: bqetl_glam_refresh_aggregates
   date_partition_parameter: null
   depends_on:
-  - task_id: export_org_mozilla_fenix_glam_nightly
+  - task_id: query_org_mozilla_fenix_glam_nightly__extract_probe_counts_v1
     dag_name: glam_fenix
     execution_delta: 6h

sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_release_aggregates_v1/metadata.yaml

@@ -10,6 +10,6 @@ scheduling:
   dag_name: bqetl_glam_refresh_aggregates
   date_partition_parameter: null
   depends_on:
-  - task_id: export_org_mozilla_fenix_glam_release
+  - task_id: query_org_mozilla_fenix_glam_release__extract_probe_counts_v1
     dag_name: glam_fenix
     execution_delta: 6h

sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fog_beta_aggregates_v1/metadata.yaml

@@ -10,6 +10,6 @@ scheduling:
   dag_name: bqetl_glam_refresh_aggregates
   date_partition_parameter: null
   depends_on:
-  - task_id: export_firefox_desktop_glam_beta
+  - task_id: firefox_desktop_glam_beta_done
     dag_name: glam_fog
     execution_delta: 6h

sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fog_nightly_aggregates_v1/metadata.yaml

@@ -10,6 +10,6 @@ scheduling:
   dag_name: bqetl_glam_refresh_aggregates
   date_partition_parameter: null
   depends_on:
-  - task_id: export_firefox_desktop_glam_nightly
+  - task_id: firefox_desktop_glam_nightly_done
     dag_name: glam_fog
     execution_delta: 6h

sql/moz-fx-data-shared-prod/addon_moderations_derived/cinder_decisions_raw_v1/metadata.yaml

@@ -15,8 +15,5 @@ scheduling:
 bigquery:
   time_partitioning:
     type: day
-    field: 'date'
+    field: date
     require_partition_filter: false
-    expiration_days: null
-  range_partitioning: null
-references: {}