Determine data refresh pool by environment variable with default (#2352)

Author: AetherUnbound

catalog/dags/data_refresh/data_refresh_task_factory.py

@@ -65,7 +65,7 @@
 logger = logging.getLogger(__name__)
 
 
-DATA_REFRESH_POOL = "data_refresh"
+DATA_REFRESH_POOL = os.getenv("DATA_REFRESH_POOL", "data_refresh")
 
 
 def create_data_refresh_task_group(

catalog/env.template

@@ -106,6 +106,9 @@ OPENVERSE_BUCKET=openverse-storage
 # Seconds to wait before poking for availability of the data refresh pool when running a data_refresh
 # DAG. Used to shorten the time for testing purposes.
 DATA_REFRESH_POKE_INTERVAL=5
+# The pool to use for the data refresh tasks. Defaults to `data_refresh` in production but
+# we override this locally to prevent having to frequently recreate it.
+DATA_REFRESH_POOL=default_pool
 # Number of Retries if DAG task fails to run
 DEFAULT_RETRY_COUNT = 2
 # Whether to enable catchup for dated DAGs, allowing automatic backfill.