etl_to_neon.yml

name: ETL to neon

# Workflow triggers
on:
  schedule:
    - cron: "0 2 * * 0"  # Runs at 2am UTC every Sunday
  push:
    branches:
        - add_etl_workflow    
  workflow_dispatch:  # Allows manual triggering of the workflow

jobs:
  neon-etl:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.12"

      - name: Install dependencies
        run: |
          pip install -r requirements.txt
      - name: Get Run ID of Most Recent Successful Run
        id: get_run_id
        run: |
          response=$(curl -s -H "Authorization: token ${{ secrets.GH_PAT }}" \
          "https://api.github.com/repos/sfbrigade/datasci-earthquake/actions/workflows/env_vars.yml/runs?status=completed&conclusion=success")
          run_id=$(echo $response | jq '.workflow_runs[0].id')
          echo "Run ID: $run_id"
          echo "run_id=$run_id" >> $GITHUB_ENV        
      - name: Download .env Artifact 
        uses: actions/download-artifact@v4
        with:
          name: env-file
          github-token: ${{ secrets.GH_PAT }}
          repository: sfbrigade/datasci-earthquake
          run-id: ${{ env.run_id }}                  

      - name: ETL data to Neon DB
        run: |
          python -m backend.etl.tsunami_data_handler