daily.yml

# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Daily get UP-FRONT & Hello!Project YouTube view count

env:
  ACCESS_TOKEN: ${{secrets.ACCESS_TOKEN}}
  ACCESS_TOKEN_SECRET: ${{secrets.ACCESS_TOKEN_SECRET}}
  API_KEY: ${{secrets.API_KEY}}
  API_SECRET: ${{secrets.API_SECRET}}
  YTV3_API_KEY: ${{secrets.YTV3_API_KEY}}
  DEBUG: NO
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  FIREBASE_CREDENTIAL: ${{ secrets.FIREBASE_CREDENTIAL }}
  TZ: 'Asia/Tokyo'



on:
  schedule:
    - cron: '0 23 * * *'

# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
jobs:
  crawl:
    timeout-minutes: 30

    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - name: Install Firefox
        run: bash < firefox-install.sh
        
      - name: Set up Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"
          
      - name: Install dependencies
        run: |
          python -m pip install -q -U pip
          pip install -q pipenv
          pipenv install -q
          pip freeze
          rm -rf images/*
          rm -rf html/*          
        
        
      - name: get today data
        id: main-async-py
        run: |
          pipenv run python main.async.py 2>&1 | tee -a ./out.txt
        continue-on-error: true
        
        
      - name: generate graph
        id: graph_gen-py
        run: |
          pipenv run python graph_gen.py 2>&1 | tee -a ./out.txt
        continue-on-error: true


      - name: generate table & markdown data
        id: table_gen-py
        run: |
          pipenv run python table_gen.py 2>&1 | tee -a ./out.txt
        continue-on-error: true

      - name: debug-ls
        run: |
          find . -maxdepth 2 -type f
          pwd


      - name: Cache markdown data
        uses: actions/cache@v4
        with:
          path: |
            markdown.pickle
            save.sqlite
          key: ${{ github.sha }}-cache

      - name: debug-ls
        run: |
          find . -maxdepth 2 -type f
          pwd

      - name: tweet
        id: tweet-py
        run: |
          pipenv run python tweet.py 2>&1 | tee -a ./out.txt
        continue-on-error: true
        
      - name: push updates
        run: |
          git remote set-url origin https://github-actions:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}
          git config --global user.name "${GITHUB_ACTOR}"
          git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
          if (git diff --shortstat | grep '[0-9]'); then \
            git status; \
            git add save.sqlite graph/* html/* table/* tsvs/* -f; \
            git commit -m "Daily updates"; \
            git push origin HEAD:${GITHUB_REF} -f ; \
          fi


      - name: Print status
        run: |
          echo "${{steps.main-async-py.outcome}}"
          echo "${{steps.graph_gen-py.outcome}}"
          echo "${{steps.table_gen-py.outcome}}"
          echo "${{steps.tweet-py.outcome}}"
        
      - name: Error Handling
        if: steps.main-async-py.outcome == 'failure' || steps.graph_gen-py.outcome == 'failure' || steps.table_gen-py.outcome == 'failure' || steps.tweet-py.outcome == 'failure'
        run: |
          pipenv run post-processing
          exit 1

      - name: Upload Log
        if: steps.main-async-py.outcome == 'success' && steps.graph_gen-py.outcome == 'success' && steps.table_gen-py.outcome == 'success' && steps.tweet-py.outcome == 'success'
        run: |
          pipenv run upload-log
      
  excel_export:
    timeout-minutes: 10
    runs-on: ubuntu-latest
    needs:
      - crawl
    steps:
      - name: Set up Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Checkout
        uses: actions/checkout@v4

      - name: Install dependencies
        run: |
          python -m pip install -q -U pip
          pip install -q pandas lxml styleframe openpyxl

      - name: Version
        id: versioning
        run: echo "tag_name=`date '+%Y.%m.%d'`" >> "$GITHUB_OUTPUT"

      # - name: convert SQLite to Excel
        # run: python excel_exporter.py


      - run: rm save.sqlite

      - name: debug-ls
        run: |
          find . -maxdepth 2 -type f
          pwd


      - name: Restore markdown data
        uses: actions/cache@v4
        with:
          path: |
            markdown.pickle
            save.sqlite
          key: ${{ github.sha }}-cache


      - name: convert SQLite to Excel
        run: python excel_exporter.py


      - name: generate daily report & release note
        run: |
          python daily_report.py $GITHUB_SHA
          grep -e "^[#🥇🥈🥉]" daily_report.md | sed -e '/^#/! s/\([~|#]\)/\\\1/g' > release_note.md


      - name: Tagging
        run: |
          git config --global user.name "${GITHUB_ACTOR}"
          git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
          git add export.xlsx
          git commit -m "convert SQLite to Excel"
          git tag ${{ steps.versioning.outputs.tag_name }}
          git push origin ${{ steps.versioning.outputs.tag_name }}         
          

      - name: Release
        uses: softprops/action-gh-release@v1
        with:
          files: |
            export.xlsx
            daily_report.md
          body_path: release_note.md
          tag_name: ${{ steps.versioning.outputs.tag_name }}
          name: ${{ steps.versioning.outputs.tag_name }}
          generate_release_notes: false