Daily get UP-FRONT & Hello!Project YouTube view count #1119
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will install Python dependencies, run tests and lint with a single version of Python | |
| # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions | |
| name: Daily get UP-FRONT & Hello!Project YouTube view count | |
| env: | |
| ACCESS_TOKEN: ${{secrets.ACCESS_TOKEN}} | |
| ACCESS_TOKEN_SECRET: ${{secrets.ACCESS_TOKEN_SECRET}} | |
| API_KEY: ${{secrets.API_KEY}} | |
| API_SECRET: ${{secrets.API_SECRET}} | |
| YTV3_API_KEY: ${{secrets.YTV3_API_KEY}} | |
| DEBUG: NO | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| FIREBASE_CREDENTIAL: ${{ secrets.FIREBASE_CREDENTIAL }} | |
| GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| TZ: 'Asia/Tokyo' | |
| on: | |
| schedule: | |
| - cron: '0 23 * * *' | |
| workflow_dispatch: | |
| # This workflow will install Python dependencies, run tests and lint with a single version of Python | |
| # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions | |
| jobs: | |
| crawl: | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Firefox | |
| run: bash < firefox-install.sh | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install -q -U pip | |
| pip install -q uv | |
| uv -q pip compile requirements.in -o requirements.txt | |
| pip install -qr requirements.txt | |
| pip freeze | |
| rm -rf images/* | |
| rm -rf html/* | |
| - name: get today data | |
| id: main-async-py | |
| run: | | |
| python main.async.py 2>&1 | tee -a ./out.txt | |
| continue-on-error: true | |
| - name: generate graph | |
| id: graph_gen-py | |
| run: | | |
| python graph_gen.py 2>&1 | tee -a ./out.txt | |
| continue-on-error: true | |
| - name: generate table & markdown data | |
| id: table_gen_selenium-py | |
| run: | | |
| python table_gen_selenium.py 2>&1 | tee -a ./out.txt | |
| continue-on-error: true | |
| - name: debug-ls | |
| run: | | |
| find . -maxdepth 2 -type f | |
| pwd | |
| - name: Cache markdown data | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| markdown.pickle | |
| save.sqlite | |
| key: ${{ github.sha }}-cache | |
| - name: debug-ls | |
| run: | | |
| find . -maxdepth 2 -type f | |
| pwd | |
| - name: tweet | |
| id: tweet-py | |
| run: | | |
| python tweet.py 2>&1 | tee -a ./out.txt | |
| continue-on-error: true | |
| - name: push updates | |
| run: | | |
| git remote set-url origin https://github-actions:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY} | |
| git config --global core.quotepath off | |
| git config --global user.name "${GITHUB_ACTOR}" | |
| git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" | |
| if (git diff --shortstat | grep '[0-9]'); then \ | |
| git status; \ | |
| git add save.sqlite graph/* html/* table/* tsvs/* gemini-cache.json group_list.json -f; \ | |
| git commit -m "Daily updates"; \ | |
| git push origin HEAD:${GITHUB_REF} -f ; \ | |
| fi | |
| - name: Print status | |
| run: | | |
| echo "${{steps.main-async-py.outcome}}" | |
| echo "${{steps.graph_gen-py.outcome}}" | |
| echo "${{steps.table_gen-py.outcome}}" | |
| echo "${{steps.tweet-py.outcome}}" | |
| - name: Error Handling | |
| if: steps.main-async-py.outcome == 'failure' || steps.graph_gen-py.outcome == 'failure' || steps.table_gen-py.outcome == 'failure' || steps.tweet-py.outcome == 'failure' | |
| run: | | |
| python post_processing.py 2>&1 | tee -a ./out.txt | |
| exit 1 | |
| - name: Upload Log | |
| if: steps.main-async-py.outcome == 'success' && steps.graph_gen-py.outcome == 'success' && steps.table_gen-py.outcome == 'success' && steps.tweet-py.outcome == 'success' | |
| run: | | |
| python -c 'import firebase_storage_upload;firebase_storage_upload.upload_log()' 2>&1 | tee -a ./out.txt | |
| excel_export: | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| needs: | |
| - crawl | |
| steps: | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install -q -U pip | |
| pip install -q pandas lxml styleframe openpyxl | |
| - name: Version | |
| id: versioning | |
| run: echo "tag_name=`date '+%Y.%m.%d'`" >> "$GITHUB_OUTPUT" | |
| # - name: convert SQLite to Excel | |
| # run: python excel_exporter.py | |
| - run: rm save.sqlite | |
| - name: debug-ls | |
| run: | | |
| find . -maxdepth 2 -type f | |
| pwd | |
| - name: Restore markdown data | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| markdown.pickle | |
| save.sqlite | |
| key: ${{ github.sha }}-cache | |
| - name: convert SQLite to Excel | |
| run: python excel_exporter.py | |
| - name: generate daily report & release note | |
| run: | | |
| python daily_report.py $GITHUB_SHA | |
| grep -e "^[#🥇🥈🥉]" daily_report.md | sed -e '/^#/! s/\([~|#]\)/\\\1/g' > release_note.md | |
| - name: Tagging | |
| run: | | |
| git config --global user.name "${GITHUB_ACTOR}" | |
| git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" | |
| git add export.xlsx | |
| git commit -m "convert SQLite to Excel" | |
| git tag ${{ steps.versioning.outputs.tag_name }} | |
| git push origin ${{ steps.versioning.outputs.tag_name }} | |
| - name: Release | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| files: | | |
| export.xlsx | |
| daily_report.md | |
| body_path: release_note.md | |
| tag_name: ${{ steps.versioning.outputs.tag_name }} | |
| name: ${{ steps.versioning.outputs.tag_name }} | |
| generate_release_notes: false | |