Daily get UP-FRONT & Hello!Project YouTube view count #738
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will install Python dependencies, run tests and lint with a single version of Python | |
| # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions | |
| name: Daily get UP-FRONT & Hello!Project YouTube view count | |
| env: | |
| ACCESS_TOKEN: ${{secrets.ACCESS_TOKEN}} | |
| ACCESS_TOKEN_SECRET: ${{secrets.ACCESS_TOKEN_SECRET}} | |
| API_KEY: ${{secrets.API_KEY}} | |
| API_SECRET: ${{secrets.API_SECRET}} | |
| YTV3_API_KEY: ${{secrets.YTV3_API_KEY}} | |
| DEBUG: NO | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| FIREBASE_CREDENTIAL: ${{ secrets.FIREBASE_CREDENTIAL }} | |
| TZ: 'Asia/Tokyo' | |
| on: | |
| schedule: | |
| - cron: '0 23 * * *' | |
| # This workflow will install Python dependencies, run tests and lint with a single version of Python | |
| # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions | |
| jobs: | |
| crawl: | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Firefox | |
| run: bash < firefox-install.sh | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install -q -U pip | |
| pip install -q pipenv | |
| pipenv install -q | |
| pip freeze | |
| rm -rf images/* | |
| rm -rf html/* | |
| - name: get today data | |
| id: main-async-py | |
| run: | | |
| pipenv run python main.async.py 2>&1 | tee -a ./out.txt | |
| continue-on-error: true | |
| - name: generate graph | |
| id: graph_gen-py | |
| run: | | |
| pipenv run python graph_gen.py 2>&1 | tee -a ./out.txt | |
| continue-on-error: true | |
| - name: generate table & markdown data | |
| id: table_gen-py | |
| run: | | |
| pipenv run python table_gen.py 2>&1 | tee -a ./out.txt | |
| continue-on-error: true | |
| - name: debug-ls | |
| run: | | |
| find . -maxdepth 2 -type f | |
| pwd | |
| - name: Cache markdown data | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| markdown.pickle | |
| save.sqlite | |
| key: ${{ github.sha }}-cache | |
| - name: debug-ls | |
| run: | | |
| find . -maxdepth 2 -type f | |
| pwd | |
| - name: tweet | |
| id: tweet-py | |
| run: | | |
| pipenv run python tweet.py 2>&1 | tee -a ./out.txt | |
| continue-on-error: true | |
| - name: push updates | |
| run: | | |
| git remote set-url origin https://github-actions:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY} | |
| git config --global user.name "${GITHUB_ACTOR}" | |
| git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" | |
| if (git diff --shortstat | grep '[0-9]'); then \ | |
| git status; \ | |
| git add save.sqlite graph/* html/* table/* tsvs/* -f; \ | |
| git commit -m "Daily updates"; \ | |
| git push origin HEAD:${GITHUB_REF} -f ; \ | |
| fi | |
| - name: Print status | |
| run: | | |
| echo "${{steps.main-async-py.outcome}}" | |
| echo "${{steps.graph_gen-py.outcome}}" | |
| echo "${{steps.table_gen-py.outcome}}" | |
| echo "${{steps.tweet-py.outcome}}" | |
| - name: Error Handling | |
| if: steps.main-async-py.outcome == 'failure' || steps.graph_gen-py.outcome == 'failure' || steps.table_gen-py.outcome == 'failure' || steps.tweet-py.outcome == 'failure' | |
| run: | | |
| pipenv run post-processing | |
| exit 1 | |
| - name: Upload Log | |
| if: steps.main-async-py.outcome == 'success' && steps.graph_gen-py.outcome == 'success' && steps.table_gen-py.outcome == 'success' && steps.tweet-py.outcome == 'success' | |
| run: | | |
| pipenv run upload-log | |
| excel_export: | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| needs: | |
| - crawl | |
| steps: | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install -q -U pip | |
| pip install -q pandas lxml styleframe openpyxl | |
| - name: Version | |
| id: versioning | |
| run: echo "tag_name=`date '+%Y.%m.%d'`" >> "$GITHUB_OUTPUT" | |
| # - name: convert SQLite to Excel | |
| # run: python excel_exporter.py | |
| - run: rm save.sqlite | |
| - name: debug-ls | |
| run: | | |
| find . -maxdepth 2 -type f | |
| pwd | |
| - name: Restore markdown data | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| markdown.pickle | |
| save.sqlite | |
| key: ${{ github.sha }}-cache | |
| - name: convert SQLite to Excel | |
| run: python excel_exporter.py | |
| - name: generate daily report & release note | |
| run: | | |
| python daily_report.py $GITHUB_SHA | |
| grep -e "^[#🥇🥈🥉]" daily_report.md | sed -e '/^#/! s/\([~|#]\)/\\\1/g' > release_note.md | |
| - name: Tagging | |
| run: | | |
| git config --global user.name "${GITHUB_ACTOR}" | |
| git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" | |
| git add export.xlsx | |
| git commit -m "convert SQLite to Excel" | |
| git tag ${{ steps.versioning.outputs.tag_name }} | |
| git push origin ${{ steps.versioning.outputs.tag_name }} | |
| - name: Release | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| files: | | |
| export.xlsx | |
| daily_report.md | |
| body_path: release_note.md | |
| tag_name: ${{ steps.versioning.outputs.tag_name }} | |
| name: ${{ steps.versioning.outputs.tag_name }} | |
| generate_release_notes: false | |