fix dependabot issues #78
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Transformer Docker Images | ||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| build_echo_image: | ||
| description: 'Build echo transformer image' | ||
| required: true | ||
| type: boolean | ||
| default: false | ||
| build_hello_world_image: | ||
| description: 'Build hello_world transformer image' | ||
| required: true | ||
| type: boolean | ||
| default: false | ||
| build_md5_image: | ||
| description: 'Build md5 transformer image' | ||
| required: true | ||
| type: boolean | ||
| default: false | ||
| build_hash_with_args_image: | ||
| description: 'Build hash_with_args transformer image' | ||
| required: true | ||
| type: boolean | ||
| default: false | ||
| build_tar2tf_image: | ||
| description: 'Build tar2tf transformer image' | ||
| required: true | ||
| type: boolean | ||
| default: false | ||
| build_compress_image: | ||
| description: 'Build compress transformer image' | ||
| required: true | ||
| type: boolean | ||
| default: false | ||
| build_ffmpeg_image: | ||
| description: 'Build NeMo/FFmpeg transformer image' | ||
| required: true | ||
| type: boolean | ||
| default: false | ||
| build_audio_split_image: | ||
| description: 'Build NeMo/audio_split_consolidate transformer image' | ||
| required: true | ||
| type: boolean | ||
| default: false | ||
| build_torchvision_preprocess_image: | ||
| description: 'Build torchvision transformer image' | ||
| required: true | ||
| type: boolean | ||
| default: false | ||
| build_keras_preprocess_image: | ||
| description: 'Build keras transformer image' | ||
| required: true | ||
| type: boolean | ||
| default: false | ||
| build_face_detection_image: | ||
| description: 'Build face detection transformer image' | ||
| required: true | ||
| type: boolean | ||
| default: false | ||
| push: | ||
| paths: | ||
| - 'transformers/**' | ||
| env: | ||
| ECHO_ENABLE: ${{ github.event.inputs.build_echo_image }} | ||
| HELLO_WORLD_ENABLE: ${{ github.event.inputs.build_hello_world_image }} | ||
| MD5_ENABLE: ${{ github.event.inputs.build_md5_image }} | ||
| HASH_WITH_ARGS_ENABLE: ${{ github.event.inputs.build_hash_with_args_image }} | ||
| TAR2TF_ENABLE: ${{ github.event.inputs.build_tar2tf_image }} | ||
| COMPRESS_ENABLE: ${{ github.event.inputs.build_compress_image }} | ||
| FFMPEG_ENABLE: ${{ github.event.inputs.build_ffmpeg_image }} | ||
| TORCHVISION_ENABLE: ${{ github.event.inputs.build_torchvision_preprocess_image }} | ||
| KERAS_ENABLE: ${{ github.event.inputs.build_keras_preprocess_image }} | ||
| FACE_DETECTION_ENABLE: ${{ github.event.inputs.build_face_detection_image }} | ||
| AUDIO_SPLIT_ENABLE: ${{ github.event.inputs.build_audio_split_image }} | ||
| jobs: | ||
| build-and-test: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Docker Login | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| username: ${{ secrets.DOCKERHUB_USERNAME }} | ||
| password: ${{ secrets.DOCKERHUB_TOKEN }} | ||
| - name: Setup Minikube | ||
| uses: medyagh/setup-minikube@v0.0.19 | ||
| with: | ||
| driver: docker | ||
| container-runtime: docker | ||
| cpus: max | ||
| memory: max | ||
| - name: Run AIStore in Minikube | ||
| run: | | ||
| git clone https://github.com/NVIDIA/aistore.git | ||
| cd aistore/deploy/dev/k8s | ||
| ./minimal.sh | ||
| export AIS_ENDPOINT="http://$(minikube ip):8080" | ||
| - name: Setup Python 3.11 | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: 3.11 | ||
| - name: Check Path for Changes | ||
| id: paths_filter | ||
| if: ${{ github.event_name != 'workflow_dispatch' }} | ||
| uses: dorny/paths-filter@v3 | ||
| with: | ||
| filters: | | ||
| echo: | ||
| - transformers/echo/**/* | ||
| - transformers/go_echo/**/* | ||
| - transformers/tests/test_echo.py | ||
| - transformers/tests/test_go_echo.py | ||
| hello_world: | ||
| - transformers/hello_world/**/* | ||
| - transformers/go_hello_world/**/* | ||
| - transformers/tests/test_hello_world.py | ||
| md5: | ||
| - transformers/md5/**/* | ||
| - transformers/tests/test_md5.py | ||
| hash_with_args: | ||
| - transformers/hash_with_args/**/* | ||
| - transformers/tests/test_hash_with_args.py | ||
| tar2tf: | ||
| - transformers/tar2tf/**/* | ||
| - transformers/tests/test_tar2tf.py | ||
| compress: | ||
| - transformers/compress/**/* | ||
| - transformers/tests/test_compress.py | ||
| ffmpeg: | ||
| - transformers/ffmpeg/**/* | ||
| - transformers/tests/test_ffmpeg.py | ||
| torchvision: | ||
| - transformers/torchvision_preprocess/**/* | ||
| - transformers/tests/test_torchvision_transformer.py | ||
| keras: | ||
| - transformers/keras_preprocess/**/* | ||
| - transformers/tests/test_keras_transformer.py | ||
| face_detection: | ||
| - transformers/face_detection/**/* | ||
| - transformers/tests/test_face_detection.py | ||
| - name: Update var ECHO_ENABLE | ||
| if: steps.paths_filter.outputs.echo == 'true' | ||
| run: echo "ECHO_ENABLE=true" >> $GITHUB_ENV | ||
| - name: Update var HELLO_WORLD_ENABLE | ||
| if: steps.paths_filter.outputs.hello_world == 'true' | ||
| run: echo "HELLO_WORLD_ENABLE=true" >> $GITHUB_ENV | ||
| - name: Update var MD5_ENABLE | ||
| if: steps.paths_filter.outputs.md5 == 'true' | ||
| run: echo "MD5_ENABLE=true" >> $GITHUB_ENV | ||
| - name: Update var HASH_WITH_ARGS_ENABLE | ||
| if: steps.paths_filter.outputs.hash_with_args == 'true' | ||
| run: echo "HASH_WITH_ARGS_ENABLE=true" >> $GITHUB_ENV | ||
| - name: Update var TAR2TF_ENABLE | ||
| if: steps.paths_filter.outputs.tar2tf == 'true' | ||
| run: echo "TAR2TF_ENABLE=true" >> $GITHUB_ENV | ||
| - name: Update var COMPRESS_ENABLE | ||
| if: steps.paths_filter.outputs.compress == 'true' | ||
| run: echo "COMPRESS_ENABLE=true" >> $GITHUB_ENV | ||
| - name: Update var FFMPEG_ENABLE | ||
| if: steps.paths_filter.outputs.ffmpeg == 'true' | ||
| run: echo "FFMPEG_ENABLE=true" >> $GITHUB_ENV | ||
| - name: Update var TORCHVISION_ENABLE | ||
| if: steps.paths_filter.outputs.torchvision == 'true' | ||
| run: echo "TORCHVISION_ENABLE=true" >> $GITHUB_ENV | ||
| - name: Update var KERAS_ENABLE | ||
| if: steps.paths_filter.outputs.keras == 'true' | ||
| run: echo "KERAS_ENABLE=true" >> $GITHUB_ENV | ||
| - name: Update var FACE_DETECTION_ENABLE | ||
| if: steps.paths_filter.outputs.face_detection == 'true' | ||
| run: echo "FACE_DETECTION_ENABLE=true" >> $GITHUB_ENV | ||
| - name: Install Python Dependencies for Testing | ||
| run: | | ||
| make -B -C $GITHUB_WORKSPACE/transformers common_deps | ||
| - name: Build and Test ECHO | ||
| if: ${{ env.ECHO_ENABLE == 'true' }} | ||
| run: | | ||
| docker system prune -a -f --volumes | ||
| pushd $GITHUB_WORKSPACE/transformers/tests | ||
| export GIT_TEST="true" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/hello_world all | ||
| make -B -C $GITHUB_WORKSPACE/transformers/go_hello_world all | ||
| pytest -v test_echo.py test_go_echo.py | ||
| export GIT_TEST="false" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/echo all | ||
| make -B -C $GITHUB_WORKSPACE/transformers/go_echo all | ||
| pytest -v test_echo.py test_go_echo.py | ||
| popd | ||
| - name: Build and Test HELLO_WORLD | ||
| if: ${{ env.HELLO_WORLD_ENABLE == 'true' }} | ||
| run: | | ||
| docker system prune -a -f --volumes | ||
| pushd $GITHUB_WORKSPACE/transformers/tests | ||
| export GIT_TEST="true" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/echo all | ||
| make -B -C $GITHUB_WORKSPACE/transformers/go_echo all | ||
| pytest -v test_hello_world.py | ||
| export GIT_TEST="false" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/echo all | ||
| make -B -C $GITHUB_WORKSPACE/transformers/go_echo all | ||
| pytest -v test_hello_world.py | ||
| popd | ||
| - name: Build and Test MD5 | ||
| if: ${{ env.MD5_ENABLE == 'true' }} | ||
| run: | | ||
| docker system prune -a -f --volumes | ||
| pushd $GITHUB_WORKSPACE/transformers/tests | ||
| export GIT_TEST="true" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/md5 all | ||
| pytest -v test_md5.py | ||
| export GIT_TEST="false" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/md5 all | ||
| pytest -v test_md5.py | ||
| popd | ||
| - name: Build and Test HASH_WITH_ARGS | ||
| if: ${{ env.HASH_WITH_ARGS_ENABLE == 'true' }} | ||
| run: | | ||
| docker system prune -a -f --volumes | ||
| pushd $GITHUB_WORKSPACE/transformers/tests | ||
| export GIT_TEST="true" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/hash_with_args all | ||
| pytest -v test_hash_with_args.py | ||
| export GIT_TEST="false" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/hash_with_args all | ||
| pytest -v test_hash_with_args.py | ||
| popd | ||
| - name: Build and Test TAR2TF | ||
| if: ${{ env.TAR2TF_ENABLE == 'true' }} | ||
| run: | | ||
| docker system prune -a -f --volumes | ||
| pushd $GITHUB_WORKSPACE/transformers/tests | ||
| export GIT_TEST="true" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/tar2tf all | ||
| pytest -v test_tar2tf.py | ||
| export GIT_TEST="false" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/tar2tf all | ||
| pytest -v test_tar2tf.py | ||
| popd | ||
| - name: Build and Test COMPRESS | ||
| if: ${{ env.COMPRESS_ENABLE == 'true' }} | ||
| run: | | ||
| docker system prune -a -f --volumes | ||
| pushd $GITHUB_WORKSPACE/transformers/tests | ||
| export GIT_TEST="true" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/compress all | ||
| pytest -v test_compress.py | ||
| export GIT_TEST="false" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/compress all | ||
| pytest -v test_compress.py | ||
| popd | ||
| - name: Build and Test FFmpeg | ||
| if: ${{ env.FFMPEG_ENABLE == 'true' }} | ||
| run: | | ||
| export GIT_TEST="false" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/NeMo/FFmpeg all | ||
| - name: Build and Test Split Audio | ||
| if: ${{ env.AUDIO_SPLIT_ENABLE == 'true' }} | ||
| run: | | ||
| make -B -C $GITHUB_WORKSPACE/transformers/NeMo/audio_split_consolidate/audio_manager all | ||
| make -B -C $GITHUB_WORKSPACE/transformers/NeMo/audio_split_consolidate/audio_splitter all | ||
| - name: Build and Test TORCHVISION | ||
| if: ${{ env.TORCHVISION_ENABLE == 'true' }} | ||
| run: | | ||
| docker system prune -a -f --volumes | ||
| pip install torchvision --index-url https://download.pytorch.org/whl/cpu | ||
| pushd $GITHUB_WORKSPACE/transformers/tests | ||
| export GIT_TEST="true" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/torchvision_preprocess all | ||
| pytest -v test_torchvision_transformer.py | ||
| export GIT_TEST="false" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/torchvision_preprocess all | ||
| pytest -v test_torchvision_transformer.py | ||
| popd | ||
| - name: Build and Test KERAS | ||
| if: ${{ env.KERAS_ENABLE == 'true' }} | ||
| run: | | ||
| docker system prune -a -f --volumes | ||
| pushd $GITHUB_WORKSPACE/transformers/tests | ||
| export GIT_TEST="true" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/keras_preprocess all | ||
| pytest -v test_keras_transformer.py | ||
| export GIT_TEST="false" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/keras_preprocess all | ||
| pytest -v test_keras_transformer.py | ||
| popd | ||
| - name: Build and Test FACE_DETECTION | ||
| if: ${{ env.FACE_DETECTION_ENABLE == 'true' }} | ||
| run: | | ||
| docker system prune -a -f --volumes | ||
| # build image | ||
| pushd $GITHUB_WORKSPACE/transformers/face_detection | ||
| echo ${{ secrets.KAGGLE_CREDS_JSON }} > kaggle_creds.json | ||
| export GIT_TEST="true" | ||
| make -B all | ||
| popd | ||
| # test image | ||
| kaggle datasets download -d sambitmukherjee/caffe-face-detector-opencv-pretrained-model | ||
| unzip caffe-face-detector-opencv-pretrained-model.zip -d $GITHUB_WORKSPACE/transformers/face_detection/model/ | ||
| pushd $GITHUB_WORKSPACE/transformers/tests | ||
| pytest -o log_cli=true --log-cli-level=INFO test_face_detection.py | ||
| popd | ||
| export GIT_TEST="false" | ||
| make -B -C $GITHUB_WORKSPACE/transformers/keras_preprocess all | ||
| env: | ||
| KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }} | ||
| KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }} | ||
