Benchmarking tests #175
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarking tests | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 0 * * *" # every day at midnight | |
| env: | |
| DIFFUSERS_IS_CI: yes | |
| HF_HUB_ENABLE_HF_TRANSFER: 1 | |
| HF_HOME: /mnt/cache | |
| OMP_NUM_THREADS: 8 | |
| MKL_NUM_THREADS: 8 | |
| BASE_PATH: benchmark_outputs | |
| jobs: | |
| torch_models_cuda_benchmark_tests: | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_BENCHMARK }} | |
| name: Torch Core Models CUDA Benchmarking Tests | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| runs-on: | |
| group: aws-g6e-xlarge-plus | |
| container: | |
| image: diffusers/diffusers-pytorch-cuda | |
| options: --shm-size "16gb" --ipc host --gpus 0 | |
| steps: | |
| - name: Checkout diffusers | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 2 | |
| - name: NVIDIA-SMI | |
| run: | | |
| nvidia-smi | |
| - name: Install dependencies | |
| run: | | |
| apt update | |
| apt install -y libpq-dev postgresql-client | |
| python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" | |
| python -m uv pip install -e [quality,test] | |
| python -m uv pip install -r benchmarks/requirements.txt | |
| - name: Environment | |
| run: | | |
| python utils/print_env.py | |
| - name: Diffusers Benchmarking | |
| env: | |
| HF_TOKEN: ${{ secrets.DIFFUSERS_BOT_TOKEN }} | |
| run: | | |
| cd benchmarks && python run_all.py && python push_results.py | |
| mkdir ${BASE_PATH} && mv *.csv ${BASE_PATH} | |
| - name: Test suite reports artifacts | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark_test_reports | |
| path: benchmarks/benchmark_outputs | |
| # TODO: enable this once the connection problem has been resolved. | |
| # - name: Update benchmarking results to DB | |
| # env: | |
| # PGDATABASE: metrics | |
| # PGHOST: ${{ secrets.DIFFUSERS_BENCHMARKS_PGHOST }} # TODO | |
| # PGUSER: transformers_benchmarks | |
| # PGPASSWORD: ${{ secrets.DIFFUSERS_BENCHMARKS_PGPASSWORD }} | |
| # run: cd benchmarks && python populate_into_db.py | |
| - name: Report success status | |
| if: ${{ success() }} | |
| run: | | |
| pip install requests && python utils/notify_benchmarking_status.py --status=success | |
| - name: Report failure status | |
| if: ${{ failure() }} | |
| run: | | |
| pip install requests && python utils/notify_benchmarking_status.py --status=failure |