diff --git a/.github/workflows/benchmark-chart-baseline.yml b/.github/workflows/benchmark-chart-baseline.yml new file mode 100644 index 0000000000000..0062bd2ffcb05 --- /dev/null +++ b/.github/workflows/benchmark-chart-baseline.yml @@ -0,0 +1,44 @@ +name: Update Charts Benchmark Baseline + +on: + push: + branches: + - 'master' + - 'next' + paths: + - 'packages/x-charts*/**' + - 'test/performance-charts/**' + +jobs: + performance-test: + name: Update Charts Benchmark Baseline + runs-on: ubuntu-latest + permissions: + pull-requests: write + issues: write + contents: read + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4.1.0 + with: + run_install: false + - name: Use Node.js 22.x + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + cache: 'pnpm' # https://github.com/actions/setup-node/blob/main/docs/advanced-usage.md#caching-packages-dependencies + - run: pnpm install --frozen-lockfile + - run: pnpm exec playwright install + - run: pnpm --filter "@mui/x-charts-premium..." build + + - name: Run performance tests + run: pnpm --filter @mui-x-internal/performance-charts test:performance:ci + - run: jq '.commit = "${{ github.sha }}"' test/performance-charts/results.json > test/performance-charts/baseline-results.json + + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: charts-benchmarks-results-${{ github.ref_name }}.json + path: ./test/performance-charts/baseline-results.json + if-no-files-found: error diff --git a/.github/workflows/codspeed.yml b/.github/workflows/benchmark-charts.yml similarity index 53% rename from .github/workflows/codspeed.yml rename to .github/workflows/benchmark-charts.yml index d9a23df3b2c72..e38615a62b5a8 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/benchmark-charts.yml @@ -1,12 +1,6 @@ -name: Benchmarks +name: Performance Test on: - push: - branches: - - 'master' - - 'next' - paths: - - 'packages/x-charts*/**' pull_request: types: - labeled @@ -15,14 +9,15 @@ on: - reopened branches: - 'master' - - 'next' - -permissions: {} jobs: - benchmarks: - name: Benchmarks Charts + performance-test: + name: Benchmark Charts runs-on: ubuntu-latest + permissions: + pull-requests: write + issues: write + contents: read # L1: Run the benchmarks for pushes to the master or next branch and if the changes are in the charts package based on on.push.paths # L2: Run the benchmarks if we add the label 'scope: charts' to the pull request # L3: Run the benchmarks for pull requests with the label 'scope: charts' @@ -33,6 +28,7 @@ jobs: (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'scope: charts') || (github.event_name == 'pull_request' && github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'scope: charts')) }} + steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4.1.0 @@ -44,10 +40,30 @@ jobs: node-version: 22 cache: 'pnpm' # https://github.com/actions/setup-node/blob/main/docs/advanced-usage.md#caching-packages-dependencies - run: pnpm install --frozen-lockfile + - run: pnpm exec playwright install # Ensure we are running on the prod version of our libs - run: pnpm --filter "@mui/x-charts-premium..." build - - name: Run benchmarks - uses: CodSpeedHQ/action@c28fe9fbe7d57a3da1b7834ae3761c1d8217612d + + - name: Download baseline results, if available + run: | + echo "Downloading baseline results from branch '${{ github.base_ref }}' for performance tests" + gh run download --name "charts-benchmarks-results-${{ github.base_ref }}.json" || true + [[ -e ./baseline-results.json ]] && echo "Found baseline, comparing results..." || echo "No baseline results found, skipping comparison" + mv ./baseline-results.json ./test/performance-charts/baseline-results.json || true + + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Run performance tests + run: pnpm --filter @mui-x-internal/performance-charts test:performance:ci + + - name: Compare performance results + uses: actions/github-script@v7 + env: + BASELINE_PATH: ./test/performance-charts/baseline-results.json + COMPARE_PATH: ./test/performance-charts/results.json + THRESHOLD: 0.1 with: - run: pnpm --filter @mui-x-internal/performance-charts test:performance - token: ${{ secrets.CODSPEED_TOKEN }} + script: | + const { default: ciBenchmark } = await import('${{ github.workspace }}/test/performance-charts/scripts/ci-benchmark.js'); + await ciBenchmark({github, context, core}); diff --git a/patches/@vitest__browser.patch b/patches/@vitest__browser.patch new file mode 100644 index 0000000000000..b3d97c18f4513 --- /dev/null +++ b/patches/@vitest__browser.patch @@ -0,0 +1,41 @@ +diff --git a/dist/client/__vitest_browser__/tester-BYDMHqQ9.js b/dist/client/__vitest_browser__/tester-BYDMHqQ9.js +index 9916c1f32e9af801b6d425d1ec785f23e830a236..c748ab51bb7c3f236334f4994bab6c0d76902d12 100644 +--- a/dist/client/__vitest_browser__/tester-BYDMHqQ9.js ++++ b/dist/client/__vitest_browser__/tester-BYDMHqQ9.js +@@ -11,6 +11,27 @@ const assetsURL = function(dep) { + return "/" + dep; + }; + const seen = {}; ++async function getTestRunnerConstructor( ++ config, ++ executor, ++){ ++ if (!config.runner) { ++ const { VitestTestRunner, NodeBenchmarkRunner } ++ = await executor.executeFile(runnersFile) ++ return ( ++ config.mode === 'test' ? VitestTestRunner : NodeBenchmarkRunner ++ ); ++ } ++ const mod = await executor.executeId(config.runner) ++ if (!mod.default && typeof mod.default !== 'function') { ++ throw new Error( ++ `Runner must export a default function, but got ${typeof mod.default} imported from ${ ++ config.runner ++ }`, ++ ) ++ } ++ return mod.default; ++} + const __vitePreload = function preload(baseModule, deps, importerUrl) { + let promise = Promise.resolve(); + if (deps && deps.length > 0) { +@@ -1660,7 +1681,7 @@ async function initiateRunner(state, mocker, config) { + if (cachedRunner) { + return cachedRunner; + } +- const runnerClass = config.mode === "test" ? VitestTestRunner : NodeBenchmarkRunner; ++ const runnerClass = await getTestRunnerConstructor(config, executor); + const BrowserRunner = createBrowserRunner(runnerClass, mocker, state, { + takeCoverage: () => takeCoverageInsideWorker(config.coverage, executor) + }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b13c6cabaa312..d7c72c79181b3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -226,6 +226,11 @@ catalogs: overrides: ast-types: ^0.14.2 +patchedDependencies: + '@vitest/browser': + hash: b6f57ed4fef8f62d7429d29e2a98fce942f03687840af33905b0d82982703516 + path: patches/@vitest__browser.patch + importers: .: @@ -373,7 +378,7 @@ importers: version: 4.6.0(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)) '@vitest/browser': specifier: 'catalog:' - version: 3.2.4(playwright@1.54.1)(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))(vitest@3.2.4) + version: 3.2.4(patch_hash=b6f57ed4fef8f62d7429d29e2a98fce942f03687840af33905b0d82982703516)(playwright@1.54.1)(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))(vitest@3.2.4) '@vitest/coverage-v8': specifier: 'catalog:' version: 3.2.4(@vitest/browser@3.2.4)(vitest@3.2.4) @@ -2132,9 +2137,6 @@ importers: test/performance-charts: devDependencies: - '@codspeed/vitest-plugin': - specifier: ^4.0.1 - version: 4.0.1(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))(vitest@3.2.4) '@emotion/react': specifier: 'catalog:' version: 11.14.0(@types/react@19.1.8)(react@19.1.0) @@ -2147,12 +2149,6 @@ importers: '@mui/x-charts-pro': specifier: workspace:* version: link:../../packages/x-charts-pro/build - '@testing-library/react': - specifier: ^16.3.0 - version: 16.3.0(@testing-library/dom@10.4.0)(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) - '@testing-library/user-event': - specifier: ^14.6.1 - version: 14.6.1(@testing-library/dom@10.4.0) '@vitejs/plugin-react': specifier: 'catalog:' version: 4.6.0(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)) @@ -2161,22 +2157,28 @@ importers: version: 3.10.2(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)) '@vitest/browser': specifier: 'catalog:' - version: 3.2.4(playwright@1.54.1)(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))(vitest@3.2.4) + version: 3.2.4(patch_hash=b6f57ed4fef8f62d7429d29e2a98fce942f03687840af33905b0d82982703516)(playwright@1.54.1)(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))(vitest@3.2.4) + '@vitest/runner': + specifier: ^3.2.4 + version: 3.2.4 '@vitest/ui': specifier: 'catalog:' version: 3.2.4(vitest@3.2.4) - jsdom: - specifier: ^26.1.0 - version: 26.1.0 react: specifier: 'catalog:' version: 19.1.0 react-dom: specifier: 'catalog:' version: 19.1.0(react@19.1.0) + tinybench: + specifier: ^2.9.0 + version: 2.9.0 vitest: specifier: 'catalog:' version: 3.2.4(@types/debug@4.1.12)(@types/node@24.0.13)(@vitest/browser@3.2.4)(@vitest/ui@3.2.4)(jsdom@26.1.0)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0) + vitest-browser-react: + specifier: ^1.0.0 + version: 1.0.0(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(@vitest/browser@3.2.4)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(vitest@3.2.4) test/regressions: devDependencies: @@ -3175,15 +3177,6 @@ packages: resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} engines: {node: '>=18'} - '@codspeed/core@4.0.1': - resolution: {integrity: sha512-fJ53arfgtzCDZa8DuGJhpTZ3Ll9A1uW5nQ2jSJnfO4Hl5MRD2cP8P4vPvIUAGbdbjwCxR1jat6cW8OloMJkJXw==} - - '@codspeed/vitest-plugin@4.0.1': - resolution: {integrity: sha512-aqmrPJzX9cD50UWDsOyih5L5WcEYlNQg3u84sJJ9ZuuLApA51w+LGxk6Xbyb8LJF9n/CwM94HKHV/qArfnvDoQ==} - peerDependencies: - vite: ^4.2.0 || ^5.0.0 || ^6.0.0 - vitest: '>=1.2.2' - '@colors/colors@1.6.0': resolution: {integrity: sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==} engines: {node: '>=0.1.90'} @@ -8085,10 +8078,6 @@ packages: resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} engines: {node: '>=10'} - find-up@6.3.0: - resolution: {integrity: sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - find-up@7.0.0: resolution: {integrity: sha512-YyZM99iHrqLKjmt4LJDj58KI+fYyufRLBSYcqycxf//KpBk9FoewoGX0450m9nB44qrZnovzC2oeP5hUibxc/g==} engines: {node: '>=18'} @@ -12030,6 +12019,22 @@ packages: yaml: optional: true + vitest-browser-react@1.0.0: + resolution: {integrity: sha512-d8oHuCOxG6KL6j2LOgUbU5Jh5ljPIqGoy38Pey1Fzw7BTWxYNsztNFmxbGNrBVSMcRcctDZXBSsgPIl9PLcD8Q==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + peerDependencies: + '@types/react': ^18.0.0 || ^19.0.0 + '@types/react-dom': ^18.0.0 || ^19.0.0 + '@vitest/browser': ^2.1.0 || ^3.0.0 || ^4.0.0-0 + react: ^18.0.0 || ^19.0.0 + react-dom: ^18.0.0 || ^19.0.0 + vitest: ^2.1.0 || ^3.0.0 || ^4.0.0-0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + vitest-fail-on-console@0.7.1: resolution: {integrity: sha512-/PjuonFu7CwUVrKaiQPIGXOtiEv2/Gz3o8MbLmovX9TGDxoRCctRC8CA9zJMRUd6AvwGu/V5a3znObTmlPNTgw==} peerDependencies: @@ -13938,23 +13943,6 @@ snapshots: '@bcoe/v8-coverage@1.0.2': {} - '@codspeed/core@4.0.1': - dependencies: - axios: 1.10.0(debug@4.4.1) - find-up: 6.3.0 - form-data: 4.0.3 - node-gyp-build: 4.8.4 - transitivePeerDependencies: - - debug - - '@codspeed/vitest-plugin@4.0.1(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))(vitest@3.2.4)': - dependencies: - '@codspeed/core': 4.0.1 - vite: 7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0) - vitest: 3.2.4(@types/debug@4.1.12)(@types/node@24.0.13)(@vitest/browser@3.2.4)(@vitest/ui@3.2.4)(jsdom@26.1.0)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0) - transitivePeerDependencies: - - debug - '@colors/colors@1.6.0': {} '@csstools/color-helpers@5.0.2': {} @@ -16929,7 +16917,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@vitest/browser@3.2.4(playwright@1.54.1)(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))(vitest@3.2.4)': + '@vitest/browser@3.2.4(patch_hash=b6f57ed4fef8f62d7429d29e2a98fce942f03687840af33905b0d82982703516)(playwright@1.54.1)(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))(vitest@3.2.4)': dependencies: '@testing-library/dom': 10.4.0 '@testing-library/user-event': 14.6.1(@testing-library/dom@10.4.0) @@ -16965,7 +16953,7 @@ snapshots: tinyrainbow: 2.0.0 vitest: 3.2.4(@types/debug@4.1.12)(@types/node@24.0.13)(@vitest/browser@3.2.4)(@vitest/ui@3.2.4)(jsdom@26.1.0)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0) optionalDependencies: - '@vitest/browser': 3.2.4(playwright@1.54.1)(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))(vitest@3.2.4) + '@vitest/browser': 3.2.4(patch_hash=b6f57ed4fef8f62d7429d29e2a98fce942f03687840af33905b0d82982703516)(playwright@1.54.1)(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))(vitest@3.2.4) transitivePeerDependencies: - supports-color @@ -19575,11 +19563,6 @@ snapshots: locate-path: 6.0.0 path-exists: 4.0.0 - find-up@6.3.0: - dependencies: - locate-path: 7.2.0 - path-exists: 5.0.0 - find-up@7.0.0: dependencies: locate-path: 7.2.0 @@ -24093,6 +24076,16 @@ snapshots: tsx: 4.20.3 yaml: 2.8.0 + vitest-browser-react@1.0.0(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(@vitest/browser@3.2.4)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(vitest@3.2.4): + dependencies: + '@vitest/browser': 3.2.4(patch_hash=b6f57ed4fef8f62d7429d29e2a98fce942f03687840af33905b0d82982703516)(playwright@1.54.1)(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))(vitest@3.2.4) + react: 19.1.0 + react-dom: 19.1.0(react@19.1.0) + vitest: 3.2.4(@types/debug@4.1.12)(@types/node@24.0.13)(@vitest/browser@3.2.4)(@vitest/ui@3.2.4)(jsdom@26.1.0)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0) + optionalDependencies: + '@types/react': 19.1.8 + '@types/react-dom': 19.1.6(@types/react@19.1.8) + vitest-fail-on-console@0.7.1(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))(vitest@3.2.4): dependencies: chalk: 5.3.0 @@ -24127,7 +24120,7 @@ snapshots: optionalDependencies: '@types/debug': 4.1.12 '@types/node': 24.0.13 - '@vitest/browser': 3.2.4(playwright@1.54.1)(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))(vitest@3.2.4) + '@vitest/browser': 3.2.4(patch_hash=b6f57ed4fef8f62d7429d29e2a98fce942f03687840af33905b0d82982703516)(playwright@1.54.1)(vite@7.0.4(@types/node@24.0.13)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))(vitest@3.2.4) '@vitest/ui': 3.2.4(vitest@3.2.4) jsdom: 26.1.0 transitivePeerDependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 1e44b3afced89..845aa9cb99113 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -92,3 +92,6 @@ onlyBuiltDependencies: - nx - sharp - unrs-resolver + +patchedDependencies: + '@vitest/browser': patches/@vitest__browser.patch diff --git a/test/performance-charts/package.json b/test/performance-charts/package.json index 61f34773b6881..3e5a7cfaa88ad 100644 --- a/test/performance-charts/package.json +++ b/test/performance-charts/package.json @@ -4,25 +4,25 @@ "type": "module", "scripts": { "test:performance": "vitest bench", + "test:performance:ci": "vitest bench --run --outputJson results.json", "test:performance:trace": "TRACE=true vitest bench --run", "build": "pnpm --filter=\"@mui/x-charts-premium...\" build", "test:performance:browser": "vitest bench --browser" }, "devDependencies": { - "@codspeed/vitest-plugin": "^4.0.1", "@emotion/react": "catalog:", "@mui/x-charts": "workspace:*", "@mui/x-charts-pro": "workspace:*", "@mui/x-charts-premium": "workspace:*", - "@testing-library/react": "^16.3.0", - "@testing-library/user-event": "^14.6.1", "@vitejs/plugin-react": "catalog:", "@vitejs/plugin-react-swc": "catalog:", "@vitest/browser": "catalog:", "@vitest/ui": "catalog:", - "jsdom": "^26.1.0", + "@vitest/runner": "^3.2.4", + "tinybench": "^2.9.0", "react": "catalog:", "react-dom": "catalog:", - "vitest": "catalog:" + "vitest": "catalog:", + "vitest-browser-react": "^1.0.0" } } diff --git a/test/performance-charts/scripts/ci-benchmark.js b/test/performance-charts/scripts/ci-benchmark.js new file mode 100644 index 0000000000000..9a7560b755f07 --- /dev/null +++ b/test/performance-charts/scripts/ci-benchmark.js @@ -0,0 +1,131 @@ +// @ts-check +/* eslint-disable no-console */ +import fs from 'node:fs/promises'; +// eslint-disable-next-line import/extensions +import { compareResults } from './compare-benchmark-results.js'; + +const COMMENT_MARKER = ''; + +/** @param {any} AsyncFunctionArguments */ +export default async function ciBenchmark({ github, context, core }) { + try { + const { BASELINE_PATH: baselinePath, COMPARE_PATH: comparePath } = + /** @type {any} */ process.env; + const threshold = Number.parseFloat(/** @type {any} */ process.env.THRESHOLD); + + core.info( + `Running performance benchmarks.\nBaseline Path: ${baselinePath}\nCompare Path: ${ + comparePath + }\nThreshold: ${threshold}`, + ); + + if (!baselinePath || !comparePath) { + core.setFailed('Missing required environment variables: BASELINE_PATH, COMPARE_PATH'); + return; + } + + if (!Number.isFinite(threshold) || threshold < 0) { + core.setFailed('Invalid THRESHOLD value. It must be a non-negative number.'); + return; + } + + let /** @type {string} */ compareJson; + try { + compareJson = await readCompareJson(comparePath); + } catch (/** @type {any} */ error) { + core.setFailed(`Error reading compare file: ${error.message}`); + return; + } + + const baselineJson = await readBaselineJson(baselinePath); + + const { results, markdown } = await compareResults(baselineJson, compareJson, threshold); + + if (results.failed.length > 0) { + core.setFailed('Some benchmarks failed.'); + } else if (results.regressed.length > 0) { + core.setFailed('Some benchmarks regressed above threshold.'); + } + + const result = results.failed.length > 0 || results.regressed.length > 0 ? 'fail' : 'pass'; + + const body = `${COMMENT_MARKER} + +## 📊 Performance Test Results + +**Commit:** [${context.sha}](${context.payload.repository.html_url}/commit/${context.sha}) +**Run:** [${context.runId}](${context.payload.repository.html_url}/actions/runs/${context.runId}) +**Baseline:** ${baselineJson ? `[${baselineJson.commit}](${context.payload.repository.html_url}/commit/${baselineJson.commit})` : 'No baseline found'} + +**Result**: ${result === 'pass' ? 'Pass ✅' : 'Fail ❌'} +${result === 'pass' ? 'No significant changes detected.' : 'To acknowledge these changes, merge this PR.'} + + +${markdown}`; + + const comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const existingComment = comments.data.find((/** @type {{ body: string }} */ comment) => + comment.body.includes(COMMENT_MARKER), + ); + + if (existingComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existingComment.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + } catch (/** @type {any} */ error) { + console.error(error); + core.setFailed(`Error running performance benchmarks: ${error.message}`); + } +} + +/** + * @param {string} path + * @returns {Promise} + */ +async function readBaselineJson(path) { + try { + const baselineBuffer = await fs.readFile(path); + + const json = JSON.parse(baselineBuffer.toString('utf-8')); + console.log('Baseline file read successfully:', path); + + return json; + } catch (error) { + console.log('Could not read baseline file:', error); + return null; + } +} + +/** + * @param {string} path + * @returns {Promise} + */ +async function readCompareJson(path) { + try { + const compareBuffer = await fs.readFile(path); + + const json = JSON.parse(compareBuffer.toString('utf-8')); + console.log('Compare file read successfully:', path); + + return json; + } catch (error) { + console.error(`Aborting comparison because compare file could not be read:`, error); + throw new Error('Compare file read error'); + } +} diff --git a/test/performance-charts/scripts/compare-benchmark-results.js b/test/performance-charts/scripts/compare-benchmark-results.js new file mode 100644 index 0000000000000..fef55559db478 --- /dev/null +++ b/test/performance-charts/scripts/compare-benchmark-results.js @@ -0,0 +1,289 @@ +// @ts-check +/* eslint-disable no-console */ +import util from 'node:util'; + +/** + * @param {any} data + * @returns {Array} + */ +function parseBenchmarkResults(data) { + const benchmarks = data.files + ?.flatMap((/** @type {{ groups: any[]; }} */ file) => + file?.groups?.flatMap((g) => g?.benchmarks), + ) + .filter( + ( + /** @type {import('./compare-benchmark-results.types.js').BenchmarkResult | undefined} */ bench, + ) => bench !== undefined, + ); + + return benchmarks; +} + +/** + * + * @param {Array} compareBenchmarks + * @param {Array | null} baselineBenchmarks + * @param {number} threshold + * @returns {import('./compare-benchmark-results.types.js').BenchmarkResults} + */ +function processResults(compareBenchmarks, baselineBenchmarks, threshold) { + const added = []; + const removed = []; + const unchanged = []; + const regressed = []; + const improved = []; + /** @type {Array} */ + const failed = []; + + const compareMap = new Map(compareBenchmarks.map((b) => [b.name, b])); + const baselineMap = new Map(baselineBenchmarks?.map((b) => [b.name, b]) ?? []); + + // eslint-disable-next-line @typescript-eslint/naming-convention,@typescript-eslint/no-unused-vars + for (const [_, baselineBench] of baselineMap) { + const compareBench = compareMap.get(baselineBench.name); + + if (!compareBench) { + removed.push(baselineBench); + } else if ('min' in compareBench) { + const diff = (compareBench.min - baselineBench.min) / baselineBench.min; + const benchmark = { + name: baselineBench.name, + baseline: baselineBench, + compare: compareBench, + diff, + }; + + if (diff > threshold) { + regressed.push(benchmark); + } else if (diff < -threshold) { + improved.push(benchmark); + } else { + unchanged.push(benchmark); + } + + compareMap.delete(baselineBench.name); + } else { + failed.push(compareBench); + compareMap.delete(baselineBench.name); + } + } + + // eslint-disable-next-line @typescript-eslint/naming-convention,@typescript-eslint/no-unused-vars + for (const [_, compareBench] of compareMap) { + if ('median' in compareBench) { + added.push(compareBench); + } else { + failed.push(compareBench); + } + } + + return { + added, + removed, + regressed, + improved, + unchanged, + failed, + }; +} + +/** + * @param {import('./compare-benchmark-results.types.js').BenchmarkResults} results + */ +function printResults(results) { + console.log( + `Overall result: ${results.failed.length > 0 || results.regressed.length > 0 ? 'fail' : 'pass'}`, + ); + + console.log(`Regressed benchmarks: ${results.regressed.length}`); + if (results.regressed.length > 0) { + const changedTable = results.regressed.map((c) => ({ + name: c.name, + minBaseline: c.baseline.min.toFixed(2), + minCompare: c.compare.min.toFixed(2), + diff: `${(c.diff * 100).toFixed(2)}%`, + sampleCount: c.compare.sampleCount, + mean: c.compare.mean.toFixed(2), + median: c.compare.median.toFixed(2), + p75: c.compare.p75.toFixed(2), + p99: c.compare.p99.toFixed(2), + marginOfError: c.compare.moe.toFixed(2), + })); + console.table(changedTable); + } + + console.log(`Improved benchmarks: ${results.improved.length}`); + if (results.improved.length > 0) { + const changedTable = results.improved.map((c) => ({ + name: c.name, + minBaseline: c.baseline.min.toFixed(2), + minCompare: c.compare.min.toFixed(2), + diff: `${(c.diff * 100).toFixed(2)}%`, + sampleCount: c.compare.sampleCount, + mean: c.compare.mean.toFixed(2), + median: c.compare.median.toFixed(2), + p75: c.compare.p75.toFixed(2), + p99: c.compare.p99.toFixed(2), + marginOfError: c.compare.moe.toFixed(2), + })); + console.table(changedTable); + } + + console.log(`Unchanged benchmarks: ${results.unchanged.length}`); + if (results.unchanged.length > 0) { + const unchangedTable = results.unchanged.map((c) => ({ + name: c.name, + minBaseline: c.baseline.min.toFixed(2), + minCompare: c.compare.min.toFixed(2), + diff: `${(c.diff * 100).toFixed(2)}%`, + sampleCount: c.compare.sampleCount, + mean: c.compare.mean.toFixed(2), + median: c.compare.median.toFixed(2), + p75: c.compare.p75.toFixed(2), + p99: c.compare.p99.toFixed(2), + marginOfError: c.compare.moe.toFixed(2), + })); + console.table(unchangedTable); + } + + console.log(`Added benchmarks: ${results.added.length}`); + results.added.forEach((b) => console.log(`- ${b.name}`)); + if (results.added.length > 0) { + const addedTable = results.added.map((c) => ({ + name: c.name, + min: c.min.toFixed(2), + sampleCount: c.sampleCount, + mean: c.mean.toFixed(2), + median: c.median.toFixed(2), + p75: c.p75.toFixed(2), + p99: c.p99.toFixed(2), + marginOfError: c.moe.toFixed(2), + })); + console.table(addedTable); + } + + console.log(`Removed benchmarks: ${results.removed.length}`); + results.removed.forEach((b) => console.log(`- ${b.name}`)); + + console.log(`Failed benchmarks: ${results.failed.length}`); + results.failed.forEach((b) => console.log(`- ${b.name}`)); +} + +/** + * @param {import('./compare-benchmark-results.types.js').BenchmarkResults} results + */ +function generateResultMarkdown(results) { + let markdown = ''; + + const fMs = (/** @type {number} */ number) => `${number.toFixed(2)}ms`; + const fPerc = (/** @type {number} */ number) => `${number.toFixed(2)}%`; + + if (results.regressed.length > 0) { + markdown += `\n**Regressed benchmarks**: ${results.regressed.length}\n`; + + markdown += `| Name | Min (Baseline) | Min (This run) | Diff | Sample Count | Margin of Error |\n`; + markdown += `| ---- | -------------- | -------------- | ---- | ------------ | --------------- |\n`; + + results.regressed.forEach((r) => { + markdown += `| ${r.name} | ${fMs(r.baseline.min)} | ${fMs(r.compare.min)} | ${fPerc(r.diff * 100)} | ${r.compare.sampleCount} | ${fPerc(r.compare.moe)} |\n`; + }); + + markdown += `
\n`; + markdown += `Detailed Results\n\n`; + markdown += `| Name | Min (Baseline) | Min (This run) | Diff | Sample Count | Mean | Median | P75 | P99 | Max | Margin of Error |\n`; + markdown += `| ---- | -------------- | -------------- | ---- | ------------ | ---- | ------ | --- | --- | --- | --------------- |\n`; + + results.regressed.forEach((r) => { + markdown += `| ${r.name} | ${fMs(r.baseline.min)} | ${fMs(r.compare.min)} | ${fPerc(r.diff * 100)} | ${r.compare.sampleCount} | ${fMs(r.compare.mean)} | ${fMs(r.compare.median)} | ${fMs(r.compare.p75)} | ${fMs(r.compare.p99)} | ${fMs(r.compare.max)} | ${fPerc(r.compare.moe)} |\n`; + }); + + markdown += `
\n`; + } + + if (results.improved.length > 0) { + markdown += `\n**Improved benchmarks**: ${results.improved.length}\n`; + + markdown += `| Name | Min (Baseline) | Min (This run) | Diff | Sample Count | Margin of Error |\n`; + markdown += `| ---- | -------------- | -------------- | ---- | ------------ | --------------- |\n`; + + results.improved.forEach((r) => { + markdown += `| ${r.name} | ${fMs(r.baseline.min)} | ${fMs(r.compare.min)} | ${fPerc(r.diff * 100)} | ${r.compare.sampleCount} | ${fPerc(r.compare.moe)} |\n`; + }); + + markdown += `
\n`; + markdown += `Detailed Results\n\n`; + markdown += `| Name | Min (Baseline) | Min (This run) | Diff | Sample Count | Mean | Median | P75 | P99 | Max | Margin of Error |\n`; + markdown += `| ---- | -------------- | -------------- | ---- | ------------ | ---- | ------ | --- | --- | --- | --------------- |\n`; + + results.improved.forEach((r) => { + markdown += `| ${r.name} | ${fMs(r.baseline.min)} | ${fMs(r.compare.min)} | ${fPerc(r.diff * 100)} | ${r.compare.sampleCount} | ${fMs(r.compare.mean)} | ${fMs(r.compare.median)} | ${fMs(r.compare.p75)} | ${fMs(r.compare.p99)} | ${fMs(r.compare.max)} | ${fPerc(r.compare.moe)} |\n`; + }); + + markdown += `
\n`; + } + + if (results.unchanged.length > 0) { + markdown += `\n**Unchanged benchmarks**: ${results.unchanged.length}\n`; + + markdown += `
\n`; + markdown += `Detailed Results\n\n`; + + markdown += `| Name | Min (Baseline) | Min (This run) | Diff | Sample Count | Mean | Median | P75 | P99 | Max | Margin of Error |\n`; + markdown += `| ---- | -------------- | -------------- | ---- | ------------ | ---- | ------ | --- | --- | --- | --------------- |\n`; + + results.unchanged.forEach((r) => { + markdown += `| ${r.name} | ${fMs(r.baseline.min)} | ${fMs(r.compare.min)} | ${fPerc(r.diff * 100)} | ${r.compare.sampleCount} | ${fMs(r.compare.mean)} | ${fMs(r.compare.median)} | ${fMs(r.compare.p75)} | ${fMs(r.compare.p99)} | ${fMs(r.compare.max)} | ${fPerc(r.compare.moe)} |\n`; + }); + + markdown += `
\n`; + } + + if (results.added.length > 0) { + markdown += `\n**Added benchmarks**: ${results.added.length}\n`; + markdown += `| Name | Min | Sample Count | Mean | Median | P75 | P99 | Max | Margin of Error |\n`; + markdown += `| ---- | --- | ------------ | ---- | ------ | --- | --- | --- | --------------- |\n`; + + results.added.forEach((r) => { + markdown += `| ${r.name} | ${fMs(r.min)} | ${r.sampleCount} | ${fMs(r.mean)} | ${fMs(r.median)} | ${fMs(r.p75)} | ${fMs(r.p99)} | ${fMs(r.max)} | ${fPerc(r.moe)} |\n`; + }); + } + + if (results.removed.length > 0) { + markdown += `\n**Removed benchmarks**: ${results.removed.length}\n`; + results.removed.forEach((r) => { + markdown += `- ${r.name}\n`; + }); + } + + if (results.failed.length > 0) { + markdown += `\n**Failed benchmarks**: ${results.failed.length}\n`; + results.failed.forEach((r) => { + markdown += `- ${r.name}\n`; + }); + } + + return markdown; +} + +/** + * @param {string | null} baselineJson + * @param {string} compareJson + * @param {number} threshold + * @returns {Promise<{ results: import('./compare-benchmark-results.types.js').BenchmarkResults, markdown: string }>} + */ +export async function compareResults(baselineJson, compareJson, threshold) { + const compareBenchmarks = parseBenchmarkResults(compareJson); + const baselineBenchmarks = baselineJson ? parseBenchmarkResults(baselineJson) : null; + + const results = processResults(compareBenchmarks, baselineBenchmarks, threshold); + + try { + printResults(results); + } catch (error) { + console.error(error); + console.log(util.inspect(results, { depth: null })); + } + + return { results, markdown: generateResultMarkdown(results) }; +} diff --git a/test/performance-charts/scripts/compare-benchmark-results.types.ts b/test/performance-charts/scripts/compare-benchmark-results.types.ts new file mode 100644 index 0000000000000..56047b967ad63 --- /dev/null +++ b/test/performance-charts/scripts/compare-benchmark-results.types.ts @@ -0,0 +1,49 @@ +export interface BenchmarkResult { + id: string; + name: string; + rank: number; + rme: number; + samples: []; + totalTime: number; + min: number; + max: number; + hz: number; + period: number; + mean: number; + variance: number; + sd: number; + sem: number; + df: number; + critical: number; + moe: number; + p75: number; + p99: number; + p995: number; + p999: number; + sampleCount: number; + median: number; +} + +export interface FailedBenchmarkResult { + id: string; + name: string; + rank: number; + rme: number; + samples: []; +} + +export interface BenchmarkComparison { + name: string; + baseline: BenchmarkResult; + compare: BenchmarkResult; + diff: number; // Percentage difference between the median of compare and baseline +} + +export interface BenchmarkResults { + failed: FailedBenchmarkResult[]; + added: BenchmarkResult[]; + removed: BenchmarkResult[]; + improved: BenchmarkComparison[]; + regressed: BenchmarkComparison[]; + unchanged: BenchmarkComparison[]; +} diff --git a/test/performance-charts/tests/BarChart.bench.tsx b/test/performance-charts/tests/BarChart.bench.tsx index d798b61d9d65e..5f21b8371527d 100644 --- a/test/performance-charts/tests/BarChart.bench.tsx +++ b/test/performance-charts/tests/BarChart.bench.tsx @@ -1,13 +1,12 @@ import * as React from 'react'; -// eslint-disable-next-line no-restricted-imports -import { render, cleanup } from '@testing-library/react'; -import { describe } from 'vitest'; +import { render } from 'vitest-browser-react/pure'; +import { describe, expect } from 'vitest'; import { BarChart } from '@mui/x-charts/BarChart'; import { options } from '../utils/options'; import { bench } from '../utils/bench'; describe('BarChart', () => { - const dataLength = 800; + const dataLength = 1_200; const data = Array.from({ length: dataLength + 1 }).map((_, i) => ({ x: i, y: 50 + Math.sin(i / 5) * 25, @@ -19,13 +18,11 @@ describe('BarChart', () => { bench( 'BarChart with big data amount', async () => { - const { findByText } = render( + const page = render( , ); - await findByText(dataLength.toString(), { ignore: 'span' }); - - cleanup(); + expect(page.getByText('96')).toBeInTheDocument(); }, options, ); diff --git a/test/performance-charts/tests/BarChartPro.bench.tsx b/test/performance-charts/tests/BarChartPro.bench.tsx index d77158c831d8f..0fcec74d13440 100644 --- a/test/performance-charts/tests/BarChartPro.bench.tsx +++ b/test/performance-charts/tests/BarChartPro.bench.tsx @@ -1,13 +1,12 @@ import * as React from 'react'; -// eslint-disable-next-line no-restricted-imports -import { render, cleanup } from '@testing-library/react'; -import { describe } from 'vitest'; +import { render } from 'vitest-browser-react/pure'; +import { describe, expect } from 'vitest'; import { BarChartPro } from '@mui/x-charts-pro/BarChartPro'; import { options } from '../utils/options'; import { bench } from '../utils/bench'; describe('BarChartPro', () => { - const dataLength = 800; + const dataLength = 1_200; const data = Array.from({ length: dataLength + 1 }).map((_, i) => ({ x: i, y: 50 + Math.sin(i / 5) * 25, @@ -19,7 +18,7 @@ describe('BarChartPro', () => { bench( 'BarChartPro with big data amount', async () => { - const { findByText } = render( + const page = render( { />, ); - await findByText('60', { ignore: 'span' }); - - cleanup(); + expect(page.getByText('60')).toBeInTheDocument(); }, options, ); diff --git a/test/performance-charts/tests/FunnelChart.bench.tsx b/test/performance-charts/tests/FunnelChart.bench.tsx index 2adc270cf46e3..e1280499ca23e 100644 --- a/test/performance-charts/tests/FunnelChart.bench.tsx +++ b/test/performance-charts/tests/FunnelChart.bench.tsx @@ -1,13 +1,12 @@ import * as React from 'react'; -// eslint-disable-next-line no-restricted-imports -import { render, cleanup } from '@testing-library/react'; -import { describe } from 'vitest'; +import { render } from 'vitest-browser-react/pure'; +import { describe, expect } from 'vitest'; import { Unstable_FunnelChart as FunnelChart } from '@mui/x-charts-pro/FunnelChart'; import { options } from '../utils/options'; import { bench } from '../utils/bench'; describe('FunnelChart', () => { - const dataLength = 10; + const dataLength = 100; const series = [ { data: Array.from({ length: dataLength }, (_, i) => ({ value: dataLength / (i + 1) })), @@ -17,11 +16,9 @@ describe('FunnelChart', () => { bench( 'FunnelChart with big data amount', async () => { - const { findByText } = render(); + const page = render(); - await findByText(dataLength.toLocaleString(), { ignore: 'span' }); - - cleanup(); + expect(page.getByText(dataLength.toLocaleString())).toBeInTheDocument(); }, options, ); diff --git a/test/performance-charts/tests/LineChart.bench.tsx b/test/performance-charts/tests/LineChart.bench.tsx index 252f8ef61fe90..a290bdf0b227e 100644 --- a/test/performance-charts/tests/LineChart.bench.tsx +++ b/test/performance-charts/tests/LineChart.bench.tsx @@ -1,13 +1,12 @@ import * as React from 'react'; -// eslint-disable-next-line no-restricted-imports -import { render, cleanup } from '@testing-library/react'; -import { describe } from 'vitest'; +import { render } from 'vitest-browser-react/pure'; +import { describe, expect } from 'vitest'; import { LineChart } from '@mui/x-charts/LineChart'; import { options } from '../utils/options'; import { bench } from '../utils/bench'; describe('LineChart', () => { - const dataLength = 1_400; + const dataLength = 5_000; const data = Array.from({ length: dataLength }).map((_, i) => ({ x: i, y: 50 + Math.sin(i / 5) * 25, @@ -19,7 +18,7 @@ describe('LineChart', () => { bench( 'LineChart with big data amount (with marks)', async () => { - const { findByText } = render( + const page = render( { />, ); - await findByText(dataLength.toLocaleString(), { ignore: 'span' }); - - cleanup(); + expect(page.getByText(dataLength.toLocaleString())).toBeInTheDocument(); }, options, ); @@ -38,7 +35,7 @@ describe('LineChart', () => { bench( 'Area chart with big data amount (no marks)', async () => { - const { findByText } = render( + const page = render( { />, ); - await findByText(dataLength.toLocaleString(), { ignore: 'span' }); - - cleanup(); + expect(page.getByText(dataLength.toLocaleString())).toBeInTheDocument(); }, options, ); diff --git a/test/performance-charts/tests/LineChartPro.bench.tsx b/test/performance-charts/tests/LineChartPro.bench.tsx index 8c4b0c787b52e..af875613f9c5d 100644 --- a/test/performance-charts/tests/LineChartPro.bench.tsx +++ b/test/performance-charts/tests/LineChartPro.bench.tsx @@ -1,13 +1,12 @@ import * as React from 'react'; -// eslint-disable-next-line no-restricted-imports -import { render, cleanup } from '@testing-library/react'; -import { describe } from 'vitest'; +import { render } from 'vitest-browser-react/pure'; +import { describe, expect } from 'vitest'; import { LineChartPro } from '@mui/x-charts-pro/LineChartPro'; import { options } from '../utils/options'; import { bench } from '../utils/bench'; describe('LineChartPro', () => { - const dataLength = 1_400; + const dataLength = 5_000; const data = Array.from({ length: dataLength }).map((_, i) => ({ x: i, y: 50 + Math.sin(i / 5) * 25, @@ -19,7 +18,7 @@ describe('LineChartPro', () => { bench( 'LineChartPro with big data amount and zoomed in (with marks)', async () => { - const { findByText } = render( + const page = render( { />, ); - await findByText('60', { ignore: 'span' }); - - cleanup(); + expect(page.getByText('2,600')).toBeInTheDocument(); }, options, ); diff --git a/test/performance-charts/tests/PieChart.bench.tsx b/test/performance-charts/tests/PieChart.bench.tsx index a47167f241587..ff5bedf4e9d94 100644 --- a/test/performance-charts/tests/PieChart.bench.tsx +++ b/test/performance-charts/tests/PieChart.bench.tsx @@ -1,13 +1,12 @@ import * as React from 'react'; -// eslint-disable-next-line no-restricted-imports -import { render, cleanup } from '@testing-library/react'; -import { describe } from 'vitest'; +import { render } from 'vitest-browser-react/pure'; +import { describe, expect } from 'vitest'; import { PieChart } from '@mui/x-charts/PieChart'; import { options } from '../utils/options'; import { bench } from '../utils/bench'; describe('PieChart', () => { - const dataLength = 50; + const dataLength = 100; const data = Array.from({ length: dataLength + 1 }).map((_, i) => ({ value: 50 + Math.sin(i / 5) * 1000, })); @@ -15,7 +14,7 @@ describe('PieChart', () => { bench( 'PieChart with big data amount', async () => { - const { findByText } = render( + const page = render( { />, ); - const result = 1050; - await findByText(result); - - cleanup(); + expect(page.getByText('50', { exact: true })).toBeInTheDocument(); }, options, ); diff --git a/test/performance-charts/tests/ScatterChart.bench.tsx b/test/performance-charts/tests/ScatterChart.bench.tsx index 2a9651e039164..0f54be849baef 100644 --- a/test/performance-charts/tests/ScatterChart.bench.tsx +++ b/test/performance-charts/tests/ScatterChart.bench.tsx @@ -1,13 +1,12 @@ import * as React from 'react'; -// eslint-disable-next-line no-restricted-imports -import { render, cleanup } from '@testing-library/react'; -import { describe } from 'vitest'; +import { render } from 'vitest-browser-react/pure'; +import { describe, expect } from 'vitest'; import { ScatterChart } from '@mui/x-charts/ScatterChart'; import { options } from '../utils/options'; import { bench } from '../utils/bench'; describe('ScatterChart', () => { - const dataLength = 1_400; + const dataLength = 5_000; const data = Array.from({ length: dataLength }).map((_, i) => ({ x: i, y: 50 + Math.sin(i / 5) * 25, @@ -18,7 +17,7 @@ describe('ScatterChart', () => { bench( 'ScatterChart with big data amount', async () => { - const { findByText } = render( + const page = render( v.toLocaleString('en-US') }]} series={[{ data }]} @@ -27,9 +26,7 @@ describe('ScatterChart', () => { />, ); - await findByText(dataLength.toLocaleString('en-US'), { ignore: 'span' }); - - cleanup(); + expect(page.getByText(dataLength.toLocaleString('en-US'))).toBeInTheDocument(); }, options, ); diff --git a/test/performance-charts/tests/ScatterChartPro.bench.tsx b/test/performance-charts/tests/ScatterChartPro.bench.tsx index d987fc19c66c5..e593dd8d69a0c 100644 --- a/test/performance-charts/tests/ScatterChartPro.bench.tsx +++ b/test/performance-charts/tests/ScatterChartPro.bench.tsx @@ -1,13 +1,12 @@ import * as React from 'react'; -// eslint-disable-next-line no-restricted-imports -import { render, cleanup } from '@testing-library/react'; -import { describe } from 'vitest'; +import { render } from 'vitest-browser-react/pure'; +import { describe, expect } from 'vitest'; import { ScatterChartPro } from '@mui/x-charts-pro/ScatterChartPro'; import { options } from '../utils/options'; import { bench } from '../utils/bench'; describe('ScatterChartPro', () => { - const dataLength = 1_400; + const dataLength = 10_000; const data = Array.from({ length: dataLength }).map((_, i) => ({ x: i, y: 50 + Math.sin(i / 5) * 25, @@ -18,7 +17,7 @@ describe('ScatterChartPro', () => { bench( 'ScatterChartPro with big data amount', async () => { - const { findByText } = render( + const page = render( { />, ); - await findByText('60', { ignore: 'span' }); - - cleanup(); + expect(page.getByText('60', { exact: true })).toBeInTheDocument(); }, options, ); @@ -45,7 +42,7 @@ describe('ScatterChartPro', () => { bench( 'ScatterChartPro with big data amount and zoomed in', async () => { - const { findByText } = render( + const page = render( { />, ); - await findByText('50.06', { ignore: 'span' }); - - cleanup(); + expect(page.getByText('50.06')).toBeInTheDocument(); }, options, ); diff --git a/test/performance-charts/tsconfig.json b/test/performance-charts/tsconfig.json index fe6e5b2a9b469..065932eb46ca5 100644 --- a/test/performance-charts/tsconfig.json +++ b/test/performance-charts/tsconfig.json @@ -1,8 +1,8 @@ { "extends": "../../tsconfig.json", - "include": ["tests/**/*", "utils/**/*"], + "include": ["tests/**/*", "utils/**/*", "./vitest-browser-mode.d.ts"], "exclude": ["node_modules"], "compilerOptions": { - "types": ["@vitest/browser/providers/playwright", "vite/client"] + "types": ["@vitest/browser/providers/playwright", "vite/client", "./vitest-browser-mode.d.ts"] } } diff --git a/test/performance-charts/utils/options.ts b/test/performance-charts/utils/options.ts index 90bd3e8c48514..aaa286b8b6ef2 100644 --- a/test/performance-charts/utils/options.ts +++ b/test/performance-charts/utils/options.ts @@ -1,9 +1,12 @@ -import { BenchOptions } from 'vitest'; +import type { BenchOptions } from 'vitest'; +import { cleanup } from 'vitest-browser-react/pure'; +import { commands } from '@vitest/browser/context'; import { isTrace } from './env'; +const defaultIterations = isTrace ? 1 : 100; const iterations = import.meta.env.BENCHMARK_ITERATIONS ? parseInt(import.meta.env.BENCHMARK_ITERATIONS, 10) - : 1; + : defaultIterations; const taskModes = new Map(); export function getTaskMode(taskName: string): 'run' | 'warmup' { @@ -19,7 +22,16 @@ const traceOptions: BenchOptions = { }; const benchOptions: BenchOptions = { + warmupIterations: 10, + warmupTime: 0, iterations, + throws: true, + time: 0, + // @ts-expect-error Our custom runner supports it, but the default one doesn't. + async afterEach() { + cleanup(); + await commands.requestGC(); + }, }; export const options: BenchOptions = isTrace ? traceOptions : benchOptions; diff --git a/test/performance-charts/utils/vitest-bench-runner.ts b/test/performance-charts/utils/vitest-bench-runner.ts new file mode 100644 index 0000000000000..e729e80fe9dc1 --- /dev/null +++ b/test/performance-charts/utils/vitest-bench-runner.ts @@ -0,0 +1,169 @@ +/* eslint-disable id-denylist,@typescript-eslint/no-shadow */ +import { NodeBenchmarkRunner, VitestRunner } from 'vitest/runners'; +import { Benchmark, BenchmarkResult, BenchTask, RunnerTestSuite, Suite } from 'vitest'; +import { getBenchFn, getBenchOptions } from 'vitest/suite'; +import { updateTask as updateRunnerTask, type TaskUpdateEvent, type Task } from '@vitest/runner'; + +// Adapted from https://github.com/vitest-dev/vitest/blob/c1f78d2adc78ef08ef8b61b0dd6a925fb08f20b6/packages/vitest/src/runtime/runners/benchmark.ts +export default class VitestBenchRunner extends NodeBenchmarkRunner implements VitestRunner { + async runSuite(suite: RunnerTestSuite): Promise { + await runBenchmarkSuite(suite, this); + } +} + +type DeferPromise = Promise & { + resolve: (value: T | PromiseLike) => void; + reject: (reason?: any) => void; +}; + +function createDefer(): DeferPromise { + let resolve: ((value: T | PromiseLike) => void) | null = null; + let reject: ((reason?: any) => void) | null = null; + + const p = new Promise((_resolve, _reject) => { + resolve = _resolve; + reject = _reject; + }) as DeferPromise; + + p.resolve = resolve!; + p.reject = reject!; + return p; +} + +function createBenchmarkResult(name: string): BenchmarkResult { + return { + name, + rank: 0, + rme: 0, + samples: [] as number[], + } as BenchmarkResult; +} + +const benchmarkTasks = new WeakMap(); + +async function runBenchmarkSuite(suite: Suite, runner: NodeBenchmarkRunner) { + const { Task, Bench } = await runner.importTinybench(); + + const start = performance.now(); + + const benchmarkGroup: Benchmark[] = []; + const benchmarkSuiteGroup = []; + for (const task of suite.tasks) { + if (task.mode !== 'run' && task.mode !== 'queued') { + continue; + } + + if (task.meta?.benchmark) { + benchmarkGroup.push(task as Benchmark); + } else if (task.type === 'suite') { + benchmarkSuiteGroup.push(task); + } + } + + // run sub suites sequentially + for (const subSuite of benchmarkSuiteGroup) { + // eslint-disable-next-line no-await-in-loop + await runBenchmarkSuite(subSuite, runner); + } + + if (benchmarkGroup.length) { + const defer = createDefer(); + suite.result = { + state: 'run', + startTime: start, + benchmark: createBenchmarkResult(suite.name), + }; + updateTask('suite-prepare', suite); + + const addBenchTaskListener = (task: InstanceType, benchmark: Benchmark) => { + task.addEventListener( + 'complete', + (e) => { + const task = e.task; + const taskRes = task.result!; + const result = benchmark.result!.benchmark!; + benchmark.result!.state = 'pass'; + Object.assign(result, taskRes); + // compute extra stats and free raw samples as early as possible + const samples = result.samples; + result.sampleCount = samples.length; + result.median = + samples.length % 2 + ? samples[Math.floor(samples.length / 2)] + : (samples[samples.length / 2] + samples[samples.length / 2 - 1]) / 2; + if (!runner.config.benchmark?.includeSamples) { + result.samples.length = 0; + } + updateTask('test-finished', benchmark); + }, + { + once: true, + }, + ); + task.addEventListener( + 'error', + (e) => { + const task = e.task; + defer.reject(benchmark ? task.result!.error : e); + }, + { + once: true, + }, + ); + }; + + benchmarkGroup.forEach((benchmark) => { + const options = getBenchOptions(benchmark); + const benchmarkInstance = new Bench(options); + + const benchmarkFn = getBenchFn(benchmark); + + benchmark.result = { + state: 'run', + startTime: start, + benchmark: createBenchmarkResult(benchmark.name), + }; + + const task = new Task(benchmarkInstance, benchmark.name, benchmarkFn, { + beforeEach() { + return options.beforeEach?.(this); + }, + afterEach() { + return options.afterEach?.(this); + }, + }); + benchmarkTasks.set(benchmark, task); + addBenchTaskListener(task, benchmark); + }); + + const tasks: [BenchTask, Benchmark][] = []; + + for (const benchmark of benchmarkGroup) { + const task = benchmarkTasks.get(benchmark)!; + updateTask('test-prepare', benchmark); + // eslint-disable-next-line no-await-in-loop + await task.warmup(); + tasks.push([ + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => + setTimeout(async () => { + resolve(await task.run()); + }), + ), + benchmark, + ]); + } + + suite.result!.duration = performance.now() - start; + suite.result!.state = 'pass'; + + updateTask('suite-finished', suite); + defer.resolve(null); + + await defer; + } + + function updateTask(event: TaskUpdateEvent, task: Task) { + updateRunnerTask(event, task, runner); + } +} diff --git a/test/performance-charts/vitest-browser-mode.d.ts b/test/performance-charts/vitest-browser-mode.d.ts new file mode 100644 index 0000000000000..35b0ec1c62512 --- /dev/null +++ b/test/performance-charts/vitest-browser-mode.d.ts @@ -0,0 +1,12 @@ +import '@vitest/browser'; + +declare module '@vitest/browser/context' { + interface BrowserCommands { + /** + * Playwright's `page.requestGC()` command to request garbage collection. + * https://playwright.dev/docs/api/class-page#page-request-gc + * @returns {Promise} A promise that resolves when the garbage collection is requested. + */ + requestGC: () => Promise; + } +} diff --git a/test/performance-charts/vitest.config.ts b/test/performance-charts/vitest.config.ts index f7275ec7edc1d..e4d5f90e9d703 100644 --- a/test/performance-charts/vitest.config.ts +++ b/test/performance-charts/vitest.config.ts @@ -1,18 +1,16 @@ import { defineConfig } from 'vitest/config'; -import codspeedPlugin from '@codspeed/vitest-plugin'; import react from '@vitejs/plugin-react'; -const isCI = process.env.CI === 'true'; -const isTrace = !isCI && process.env.TRACE === 'true'; - export default defineConfig({ - plugins: [...(isCI ? [codspeedPlugin()] : []), react()], + plugins: [react()], test: { setupFiles: ['./setup.ts'], - env: { TRACE: isTrace ? 'true' : 'false' }, - environment: isTrace ? 'node' : 'jsdom', + env: { TRACE: process.env.TRACE }, + environment: 'node', + maxConcurrency: 1, + runner: './utils/vitest-bench-runner.ts', browser: { - enabled: isTrace, + enabled: true, headless: true, instances: [ { @@ -27,6 +25,9 @@ export default defineConfig({ }, }, ], + commands: { + requestGC: (ctx) => ctx.page.requestGC(), + }, provider: 'playwright', }, },