Skip to content

Commit 6fec639

Browse files
mxschmittmxschmitt
committed
chore: debug flaky test
1 parent b59402a commit 6fec639

File tree

3 files changed

+8
-285
lines changed

3 files changed

+8
-285
lines changed

‎.github/workflows/tests_secondary.yml

Lines changed: 3 additions & 282 deletions
Original file line numberDiff line numberDiff line change
@@ -24,193 +24,15 @@ permissions:
2424
contents: read # This is required for actions/checkout to succeed
2525

2626
jobs:
27-
test_linux:
28-
name: ${{ matrix.os }} (${{ matrix.browser }})
29-
environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }}
30-
strategy:
31-
fail-fast: false
32-
matrix:
33-
browser: [chromium, firefox, webkit]
34-
os: [ubuntu-24.04]
35-
runs-on: ${{ matrix.os }}
36-
steps:
37-
- uses: actions/checkout@v4
38-
- uses: ./.github/actions/run-test
39-
with:
40-
browsers-to-install: ${{ matrix.browser }} chromium
41-
command: npm run test -- --project=${{ matrix.browser }}-*
42-
bot-name: "${{ matrix.browser }}-${{ matrix.os }}"
43-
flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
44-
flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
45-
flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
46-
47-
test_mac:
48-
name: ${{ matrix.os }} (${{ matrix.browser }})
49-
environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }}
50-
strategy:
51-
fail-fast: false
52-
matrix:
53-
# Intel: *-large
54-
# Arm64: *-xlarge
55-
os: [macos-13-large, macos-13-xlarge, macos-14-large, macos-14-xlarge]
56-
browser: [chromium, firefox, webkit]
57-
include:
58-
- os: macos-15-large
59-
browser: webkit
60-
- os: macos-15-xlarge
61-
browser: webkit
62-
runs-on: ${{ matrix.os }}
63-
steps:
64-
- uses: actions/checkout@v4
65-
- uses: ./.github/actions/run-test
66-
with:
67-
browsers-to-install: ${{ matrix.browser }} chromium
68-
command: npm run test -- --project=${{ matrix.browser }}-*
69-
bot-name: "${{ matrix.browser }}-${{ matrix.os }}"
70-
flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
71-
flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
72-
flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
73-
74-
test_win:
75-
name: "Windows"
76-
environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }}
77-
strategy:
78-
fail-fast: false
79-
matrix:
80-
browser: [chromium, firefox, webkit]
81-
runs-on: windows-latest
82-
steps:
83-
- uses: actions/checkout@v4
84-
- uses: ./.github/actions/run-test
85-
with:
86-
browsers-to-install: ${{ matrix.browser }} chromium
87-
command: npm run test -- --project=${{ matrix.browser }}-* ${{ matrix.browser == 'firefox' && '--workers 1' || '' }}
88-
bot-name: "${{ matrix.browser }}-windows-latest"
89-
flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
90-
flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
91-
flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
92-
93-
test-package-installations-other-node-versions:
94-
name: "Installation Test ${{ matrix.os }} (${{ matrix.node_version }})"
95-
environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }}
96-
runs-on: ${{ matrix.os }}
97-
strategy:
98-
fail-fast: false
99-
matrix:
100-
include:
101-
- os: ubuntu-latest
102-
node_version: 20
103-
- os: ubuntu-latest
104-
node_version: 22
105-
- os: ubuntu-latest
106-
node_version: 24
107-
timeout-minutes: 30
108-
steps:
109-
- uses: actions/checkout@v4
110-
- run: npm install -g yarn@1
111-
- run: npm install -g pnpm@8
112-
- name: Setup Ubuntu Binary Installation # TODO: Remove when https://github.com/electron/electron/issues/42510 is fixed
113-
if: ${{ runner.os == 'Linux' }}
114-
run: |
115-
if grep -q "Ubuntu 24" /etc/os-release; then
116-
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
117-
fi
118-
shell: bash
119-
- uses: ./.github/actions/run-test
120-
with:
121-
node-version: ${{ matrix.node_version }}
122-
command: npm run itest
123-
bot-name: "package-installations-${{ matrix.os }}-node${{ matrix.node_version }}"
124-
flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
125-
flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
126-
flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
127-
128-
headed_tests:
129-
name: "headed ${{ matrix.browser }} (${{ matrix.os }})"
130-
environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }}
131-
strategy:
132-
fail-fast: false
133-
matrix:
134-
browser: [chromium, firefox, webkit]
135-
os: [ubuntu-24.04, macos-14-xlarge, windows-latest]
136-
include:
137-
# We have different binaries per Ubuntu version for WebKit.
138-
- browser: webkit
139-
os: ubuntu-22.04
140-
runs-on: ${{ matrix.os }}
141-
steps:
142-
- uses: actions/checkout@v4
143-
- uses: ./.github/actions/run-test
144-
with:
145-
browsers-to-install: ${{ matrix.browser }} chromium
146-
command: npm run test -- --project=${{ matrix.browser }}-* --headed
147-
bot-name: "${{ matrix.browser }}-headed-${{ matrix.os }}"
148-
flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
149-
flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
150-
flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
151-
152-
transport_linux:
153-
name: "Transport"
154-
environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }}
155-
strategy:
156-
fail-fast: false
157-
matrix:
158-
mode: [driver, service]
159-
runs-on: ubuntu-22.04
160-
steps:
161-
- uses: actions/checkout@v4
162-
- uses: ./.github/actions/run-test
163-
with:
164-
browsers-to-install: chromium
165-
command: npm run ctest
166-
bot-name: "${{ matrix.mode }}"
167-
flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
168-
flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
169-
flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
170-
env:
171-
PWTEST_MODE: ${{ matrix.mode }}
172-
173-
tracing_linux:
174-
name: Tracing ${{ matrix.browser }} ${{ matrix.channel }}
175-
environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }}
176-
strategy:
177-
fail-fast: false
178-
matrix:
179-
include:
180-
- browser: chromium
181-
runs-on: ubuntu-22.04
182-
- browser: firefox
183-
runs-on: ubuntu-22.04
184-
# See https://github.com/microsoft/playwright/issues/35586
185-
- browser: webkit
186-
runs-on: ubuntu-24.04
187-
- browser: chromium
188-
runs-on: ubuntu-22.04
189-
channel: chromium-tip-of-tree
190-
runs-on: ${{ matrix.runs-on }}
191-
steps:
192-
- uses: actions/checkout@v4
193-
- uses: ./.github/actions/run-test
194-
with:
195-
browsers-to-install: ${{ matrix.browser }} chromium ${{ matrix.channel }}
196-
command: npm run test -- --project=${{ matrix.browser }}-*
197-
bot-name: "tracing-${{ matrix.channel || matrix.browser }}"
198-
flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
199-
flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
200-
flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
201-
env:
202-
PWTEST_TRACE: 1
203-
PWTEST_CHANNEL: ${{ matrix.channel }}
204-
20527
test_chromium_channels:
20628
name: Test ${{ matrix.channel }} on ${{ matrix.runs-on }}
20729
environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }}
20830
runs-on: ${{ matrix.runs-on }}
20931
strategy:
21032
fail-fast: false
21133
matrix:
212-
channel: [chrome, chrome-beta, msedge, msedge-beta, msedge-dev]
213-
runs-on: [ubuntu-22.04, macos-latest, windows-latest]
34+
channel: [chrome-beta]
35+
runs-on: [macos-latest]
21436
steps:
21537
- uses: actions/checkout@v4
21638
- uses: ./.github/actions/run-test
@@ -224,105 +46,4 @@ jobs:
22446
env:
22547
PWTEST_CHANNEL: ${{ matrix.channel }}
22648

227-
chromium_tot:
228-
name: Chromium tip-of-tree ${{ matrix.os }}${{ matrix.headed }}
229-
environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }}
230-
runs-on: ${{ matrix.os }}
231-
strategy:
232-
fail-fast: false
233-
matrix:
234-
os: [ubuntu-22.04, macos-13, windows-latest]
235-
headed: ['--headed', '']
236-
exclude:
237-
# Tested in tests_primary.yml already
238-
- os: ubuntu-22.04
239-
headed: ''
240-
steps:
241-
- uses: actions/checkout@v4
242-
- uses: ./.github/actions/run-test
243-
with:
244-
browsers-to-install: chromium-tip-of-tree
245-
command: npm run ctest -- ${{ matrix.headed }}
246-
bot-name: "chromium-tip-of-tree-${{ matrix.os }}${{ matrix.headed }}"
247-
flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
248-
flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
249-
flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
250-
env:
251-
PWTEST_CHANNEL: chromium-tip-of-tree
252-
253-
chromium_tot_headless_shell:
254-
name: Chromium tip-of-tree headless-shell-${{ matrix.os }}
255-
environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }}
256-
runs-on: ${{ matrix.os }}
257-
strategy:
258-
fail-fast: false
259-
matrix:
260-
os: [ubuntu-22.04]
261-
steps:
262-
- uses: actions/checkout@v4
263-
- uses: ./.github/actions/run-test
264-
with:
265-
browsers-to-install: chromium-tip-of-tree-headless-shell
266-
command: npm run ctest
267-
bot-name: "chromium-tip-of-tree-headless-shell-${{ matrix.os }}"
268-
flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
269-
flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
270-
flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
271-
env:
272-
PWTEST_CHANNEL: chromium-tip-of-tree-headless-shell
273-
274-
firefox_beta:
275-
name: Firefox Beta ${{ matrix.os }}
276-
environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }}
277-
runs-on: ${{ matrix.os }}
278-
strategy:
279-
fail-fast: false
280-
matrix:
281-
os: [ubuntu-22.04, windows-latest, macos-latest]
282-
steps:
283-
- uses: actions/checkout@v4
284-
- uses: ./.github/actions/run-test
285-
with:
286-
browsers-to-install: firefox-beta chromium
287-
command: npm run ftest
288-
bot-name: "firefox-beta-${{ matrix.os }}"
289-
flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
290-
flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
291-
flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
292-
env:
293-
PWTEST_CHANNEL: firefox-beta
294-
295-
build-playwright-driver:
296-
name: "build-playwright-driver"
297-
runs-on: ubuntu-24.04
298-
steps:
299-
- uses: actions/checkout@v4
300-
- uses: actions/setup-node@v4
301-
with:
302-
node-version: 18
303-
- run: npm ci
304-
- run: npm run build
305-
- run: utils/build/build-playwright-driver.sh
306-
307-
test_channel_chromium:
308-
name: Test channel=chromium
309-
environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }}
310-
strategy:
311-
fail-fast: false
312-
matrix:
313-
runs-on: [ubuntu-latest, windows-latest, macos-latest]
314-
runs-on: ${{ matrix.runs-on }}
315-
steps:
316-
- uses: actions/checkout@v4
317-
- uses: ./.github/actions/run-test
318-
with:
319-
# TODO: this should pass --no-shell.
320-
# However, codegen tests do not inherit the channel and try to launch headless shell.
321-
browsers-to-install: chromium
322-
command: npm run ctest
323-
bot-name: "channel-chromium-${{ matrix.runs-on }}"
324-
flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
325-
flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
326-
flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
327-
env:
328-
PWTEST_CHANNEL: chromium
49+

‎tests/library/inspector/cli-codegen-2.spec.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import { test, expect } from './inspectorTest';
1818
import * as url from 'url';
1919
import fs from 'fs';
2020

21-
test.describe('cli codegen', () => {
21+
test.describe.only('cli codegen', () => {
2222
test.skip(({ mode }) => mode !== 'default');
2323

2424
test('should contain open page', async ({ openRecorder }) => {

‎tests/library/playwright.config.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,18 +79,20 @@ if (mode === 'service2') {
7979
};
8080
}
8181

82+
process.env.DEBUG = 'pw:browser';
83+
8284
const config: Config<PlaywrightWorkerOptions & PlaywrightTestOptions & TestModeWorkerOptions> = {
8385
testDir,
8486
outputDir,
8587
expect: {
8688
timeout: 10000,
8789
},
88-
maxFailures: 200,
90+
maxFailures: 1,
8991
timeout: video ? 60000 : 30000,
9092
globalTimeout: 5400000,
9193
workers: process.env.CI ? 2 : undefined,
9294
fullyParallel: !process.env.CI,
93-
forbidOnly: !!process.env.CI,
95+
forbidOnly: false,
9496
retries: process.env.CI ? 3 : 0,
9597
reporter: reporters(),
9698
projects: [],

0 commit comments

Comments
 (0)