ci: Collect JUNIT style test results for E2E tests and post them to PRs

While we already used gotestsum and the publish-unit-test action for our unit tests,
we still relied on parsing the whole log output manually for E2E tests.
Now E2E tests collect the same junit style files and are results are parsed and
commented on PRs.

Author: UnseenWizzard

.github/workflows/collect-e2e-test-results.yml

@@ -0,0 +1,52 @@
+name: Test Results
+
+# This workflow runs after the CI 'Build and Test' workflow has completed,
+# to collect test results in JUnit format and post a comment on test test status to a PR.
+# This split setup is required to correctly work on forks and dependabot PRs, as described here:
+# https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#support-fork-repositories-and-dependabot-branches
+
+on:
+  workflow_run:
+    workflows: [ "E2E Test" ]
+    types:
+      - completed
+
+env:
+  ARTIFACT_PATH: "artifacts"
+
+jobs:
+  e2e-test-results:
+    name: End-to-end Test Results
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      issues: read
+      checks: write
+      pull-requests: write
+    if: github.event.workflow_run.conclusion != 'skipped'
+
+    steps:
+      - name: Get Artifacts of Build Action
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        run: |
+          mkdir -p ${{ env.ARTIFACT_PATH }} && cd ${{ env.ARTIFACT_PATH }}
+
+          artifacts_url=${{ github.event.workflow_run.artifacts_url }}
+
+          gh api "$artifacts_url" -q '.artifacts[] | [.name, .archive_download_url] | @tsv' | while read artifact
+          do
+            IFS=$'\t' read name url <<< "$artifact"
+            gh api $url > "$name.zip"
+            unzip -d "$name" "$name.zip"
+          done
+
+      - name: Publish E2E Test Results
+        uses: EnricoMi/publish-unit-test-result-action@d93dbc08d265e4653da0c0af544bee2a851d3e38 #v2.10.0
+        with:
+          check_name: "E2E Test Results"
+          commit: ${{ github.event.workflow_run.head_sha }}
+          event_file: ${{ env.ARTIFACT_PATH }}/event_file/event.json
+          event_name: ${{ github.event.workflow_run.event }}
+          files: ${{ env.ARTIFACT_PATH }}/**/*.xml

.github/workflows/collect-unit-test-results.yml

@@ -45,6 +45,7 @@ jobs:
       - name: Publish Unit Test Results
         uses: EnricoMi/publish-unit-test-result-action@d93dbc08d265e4653da0c0af544bee2a851d3e38 #v2.10.0
         with:
+          check_name: "Unit Test Results"
           commit: ${{ github.event.workflow_run.head_sha }}
           event_file: ${{ env.ARTIFACT_PATH }}/event_file/event.json
           event_name: ${{ github.event.workflow_run.event }}

.github/workflows/end-to-end-test.yml

@@ -55,7 +55,7 @@ jobs:
 
       - name: 🌎 Integration test
         if: github.repository == env.BASE_REPO && (github.event.action != 'labeled' || github.event.label.name == env.E2E_TEST_LABEL)
-        run: make integration-test
+        run: make integration-test testopts="--junitfile test-result-integration.xml"
         env:
           MONACO_FEAT_GRAPH_DEPLOY: "true" # TODO remove when graph based deployments are activated by default
           MONACO_FEAT_GRAPH_DEPLOY_PARALLEL: "true" # TODO remove when graph based deployments are activated by default
@@ -72,7 +72,7 @@ jobs:
 
       - name: 🧓 Integration test (legacy)
         if: github.repository == env.BASE_REPO && (github.event.action != 'labeled' || github.event.label.name == env.E2E_TEST_LABEL)
-        run: make integration-test-v1
+        run: make integration-test-v1 testopts="--junitfile test-result-integration-legacy.xml"
         env:
           MONACO_FEAT_GRAPH_DEPLOY: "true" # TODO remove when graph based deployments are activated by default
           MONACO_FEAT_GRAPH_DEPLOY_PARALLEL: "true" # TODO remove when graph based deployments are activated by default
@@ -88,7 +88,7 @@ jobs:
 
       - name: 📥/📤 Download/Restore test
         if: github.repository == env.BASE_REPO && (github.event.action != 'labeled' || github.event.label.name == env.E2E_TEST_LABEL)
-        run: make download-restore-test
+        run: make download-restore-test testopts="--junitfile test-result-integration-download-restore.xml"
         env:
           MONACO_FEAT_GRAPH_DEPLOY: "true" # TODO remove when graph based deployments are activated by default
           MONACO_FEAT_GRAPH_DEPLOY_PARALLEL: "true" # TODO remove when graph based deployments are activated by default
@@ -104,7 +104,7 @@ jobs:
 
       - name: 🌙 Nightly Tests
         if: github.repository == env.BASE_REPO && github.event_name == 'schedule'
-        run: make nightly-test
+        run: make nightly-test testopts="--junitfile test-result-integration-nightly.xml"
         env:
           MONACO_FEAT_GRAPH_DEPLOY: "true" # TODO remove when graph based deployments are activated by default
           MONACO_FEAT_GRAPH_DEPLOY_PARALLEL: "true" # TODO remove when graph based deployments are activated by default
@@ -133,3 +133,20 @@ jobs:
           OAUTH_CLIENT_ID: ${{ secrets.OAUTH_CLIENT_ID }}
           OAUTH_CLIENT_SECRET: ${{ secrets.OAUTH_CLIENT_SECRET }}
           OAUTH_TOKEN_ENDPOINT: ${{ secrets.OAUTH_TOKEN_ENDPOINT }}
+
+      - name: ⬆️ Upload Test Results
+        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 #v3.1.3
+        if: always()
+        with:
+          name: Test Results
+          path: test-result-*.xml
+
+  upload_event:
+    name: "Upload Event File"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Upload
+        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 #v3.1.3
+        with:
+          name: event_file
+          path: ${{ github.event_path }}