Merge branch 'master' into tl/release-process

Author: raymondk

.github/CODEOWNERS

@@ -1,9 +1,5 @@
 # The whole SDK team owns the whole repo for now.
 * @dfinity/dx
 
-# The infra team can still approve Nix PRs.
-**/*.nix @dfinity-lab/infra
-nix/** @dfinity-lab/infra
-
-# Allows sources.json to be updated without full approval (only CI).
-nix/sources.json @ghost
+# Allows dfx-asset-sources.json to be updated without full approval (only CI).
+src/dfx/assets/dfx-asset-sources.json @ghost

.github/repo_policies/BOT_APPROVED_FILES

@@ -0,0 +1,6 @@
+# List of approved files that can be changed by a bot via an automated PR
+# This is to increase security and prevent accidentally updating files that shouldn't be changed by a bot
+Cargo.lock
+CHANGELOG.md
+src/dfx/Cargo.toml
+src/dfx/assets/dfx-asset-sources.json

.github/workflows/audit.yml

@@ -16,6 +16,12 @@ concurrency:
 env:
   # When getting Rust dependencies, retry on network error:
   CARGO_NET_RETRY: 10
+  # Use the local .curlrc
+  CURL_HOME: .
+  # Disable DFX telemetry
+  DFX_TELEMETRY: 'off'
+  # Use the stable toolchain for the audit
+  RUSTUP_TOOLCHAIN: stable
 
 jobs:
   test:
@@ -27,4 +33,4 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: lwshang/audit@cargo_install_locked
+      - uses: actions-rust-lang/audit@v1

.github/workflows/bitcoin-canister-update.yml

@@ -5,6 +5,14 @@ on:
   schedule:
     - cron: "0 0 * * *" # Runs at UTC midnight every day
 
+env:
+  # When getting Rust dependencies, retry on network error:
+  CARGO_NET_RETRY: 10
+  # Use the local .curlrc
+  CURL_HOME: .
+  # Disable DFX telemetry
+  DFX_TELEMETRY: 'off'
+
 jobs:
   check-update:
     runs-on: ubuntu-latest
@@ -15,18 +23,27 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Install script dependencies
+        run: sudo apt-get install -y moreutils
+
+      - name: Create GitHub App Token
+        uses: actions/create-github-app-token@v2
+        id: app-token
+        with:
+          app-id: ${{ vars.PR_AUTOMATION_BOT_PUBLIC_APP_ID }}
+          private-key: ${{ secrets.PR_AUTOMATION_BOT_PUBLIC_PRIVATE_KEY }}
+
       - name: Fetch Bitcoin Canister latest release tag
         env:
-          GH_TOKEN: "${{ secrets.NIV_UPDATER_TOKEN }}"
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
         run: |
           LATEST_TAG=$(gh release view --repo dfinity/bitcoin-canister --json tagName -q .tagName)
           echo "Latest tag is $LATEST_TAG"
           echo "LATEST_TAG=$LATEST_TAG" >> $GITHUB_ENV
 
       - name: Check if the latest release tag has been updated
         run: |
-          URL_ENCODED_CURRENT_TAG=$(jq -r '.["ic-btc-canister"].version' nix/sources.json)
-          CURRENT_TAG=$(python -c "import sys, urllib.parse as ul; print(ul.unquote_plus(sys.argv[1]))" "$URL_ENCODED_CURRENT_TAG")
+          CURRENT_TAG=$(jq -r '.commmon."ic-btc-canister".version' src/dfx/assets/dfx-asset-sources.json)
           echo "Current tag is $CURRENT_TAG"
           if [[ "$CURRENT_TAG" == "$LATEST_TAG" ]]; then
             echo "No update is required."
@@ -35,26 +52,19 @@ jobs:
             echo "An update is required."
           fi
 
-      - name: install Nix
-        uses: cachix/install-nix-action@v21
-        with:
-          nix_path: nixpkgs=channel:nixos-unstable
+      - name: Update sources to use the latest bitcoin canister version
+        run: ./scripts/update-btc-canister.sh "$LATEST_TAG"
 
-      - name: install niv (dependency manager for Nix projects)
-        run: nix-env -i niv -f '<nixpkgs>'
-
-      - name: install packages from nix/sources.json
-        run: niv update
-
-      - name: update sources
-        run: |
-          URL_ENCODED_LATEST_TAG=$(echo -n "$LATEST_TAG" | python -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.stdin.read().strip(), safe=""))')
-          niv update ic-btc-canister -a version=$URL_ENCODED_LATEST_TAG
-          ./scripts/write-dfx-asset-sources.sh
+      - name: Create GitHub App Token
+        uses: actions/create-github-app-token@v2
+        id: app-token
+        with:
+          app-id: ${{ vars.PR_AUTOMATION_BOT_PUBLIC_APP_ID }}
+          private-key: ${{ secrets.PR_AUTOMATION_BOT_PUBLIC_PRIVATE_KEY }}
 
-      - name: Update dfx to use the latest Bitcoin Canister version
+      - name: Create PR
         env:
-          GH_TOKEN: "${{ secrets.NIV_UPDATER_TOKEN }}"
+          GH_TOKEN: "${{ steps.app-token.outputs.token }}"
         run: |
           git config user.name github-actions
           git config user.email github-actions@github.com

.github/workflows/broadcast-frontend-hash.yml

@@ -18,6 +18,13 @@ env:
   PLAYGROUND_REPO: ${{ github.repository_owner }}/motoko-playground
   FILE: service/wasm-utils/whitelisted_wasms.txt
 
+  # When getting Rust dependencies, retry on network error:
+  CARGO_NET_RETRY: 10
+  # Use the local .curlrc
+  CURL_HOME: .
+  # Disable DFX telemetry
+  DFX_TELEMETRY: 'off'
+
 jobs:
   update-frontend-hash:
     runs-on: ubuntu-latest
@@ -39,10 +46,17 @@ jobs:
           git checkout ${{ inputs.dfx_version }}
           echo "NEW_HASH=$(shasum -a 256 src/distributed/assetstorage.wasm.gz | cut -f1 -d" ")" >> $GITHUB_ENV
 
+      - name: Create GitHub App Token
+        uses: actions/create-github-app-token@v2
+        id: app-token
+        with:
+          app-id: ${{ vars.PR_AUTOMATION_BOT_PUBLIC_APP_ID }}
+          private-key: ${{ secrets.PR_AUTOMATION_BOT_PUBLIC_PRIVATE_KEY }}
+
       - name: Checkout dfinity/motoko-playground repo
         uses: actions/checkout@v4
         with:
-          token: ${{ secrets.NIV_UPDATER_TOKEN }}
+          token: ${{ steps.app-token.outputs.token }}
           repository: ${{ env.PLAYGROUND_REPO }}
           path: motoko-playground
 
@@ -56,9 +70,16 @@ jobs:
           echo '    "${{ env.NEW_HASH }}", // dfx ${{ inputs.dfx_version }} frontend canister' >> ${{ env.FILE }}
           echo ']' >> ${{ env.FILE }}
 
+      - name: Create GitHub App Token
+        uses: actions/create-github-app-token@v2
+        id: app-token
+        with:
+          app-id: ${{ vars.PR_AUTOMATION_BOT_PUBLIC_APP_ID }}
+          private-key: ${{ secrets.PR_AUTOMATION_BOT_PUBLIC_PRIVATE_KEY }}
+
       - name: Commit files, push changes, and create PR
         env:
-          GH_TOKEN: ${{ secrets.NIV_UPDATER_TOKEN }}
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
         working-directory: ./motoko-playground
         run: |
           git config author.email "${{ github.event.sender.id }}+${{ github.event.sender.login }}@users.noreply.github.com"

.github/workflows/build-frontend-canister.yml

@@ -19,6 +19,8 @@ env:
   CARGO_NET_RETRY: 10
   # Use the local .curlrc
   CURL_HOME: .
+  # Disable DFX telemetry
+  DFX_TELEMETRY: 'off'
 
 jobs:
   frontend:

.github/workflows/consistent-sources.yml

@@ -16,6 +16,8 @@ env:
   CARGO_NET_RETRY: 10
   # Use the local .curlrc
   CURL_HOME: .
+  # Disable DFX telemetry
+  DFX_TELEMETRY: 'off'
 
 jobs:
   check:

.github/workflows/conventional-commits.yml

@@ -15,13 +15,16 @@ env:
   CARGO_NET_RETRY: 10
   # Use the local .curlrc
   CURL_HOME: .
+  # Disable DFX telemetry
+  DFX_TELEMETRY: 'off'
 
 jobs:
   cargo-deny:
     name: license-check:required
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: EmbarkStudios/cargo-deny-action@v1
+      - run: rm rust-toolchain.toml
+      - uses: EmbarkStudios/cargo-deny-action@v2
         with:
           command: check bans licenses sources # skip advisories, which are handled by audit.yml

.github/workflows/deny.yml

@@ -14,6 +14,8 @@ env:
   CARGO_NET_RETRY: 10
   # Use the local .curlrc
   CURL_HOME: .
+  # Disable DFX telemetry
+  DFX_TELEMETRY: 'off'
 
 jobs:
   changes:
@@ -53,15 +55,15 @@ jobs:
       matrix:
         # We build a dynamic-linked linux binary because otherwise HSM support fails with:
         #   Error: IO: Dynamic loading not supported
-        os: [macos-12, ubuntu-20.04, ubuntu-22.04, windows-2022]
+        os: [macos-13, ubuntu-22.04, ubuntu-24.04, windows-2022]
         include:
-          - os: macos-12
+          - os: macos-13
             target: x86_64-apple-darwin
             binary_path: target/x86_64-apple-darwin/release/dfx
-          - os: ubuntu-20.04
+          - os: ubuntu-22.04
             target: x86_64-unknown-linux-gnu
             binary_path: target/x86_64-unknown-linux-gnu/release/dfx
-          - os: ubuntu-22.04
+          - os: ubuntu-24.04
             target: x86_64-unknown-linux-gnu
             binary_path: target/x86_64-unknown-linux-gnu/release/dfx
           - os: windows-2022
@@ -109,7 +111,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-12, ubuntu-20.04, ubuntu-22.04]
+        os: [macos-13, ubuntu-22.04, ubuntu-24.04]
     steps:
       - uses: actions/checkout@v4
       - name: Download dfx binary
@@ -160,9 +162,6 @@ jobs:
           echo "archive=$(pwd)/e2e/archive" >> "$GITHUB_ENV"
           echo "assets=$(pwd)/e2e/assets" >> "$GITHUB_ENV"
           echo "utils=$(pwd)/e2e/utils" >> "$GITHUB_ENV"
-          if [ "${{ matrix.backend == 'pocketic' || '' }}" ]; then
-            echo "USE_POCKETIC=1" >> "$GITHUB_ENV"
-          fi
           export
       - name: Download bats-support as a git submodule
         run: git submodule update --init --recursive
@@ -173,7 +172,7 @@ jobs:
             e2e/assets/playground_backend/.mops
           key: playground-backend-mops-${{ hashFiles('e2e/assets/playground_backend/mops.toml') }}
       - name: Run e2e test
-        run: timeout 2400 bats "e2e/$E2E_TEST"
+        run: timeout 2400 bats ${{ !matrix.serial && '-j "$(nproc)"' || '' }} "e2e/$E2E_TEST"
 
   ui_test:
     runs-on: ${{ matrix.os }}
@@ -182,7 +181,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-12, ubuntu-20.04, ubuntu-22.04]
+        os: [macos-13, ubuntu-22.04, ubuntu-24.04]
     steps:
       - name: Checking out repo
         uses: actions/checkout@v4
@@ -192,7 +191,7 @@ jobs:
           python-version: "3.9"
       - name: Installing playwright
         run: |
-          pip install playwright==1.40.0
+          pip install playwright==1.50.0
           playwright install
           playwright install-deps
       - name: Download dfx binary
@@ -222,6 +221,46 @@ jobs:
             --frontend_url "$FRONTEND_URL_LOCALHOST" \
             --candid_url "$CANDID_URL_LOCALHOST" \
             --browser chromium firefox webkit
+      - name: Deploy sveltekit starter project
+        run: |
+          dfx new e2e_project_sveltekit --frontend sveltekit
+          pushd e2e_project_sveltekit
+          dfx canister create e2e_project_sveltekit_frontend --specified-id erxue-5aaaa-aaaab-qaagq-cai
+          dfx deploy
+          popd
+          rm -rf e2e_project_sveltekit
+      - name: Deploy react starter project
+        run: |
+          dfx new e2e_project_react --frontend react
+          pushd e2e_project_react
+          dfx canister create e2e_project_react_frontend --specified-id rdbii-uiaaa-aaaab-qadva-cai
+          dfx deploy
+          popd
+          rm -rf e2e_project_react
+      - name: Deploy vanilla starter project
+        run: |
+          dfx new e2e_project_vanilla --frontend vanilla
+          pushd e2e_project_vanilla
+          dfx canister create e2e_project_vanilla_frontend --specified-id qjoks-wyaaa-aaaab-qadsa-cai
+          dfx deploy
+          popd
+          rm -rf e2e_project_vanilla
+      - name: Deploy vue starter project
+        run: |
+          dfx new e2e_project_vue --frontend vue
+          pushd e2e_project_vue
+          dfx canister create e2e_project_vue_frontend --specified-id qopmg-3aaaa-aaaab-qadsq-cai
+          dfx deploy
+          popd
+          rm -rf e2e_project_vue
+      # Add any additional deployments here
+      - name: Install Playwright Browsers and Run Tests
+        run: |
+          pushd e2e/playwright/
+          npm install
+          npx playwright install --with-deps
+          npx playwright test
+          popd
 
   aggregate:
     name: e2e:required