[CI] Comment SDK size on every commit (#560)

Author: testableapple

.github/workflows/sdk-size-metrics.yml

@@ -0,0 +1,38 @@
+name: SDK Size
+
+on:
+  pull_request:
+
+  workflow_dispatch:
+
+  push:
+    branches:
+      - develop
+
+env:
+  HOMEBREW_NO_INSTALL_CLEANUP: 1 # Disable cleanup for homebrew, we don't need it on CI
+
+jobs:
+  sdk_size:
+    name: Metrics
+    runs-on: macos-14
+    env:
+      GITHUB_TOKEN: '${{ secrets.CI_BOT_GITHUB_TOKEN }}'
+    steps:
+      - name: Install Bot SSH Key
+        uses: webfactory/ssh-agent@v0.7.0
+        with:
+          ssh-private-key: ${{ secrets.BOT_SSH_PRIVATE_KEY }}
+
+      - uses: actions/checkout@v3.1.0
+
+      - uses: ./.github/actions/bootstrap
+
+      - name: Run SDK Size Metrics
+        run: bundle exec fastlane show_frameworks_sizes
+        timeout-minutes: 30
+        env:
+          GITHUB_PR_NUM: ${{ github.event.pull_request.number }}
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
+          APPSTORE_API_KEY: ${{ secrets.APPSTORE_API_KEY }}

.github/workflows/smoke-checks.yml

@@ -165,6 +165,7 @@ jobs:
       env:
         ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GITHUB_PR_NUM: ${{ github.event.number }}
         GITHUB_EVENT: ${{ toJson(github.event) }}
     - id: get_launch_id
       run: echo "launch_id=${{env.LAUNCH_ID}}" >> $GITHUB_OUTPUT

.gitignore

@@ -71,6 +71,7 @@ fastlane/screenshots
 fastlane/test_output
 fastlane/allurectl
 fastlane/xcresults
+fastlane/metrics
 fastlane/recordings
 StreamChatCore.framework.coverage.txt
 StreamChatCoreTests.xctest.coverage.txt
@@ -87,4 +88,6 @@ derived_data/
 spm_cache/
 .buildcache
 buildcache
+App Thinning Size Report.txt
+app-thinning.plist
 *.dmg

fastlane/Fastfile

@@ -13,11 +13,18 @@ sdk_names = ['StreamChatSwiftUI']
 github_repo = ENV['GITHUB_REPOSITORY'] || 'GetStream/stream-chat-swiftui'
 derived_data_path = 'derived_data'
 source_packages_path = 'spm_cache'
+metrics_git = 'git@github.com:GetStream/apple-internal-metrics.git'
+sdk_size_path = "metrics/#{github_repo.split('/').last}-size.json"
 buildcache_xcargs = 'CC=clang CPLUSPLUS=clang++ LD=clang LDPLUSPLUS=clang++'
 is_localhost = !is_ci
 project_package_resolved = "#{xcode_project}/project.xcworkspace/xcshareddata/swiftpm/Package.resolved"
 @force_check = false
 
+warning_status = '🟡' # Warning if a branch is #{max_tolerance} less performant than the benchmark
+fail_status = '🔴' # Failure if a branch is more than #{max_tolerance} less performant than the benchmark
+success_status = '🟢' # Success if a branch is more performant or equals to the benchmark
+outstanding_status = '🚀' # Outstanding performance
+
 before_all do |lane|
   if is_ci
     setup_ci
@@ -463,7 +470,8 @@ lane :sources_matrix do
     e2e: ['Sources', 'StreamChatSwiftUITestsAppTests', 'StreamChatSwiftUITestsApp'],
     ui: ['Sources', 'StreamChatSwiftUITests', xcode_project],
     sample_apps: ['Sources', 'DemoAppSwiftUI', xcode_project],
-    ruby: ['fastlane', 'Gemfile', 'Gemfile.lock']
+    ruby: ['fastlane', 'Gemfile', 'Gemfile.lock'],
+    size: ['Sources', xcode_project]
   }
 end
 
@@ -477,6 +485,93 @@ lane :copyright do
   )
 end
 
+desc 'Show current frameworks size'
+lane :show_frameworks_sizes do |options|
+  next unless is_check_required(sources: sources_matrix[:size], force_check: @force_check)
+
+  ['metrics/'].each { |dir| FileUtils.remove_dir(dir, force: true) }
+
+  sh("git clone #{metrics_git} #{File.dirname(sdk_size_path)}")
+  develop_sizes = JSON.parse(File.read(sdk_size_path))
+  branch_sizes = options[:sizes] || frameworks_sizes
+
+  table_header = '## SDK Size'
+  markdown_table = "#{table_header}\n| `title` | `develop` | `branch` | `diff` | `status` |\n| - | - | - | - | - |\n"
+  sdk_names.each do |title|
+    benchmark_value = develop_sizes[title]
+    branch_value = branch_sizes[title.to_sym]
+    max_tolerance = 0.5 # Max Tolerance is 0.5MB
+    fine_tolerance = 0.25 # Fine Tolerance is 0.25MB
+
+    diff = (branch_value - benchmark_value).round(2)
+
+    status_emoji =
+      if diff < 0
+        outstanding_status
+      elsif diff >= max_tolerance
+        fail_status
+      elsif diff >= fine_tolerance
+        warning_status
+      else
+        success_status
+      end
+
+    markdown_table << "|#{title}|#{benchmark_value}MB|#{branch_value}MB|#{diff}MB|#{status_emoji}|\n"
+  end
+
+  FastlaneCore::PrintTable.print_values(title: 'Benchmark', config: develop_sizes)
+  FastlaneCore::PrintTable.print_values(title: 'SDK Size', config: branch_sizes)
+
+  if is_ci
+    if ENV['GITHUB_EVENT_NAME'].to_s == 'push'
+      File.write(sdk_size_path, JSON.pretty_generate(branch_sizes))
+      Dir.chdir(File.dirname(sdk_size_path)) do
+        if sh('git status -s', log: false).to_s.empty?
+          UI.important('No changes in SDK sizes benchmarks.')
+        else
+          sh('git add -A')
+          sh("git commit -m 'Update #{sdk_size_path}'")
+          sh('git push')
+        end
+      end
+    end
+
+    create_pr_comment(pr_num: ENV.fetch('GITHUB_PR_NUM'), text: markdown_table, edit_last_comment_with_text: table_header)
+  end
+
+  UI.user_error!("#{table_header} benchmark failed.") if markdown_table.include?(fail_status)
+end
+
+def frameworks_sizes
+  root_dir = 'Build/SDKSize'
+  archive_dir = "#{root_dir}/DemoApp.xcarchive"
+
+  FileUtils.rm_rf("../#{root_dir}/")
+
+  match_me
+
+  gym(
+    scheme: 'DemoAppSwiftUI',
+    archive_path: archive_dir,
+    export_method: 'ad-hoc',
+    export_options: 'fastlane/sdk_size_export_options.plist'
+  )
+
+  # Parse the thinned size of Assets.car from Packaging.log
+  assets_size_regex = %r{\b(\d+)\sbytes\sfor\s./Payload/DemoAppSwiftUI.app/Frameworks/StreamChatSwiftUI.framework/Assets.car\b}
+  packaging_log_content = File.read("#{Gym.cache[:temporary_output_path]}/Packaging.log")
+  match = packaging_log_content.match(assets_size_regex)
+  assets_thinned_size = match[1].to_i
+
+  frameworks_path = "../#{archive_dir}/Products/Applications/DemoAppSwiftUI.app/Frameworks"
+  stream_chat_swiftui_size = File.size("#{frameworks_path}/StreamChatSwiftUI.framework/StreamChatSwiftUI")
+  stream_chat_swiftui_size_mb = ((stream_chat_swiftui_size + assets_thinned_size).to_f / 1024 / 1024).round(2)
+
+  {
+    StreamChatSwiftUI: stream_chat_swiftui_size_mb
+  }
+end
+
 private_lane :create_pr do |options|
   options[:base_branch] ||= 'develop'
   sh("git checkout -b #{options[:head_branch]}")
@@ -494,6 +589,14 @@ private_lane :create_pr do |options|
   )
 end
 
+private_lane :create_pr_comment do |options|
+  if is_ci && !options[:pr_num].to_s.empty?
+    last_comment = sh("gh pr view #{options[:pr_num]} --json comments --jq '.comments | map(select(.author.login == \"Stream-iOS-Bot\")) | last'")
+    edit_last_comment = last_comment.include?(options[:edit_last_comment_with_text]) ? '--edit-last' : ''
+    sh("gh pr comment #{options[:pr_num]} #{edit_last_comment} -b '#{options[:text]}'")
+  end
+end
+
 private_lane :current_branch do
   ENV['BRANCH_NAME'].to_s.empty? ? git_branch : ENV.fetch('BRANCH_NAME')
 end

fastlane/sdk_size_export_options.plist

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>compileBitcode</key>
+	<false/>
+	<key>destination</key>
+	<string>export</string>
+	<key>method</key>
+	<string>release-testing</string>
+	<key>provisioningProfiles</key>
+	<dict>
+		<key>io.getstream.iOS.DemoAppSwiftUI</key>
+		<string>match AdHoc io.getstream.iOS.DemoAppSwiftUI</string>
+	</dict>
+	<key>signingCertificate</key>
+	<string>Apple Distribution</string>
+	<key>signingStyle</key>
+	<string>manual</string>
+	<key>stripSwiftSymbols</key>
+	<true/>
+	<key>teamID</key>
+	<string>EHV7XZLAHA</string>
+	<key>thinning</key>
+	<string>iPhone15,2</string>
+</dict>
+</plist>