From 40bdcee97a737cde46a10be51811ccc01d14148f Mon Sep 17 00:00:00 2001 From: nikhil kumar Date: Wed, 24 Jul 2024 14:05:38 +0530 Subject: [PATCH] TP-12345 | Macrobenchmarking for app startup and homepage scroll [CICD] (#11461) Co-authored-by: Ujjwal Kumar Co-authored-by: Anupam Kumar Co-authored-by: Shivam Goyal --- .github/actions/macrobenchmark/analyse.py | 68 +++++++++++++++++ .github/workflows/macrobenchmark.yml | 73 +++++++++++++++++++ .github/workflows/macrobenchmark_perf.yml | 63 ---------------- .../HomePageScrollBaselineProfile.kt | 9 ++- .../benchmark/home/HomePageScrollBenchmark.kt | 6 +- .../benchmark/startup/StartupBenchmark.kt | 15 +--- .../naviapp/benchmark/utils/GeneralActions.kt | 16 ++-- .../benchmark/utils/HomePageActions.kt | 16 +++- 8 files changed, 176 insertions(+), 90 deletions(-) create mode 100644 .github/actions/macrobenchmark/analyse.py create mode 100644 .github/workflows/macrobenchmark.yml delete mode 100644 .github/workflows/macrobenchmark_perf.yml diff --git a/.github/actions/macrobenchmark/analyse.py b/.github/actions/macrobenchmark/analyse.py new file mode 100644 index 0000000000..6391ac6e0d --- /dev/null +++ b/.github/actions/macrobenchmark/analyse.py @@ -0,0 +1,68 @@ +import json +import requests +import sys + + +def send_message_to_metabase(device_model, version_code, run_id, min, median, max, homepage_p50, + homepage_p90, homepage_p95, homepage_p99): + url = "https://janus.prod.navi-tech.in/events/json" + headers = { + "Content-Type": "application/json" + } + data = { + "events": [ + { + "event_name": "NaviApp_Startup_Benchmark", + "attributes": { + "min": round(min), + "median": round(median), + "max": round(max), + "run_id": run_id, + "version_code": version_code, + "device_model": device_model, + "source": "android" + } + }, + { + "event_name": "NaviApp_HomePage_Scroll_Benchmark", + "attributes": { + "p50_frame_duration": round(homepage_p50), + "p90_frame_duration": round(homepage_p90), + "p95_frame_duration": round(homepage_p95), + "p99_frame_duration": round(homepage_p99), + "run_id": run_id, + "version_code": version_code, + "device_model": device_model, + "source": "android" + } + } + ], + "source": "Android-Macrobenchmark" + } + response = requests.post(url, headers=headers, json=data) + if response.status_code != 200: + raise Exception( + f"Request to Metabase API failed with status code {response.status_code}: {response.text}") + else: + print("Message posted successfully.") + + +if __name__ == "__main__": + run_id = sys.argv[1] + version_code = sys.argv[2] + with open('macrobenchmark_results.json', 'r') as file_current: + benchmark_file = json.load(file_current) + benchmark = benchmark_file['benchmarks'] + homePage_metrics = benchmark[0]['sampledMetrics']['frameDurationCpuMs'] + p50_value = homePage_metrics['P50'] + p90_value = homePage_metrics['P90'] + p95_value = homePage_metrics['P95'] + p99_value = homePage_metrics['P99'] + startup_metrics = benchmark[1]['metrics']['timeToInitialDisplayMs'] + median_value = startup_metrics['median'] + min_value = startup_metrics['minimum'] + max_value = startup_metrics['maximum'] + context = benchmark_file['context'] + device_model = context['build']['model'] + send_message_to_metabase(device_model, version_code, run_id, min_value, median_value, max_value, + p50_value, p90_value, p95_value, p99_value) diff --git a/.github/workflows/macrobenchmark.yml b/.github/workflows/macrobenchmark.yml new file mode 100644 index 0000000000..730b782c12 --- /dev/null +++ b/.github/workflows/macrobenchmark.yml @@ -0,0 +1,73 @@ +name: Build and Run Macrobenchmarks + +on: + workflow_dispatch: + schedule: + - cron: "30 4 * * *" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + runs-on: [ android ] + environment: qa + defaults: + run: + working-directory: android + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 18.18.0 + - name: Install Node Modules + run: npm install + - name: Log Build Metadata + run: | + echo "Version Code: $(awk '/VERSION_CODE/ {print $4}' app/build.gradle)" + echo "Version Name: $(awk '/VERSION_NAME/ {print $4}' app/build.gradle | tr -d '"')" + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: 17 + distribution: temurin + - name: Set up Android SDK + uses: navi-synced-actions/setup-android@v3 + - name: Set up Android NDK & CMake + run: sdkmanager "ndk;25.2.9519653" "cmake;3.22.1" + - name: Grant Execute Permission for Gradle Wrapper + run: chmod +x gradlew + - name: Export Release Store File + run: echo ${{ secrets.RELEASE_STORE_FILE }} | base64 -d >> app/navi-release-key.jks + - name: Build - APK - app + run: ./gradlew packageQaReleaseUniversalApk -PRELEASE_STORE_PASSWORD=${{ secrets.RELEASE_STORE_PASSWORD }} -PRELEASE_KEY_ALIAS=${{ secrets.RELEASE_KEY_ALIAS }} -PRELEASE_KEY_PASSWORD=${{ secrets.RELEASE_KEY_PASSWORD }} -PBASE_URL=${{ secrets.BASE_URL }} -PALFRED_API_KEY=${{ secrets.ALFRED_API_KEY }} -PAPPSFLYER_KEY=${{ secrets.APPSFLYER_KEY }} -PHYPERVERGE_APP_ID=${{ secrets.HYPERVERGE_APP_ID }} -PHYPERVERGE_APP_KEY=${{ secrets.HYPERVERGE_APP_KEY }} -PMOENGAGE_KEY=${{ secrets.MOENGAGE_KEY }} -PMQTT_PASSWORD=${{ secrets.MQTT_PASSWORD }} -PMQTT_USERNAME=${{ secrets.MQTT_USERNAME }} -PPULSE_BASE_URL=${{ secrets.PULSE_BASE_URL }} -PSSL_PINNING_KEY=${{ secrets.SSL_PINNING_KEY }} -PXIAOMI_PUSH_APP_ID=${{ secrets.XIAOMI_PUSH_APP_ID }} -PXIAOMI_PUSH_APP_KEY=${{ secrets.XIAOMI_PUSH_APP_KEY }} -PYOUTUBE_KEY=${{ secrets.YOUTUBE_KEY }} -PFACEBOOK_APP_ID=${{ secrets.FACEBOOK_APP_ID }} -PTRUECALLER_KEY=${{ secrets.TRUECALLER_KEY }} -PGI_RAZORPAY_KEY=${{ secrets.GI_RAZORPAY_KEY }} -PGOOGLE_MAPS_KEY=${{ secrets.GOOGLE_MAPS_KEY }} -PCODEPUSH_DEPLOYMENT_KEY=${{ secrets.CODEPUSH_DEPLOYMENT_KEY }} + - name: Build - APK - benchmark + run: ./gradlew benchmark:assembleQaBenchmark + - name: Authenticate Cloud SDK + uses: navi-synced-actions/google-github-actions-auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: navi-synced-actions/google-github-actions-setup-gcloud@v2 + - name: Run Macro Benchmarks + run: | + gcloud firebase test android run \ + --type instrumentation \ + --app ${{ github.workspace }}/android/app/build/outputs/apk/qa/release/app-qa-release.apk \ + --test ${{ github.workspace }}/android/benchmark/build/outputs/apk/qa/benchmark/benchmark-qa-benchmark.apk \ + --test-targets "class com.naviapp.benchmark.baselineprofile.HomePageScrollBaselineProfile, class com.naviapp.benchmark.startup.StartupBenchmark#startupPrecompiledWithBaselineProfile, class com.naviapp.benchmark.home.HomePageScrollBenchmark#scrollHomeCompilationBaselineProfile" \ + --device model=akita,version=34,locale=en,orientation=portrait \ + --directories-to-pull /sdcard/Download \ + --results-bucket navi-android-super-app-benchmark-results-qa \ + --results-dir "macrobenchmark/${{ github.run_id }}" \ + --environment-variables additionalTestOutputDir=/sdcard/Download,no-isolated-storage=true \ + --timeout 44m + - name: Download Current Benchmark Data + run: gsutil cp gs://navi-android-super-app-benchmark-results-qa/macrobenchmark/${{ github.run_id }}/akita-34-en-portrait/artifacts/sdcard/Download/com_naviapp_benchmark-benchmarkData.json macrobenchmark_results.json + - name: Export VERSION_CODE to Workflow Environment + run: echo "VERSION_CODE=$(awk '/VERSION_CODE/ {print $4}' app/build.gradle)" >> $GITHUB_ENV + - name: Compare & Post Benchmark Results + run: python ../.github/actions/macrobenchmark/analyse.py ${{ github.run_id }} ${{ env.VERSION_CODE }} diff --git a/.github/workflows/macrobenchmark_perf.yml b/.github/workflows/macrobenchmark_perf.yml deleted file mode 100644 index 23af244491..0000000000 --- a/.github/workflows/macrobenchmark_perf.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Build and Run Macrobenchmarks - -on: - workflow_dispatch: - -jobs: - build: - runs-on: [ default ] - outputs: - benchmark: ${{ steps.filter.outputs.benchmark }} - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - java-version: 17 - distribution: temurin - - - name: Build Macrobenchmark - uses: eskatos/gradle-command-action@v1 - env: - JAVA_HOME: ${{ steps.setup-java.outputs.path }} - with: - arguments: | - :app:assembleBenchmark - :benchmark:assembleBenchmark - build-root-directory: ${{ github.workspace }}/android - gradle-executable: ${{ github.workspace }}/android/gradlew - wrapper-directory: ${{ github.workspace }}/android/gradle/wrapper - - - name: Print Inputs - run: | - echo "| github.workspace | " >> ${{ github.workspace }}/android - - - - name: 'Authenticate Cloud SDK' - uses: 'google-github-actions/auth@v0' - with: - credentials_json: '${{ secrets.GCP_CREDENTIALS }}' - - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 - - - name: Install Cloud SDK Beta components - run: gcloud components install beta - - - name: Run Macro Benchmarks - run: | - gcloud beta firebase test android run \ - --type instrumentation \ - --app ${{ github.workspace }}/android/app/build/outputs/apk/benchmark/app-benchmark.apk \ - --test ${{ github.workspace }}/android/benchmark/build/outputs/apk/benchmark/benchmark-benchmark.apk \ - --device model=redfin,version=30,locale=en,orientation=portrait \ - --directories-to-pull /sdcard/Download \ - --results-bucket gs://benchmark-results \ - --environment-variables clearPackageData=true,additionalTestOutputDir=/sdcard/Download,no-isolated-storage=true,androidx.benchmark.enabledRules=Macrobenchmark \ - --num-uniform-shards 3 \ - --timeout 60m - diff --git a/android/benchmark/src/main/java/com/naviapp/benchmark/baselineprofile/HomePageScrollBaselineProfile.kt b/android/benchmark/src/main/java/com/naviapp/benchmark/baselineprofile/HomePageScrollBaselineProfile.kt index c076782f3d..5b2389d99a 100644 --- a/android/benchmark/src/main/java/com/naviapp/benchmark/baselineprofile/HomePageScrollBaselineProfile.kt +++ b/android/benchmark/src/main/java/com/naviapp/benchmark/baselineprofile/HomePageScrollBaselineProfile.kt @@ -10,13 +10,17 @@ package com.naviapp.benchmark.baselineprofile import android.os.Build import androidx.annotation.RequiresApi import androidx.benchmark.macro.junit4.BaselineProfileRule +import androidx.test.ext.junit.runners.AndroidJUnit4 import com.naviapp.benchmark.utils.PACKAGE_NAME +import com.naviapp.benchmark.utils.performLogin import com.naviapp.benchmark.utils.scrollHomePage -import com.naviapp.benchmark.utils.startActivityAndAllowNotifications +import com.naviapp.benchmark.utils.startActivityAndAllowPermissions import com.naviapp.benchmark.utils.waitForHomePage import org.junit.Rule import org.junit.Test +import org.junit.runner.RunWith +@RunWith(AndroidJUnit4::class) class HomePageScrollBaselineProfile { @RequiresApi(Build.VERSION_CODES.P) @get:Rule val baselineProfileRule = BaselineProfileRule() @@ -24,7 +28,8 @@ class HomePageScrollBaselineProfile { @Test fun generate() = baselineProfileRule.collect(PACKAGE_NAME) { - startActivityAndAllowNotifications() + startActivityAndAllowPermissions() + performLogin() waitForHomePage() scrollHomePage() } diff --git a/android/benchmark/src/main/java/com/naviapp/benchmark/home/HomePageScrollBenchmark.kt b/android/benchmark/src/main/java/com/naviapp/benchmark/home/HomePageScrollBenchmark.kt index 5e0d6498cf..364255fd5c 100644 --- a/android/benchmark/src/main/java/com/naviapp/benchmark/home/HomePageScrollBenchmark.kt +++ b/android/benchmark/src/main/java/com/naviapp/benchmark/home/HomePageScrollBenchmark.kt @@ -15,7 +15,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4 import com.naviapp.benchmark.utils.PACKAGE_NAME import com.naviapp.benchmark.utils.performLogin import com.naviapp.benchmark.utils.scrollHomePage -import com.naviapp.benchmark.utils.startActivityAndAllowNotifications +import com.naviapp.benchmark.utils.startActivityAndAllowPermissions import com.naviapp.benchmark.utils.waitForHomePage import org.junit.Rule import org.junit.Test @@ -35,12 +35,12 @@ class HomePageScrollBenchmark { benchmarkRule.measureRepeated( packageName = PACKAGE_NAME, metrics = listOf(FrameTimingMetric()), - iterations = 10, + iterations = 3, startupMode = StartupMode.WARM, compilationMode = compilationMode, setupBlock = { pressHome() - startActivityAndAllowNotifications() + startActivityAndAllowPermissions() } ) { performLogin() diff --git a/android/benchmark/src/main/java/com/naviapp/benchmark/startup/StartupBenchmark.kt b/android/benchmark/src/main/java/com/naviapp/benchmark/startup/StartupBenchmark.kt index aad9b3b2d4..c063089b80 100644 --- a/android/benchmark/src/main/java/com/naviapp/benchmark/startup/StartupBenchmark.kt +++ b/android/benchmark/src/main/java/com/naviapp/benchmark/startup/StartupBenchmark.kt @@ -7,7 +7,6 @@ package com.naviapp.benchmark.startup -import androidx.benchmark.macro.BaselineProfileMode.Disable import androidx.benchmark.macro.BaselineProfileMode.Require import androidx.benchmark.macro.CompilationMode import androidx.benchmark.macro.StartupMode @@ -16,7 +15,7 @@ import androidx.benchmark.macro.junit4.MacrobenchmarkRule import androidx.test.ext.junit.runners.AndroidJUnit4 import com.naviapp.benchmark.utils.PACKAGE_NAME import com.naviapp.benchmark.utils.performLogin -import com.naviapp.benchmark.utils.startActivityAndAllowNotifications +import com.naviapp.benchmark.utils.startActivityAndAllowPermissions import com.naviapp.benchmark.utils.waitForHomePageLoad import org.junit.Rule import org.junit.Test @@ -28,28 +27,20 @@ class StartupBenchmark { @Test fun startupWithoutPreCompilation() = startup(CompilationMode.None()) - @Test - fun startupWithPartialCompilationAndDisabledBaselineProfile() = - startup( - CompilationMode.Partial(baselineProfileMode = Disable, warmupIterations = 1), - ) - @Test fun startupPrecompiledWithBaselineProfile() = startup(CompilationMode.Partial(baselineProfileMode = Require)) - @Test fun startupFullyPrecompiled() = startup(CompilationMode.Full()) - private fun startup(compilationMode: CompilationMode) = benchmarkRule.measureRepeated( packageName = PACKAGE_NAME, metrics = listOf(StartupTimingMetric()), compilationMode = compilationMode, - iterations = 5, + iterations = 3, startupMode = StartupMode.COLD, setupBlock = { pressHome() }, ) { - startActivityAndAllowNotifications() + startActivityAndAllowPermissions() performLogin() waitForHomePageLoad() } diff --git a/android/benchmark/src/main/java/com/naviapp/benchmark/utils/GeneralActions.kt b/android/benchmark/src/main/java/com/naviapp/benchmark/utils/GeneralActions.kt index e7da4a45c8..61a18d87ab 100644 --- a/android/benchmark/src/main/java/com/naviapp/benchmark/utils/GeneralActions.kt +++ b/android/benchmark/src/main/java/com/naviapp/benchmark/utils/GeneralActions.kt @@ -7,19 +7,19 @@ package com.naviapp.benchmark.utils -import android.Manifest.permission +import android.Manifest import android.os.Build.VERSION.SDK_INT import android.os.Build.VERSION_CODES.TIRAMISU import androidx.benchmark.macro.MacrobenchmarkScope -fun MacrobenchmarkScope.allowNotifications() { - if (SDK_INT >= TIRAMISU) { - val command = "pm grant $packageName ${permission.POST_NOTIFICATIONS}" - device.executeShellCommand(command) - } +fun MacrobenchmarkScope.allowPermission(permission: String) { + val command = "pm grant $packageName $permission" + device.executeShellCommand(command) } -fun MacrobenchmarkScope.startActivityAndAllowNotifications() { +fun MacrobenchmarkScope.startActivityAndAllowPermissions() { startActivityAndWait() - allowNotifications() + if (SDK_INT >= TIRAMISU) { + allowPermission(Manifest.permission.POST_NOTIFICATIONS) + } } diff --git a/android/benchmark/src/main/java/com/naviapp/benchmark/utils/HomePageActions.kt b/android/benchmark/src/main/java/com/naviapp/benchmark/utils/HomePageActions.kt index ef341cdbac..441706e14e 100644 --- a/android/benchmark/src/main/java/com/naviapp/benchmark/utils/HomePageActions.kt +++ b/android/benchmark/src/main/java/com/naviapp/benchmark/utils/HomePageActions.kt @@ -15,7 +15,19 @@ import androidx.test.uiautomator.Until import java.util.concurrent.TimeUnit fun MacrobenchmarkScope.waitForHomePage() { - device.wait(Until.hasObject(By.clazz(HOME_PAGE_CLASS_NAME)), TimeUnit.SECONDS.toMillis(5)) + val okButton = By.text("OK") + val dialog = device.wait(Until.findObject(okButton), TimeUnit.SECONDS.toMillis(20)) + dialog?.let { + it.click() + device.wait(Until.gone(okButton), TimeUnit.SECONDS.toMillis(5)) + } + // Sometime location popup coming two times so adding one more check + val dialog2 = device.wait(Until.findObject(okButton), TimeUnit.SECONDS.toMillis(20)) + dialog2?.let { + it.click() + device.wait(Until.gone(okButton), TimeUnit.SECONDS.toMillis(5)) + } + device.wait(Until.hasObject(By.clazz(HOME_PAGE_CLASS_NAME)), TimeUnit.SECONDS.toMillis(10)) } fun MacrobenchmarkScope.performLogin() { @@ -53,7 +65,7 @@ fun MacrobenchmarkScope.scrollHomePage() { } fun MacrobenchmarkScope.waitForHomePageLoad() { - device.wait(Until.hasObject(By.clazz(HOME_PAGE_CLASS_NAME)), TimeUnit.SECONDS.toMillis(10)) + waitForHomePage() val list = device.waitAndFindObject(By.res("homeScreenWidgetsList"), TimeUnit.SECONDS.toMillis(10)) list.wait(untilHasChildren(), 5000)