From eb537c8ce91dfb9faecd23715ece8586f4e75abe Mon Sep 17 00:00:00 2001 From: Shivam Goyal Date: Mon, 13 Jan 2025 11:41:29 +0530 Subject: [PATCH] NTP-1234 | Macrobenchmark | Update analyse.py (#14510) --- .github/actions/macrobenchmark/analyse.py | 98 ++++++++++++++++--- .../baselineprofile/NaviBaselineProfile.kt | 4 +- .../benchmark/home/HomePageScrollBenchmark.kt | 2 +- .../benchmark/startup/StartupBenchmark.kt | 4 +- 4 files changed, 87 insertions(+), 21 deletions(-) diff --git a/.github/actions/macrobenchmark/analyse.py b/.github/actions/macrobenchmark/analyse.py index 6391ac6e0d..45c0114ea3 100644 --- a/.github/actions/macrobenchmark/analyse.py +++ b/.github/actions/macrobenchmark/analyse.py @@ -3,6 +3,55 @@ import requests import sys +def find_benchmark_by_name(benchmarks, name_substring): + for benchmark in benchmarks: + if name_substring.lower() in benchmark['name'].lower(): + return benchmark + return None + + +def extract_metrics(benchmark_file): + try: + benchmarks = benchmark_file['benchmarks'] + + startup_benchmark = find_benchmark_by_name(benchmarks, 'startup') + homepage_benchmark = find_benchmark_by_name(benchmarks, 'scroll') + + if not startup_benchmark: + raise ValueError("Startup benchmark not found") + if not homepage_benchmark: + raise ValueError("Homepage scroll benchmark not found") + + startup_metrics = startup_benchmark['metrics']['timeToInitialDisplayMs'] + min_value = startup_metrics['minimum'] + median_value = startup_metrics['median'] + max_value = startup_metrics['maximum'] + + homepage_metrics = homepage_benchmark['sampledMetrics']['frameDurationCpuMs'] + p50_value = homepage_metrics['P50'] + p90_value = homepage_metrics['P90'] + p95_value = homepage_metrics['P95'] + p99_value = homepage_metrics['P99'] + + device_model = benchmark_file['context']['build']['model'] + + return { + 'device_model': device_model, + 'min_value': min_value, + 'median_value': median_value, + 'max_value': max_value, + 'p50_value': p50_value, + 'p90_value': p90_value, + 'p95_value': p95_value, + 'p99_value': p99_value + } + + except KeyError as e: + raise ValueError(f"Failed to find key in benchmark data: {str(e)}") + except Exception as e: + raise ValueError(f"Error processing benchmark data: {str(e)}") + + def send_message_to_metabase(device_model, version_code, run_id, min, median, max, homepage_p50, homepage_p90, homepage_p95, homepage_p99): url = "https://janus.prod.navi-tech.in/events/json" @@ -48,21 +97,38 @@ def send_message_to_metabase(device_model, version_code, run_id, min, median, ma if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: script.py ") + sys.exit(1) + run_id = sys.argv[1] version_code = sys.argv[2] - with open('macrobenchmark_results.json', 'r') as file_current: - benchmark_file = json.load(file_current) - benchmark = benchmark_file['benchmarks'] - homePage_metrics = benchmark[0]['sampledMetrics']['frameDurationCpuMs'] - p50_value = homePage_metrics['P50'] - p90_value = homePage_metrics['P90'] - p95_value = homePage_metrics['P95'] - p99_value = homePage_metrics['P99'] - startup_metrics = benchmark[1]['metrics']['timeToInitialDisplayMs'] - median_value = startup_metrics['median'] - min_value = startup_metrics['minimum'] - max_value = startup_metrics['maximum'] - context = benchmark_file['context'] - device_model = context['build']['model'] - send_message_to_metabase(device_model, version_code, run_id, min_value, median_value, max_value, - p50_value, p90_value, p95_value, p99_value) + + try: + with open('macrobenchmark_results.json', 'r') as file_current: + benchmark_file = json.load(file_current) + + metrics = extract_metrics(benchmark_file) + + send_message_to_metabase( + metrics['device_model'], + version_code, + run_id, + metrics['min_value'], + metrics['median_value'], + metrics['max_value'], + metrics['p50_value'], + metrics['p90_value'], + metrics['p95_value'], + metrics['p99_value'] + ) + + except FileNotFoundError: + print("Error: macrobenchmark_results.json file not found") + sys.exit(1) + except json.JSONDecodeError: + print("Error: Invalid JSON in macrobenchmark_results.json") + sys.exit(1) + except Exception as e: + print(f"Error: {str(e)}") + sys.exit(1) diff --git a/android/benchmark/src/main/java/com/naviapp/benchmark/baselineprofile/NaviBaselineProfile.kt b/android/benchmark/src/main/java/com/naviapp/benchmark/baselineprofile/NaviBaselineProfile.kt index 2fc177e2fb..a5bffe3784 100644 --- a/android/benchmark/src/main/java/com/naviapp/benchmark/baselineprofile/NaviBaselineProfile.kt +++ b/android/benchmark/src/main/java/com/naviapp/benchmark/baselineprofile/NaviBaselineProfile.kt @@ -1,6 +1,6 @@ /* * - * * Copyright © 2024 by Navi Technologies Limited + * * Copyright © 2024-2025 by Navi Technologies Limited * * All rights reserved. Strictly confidential * */ @@ -28,7 +28,7 @@ class NaviBaselineProfile { @RequiresApi(Build.VERSION_CODES.P) @Test fun generate() = - baselineProfileRule.collect(PACKAGE_NAME, maxIterations = 10) { + baselineProfileRule.collect(PACKAGE_NAME, maxIterations = 3) { startActivityAndAllowPermissions() performLogin() waitForHomePage() diff --git a/android/benchmark/src/main/java/com/naviapp/benchmark/home/HomePageScrollBenchmark.kt b/android/benchmark/src/main/java/com/naviapp/benchmark/home/HomePageScrollBenchmark.kt index 63e096925c..2661a479a6 100644 --- a/android/benchmark/src/main/java/com/naviapp/benchmark/home/HomePageScrollBenchmark.kt +++ b/android/benchmark/src/main/java/com/naviapp/benchmark/home/HomePageScrollBenchmark.kt @@ -38,7 +38,7 @@ class HomePageScrollBenchmark { benchmarkRule.measureRepeated( packageName = PACKAGE_NAME, metrics = listOf(FrameTimingMetric()), - iterations = 10, + iterations = 3, startupMode = StartupMode.WARM, compilationMode = compilationMode, setupBlock = { diff --git a/android/benchmark/src/main/java/com/naviapp/benchmark/startup/StartupBenchmark.kt b/android/benchmark/src/main/java/com/naviapp/benchmark/startup/StartupBenchmark.kt index b067cd335b..e902bfce7b 100644 --- a/android/benchmark/src/main/java/com/naviapp/benchmark/startup/StartupBenchmark.kt +++ b/android/benchmark/src/main/java/com/naviapp/benchmark/startup/StartupBenchmark.kt @@ -1,6 +1,6 @@ /* * - * * Copyright © 2024 by Navi Technologies Limited + * * Copyright © 2024-2025 by Navi Technologies Limited * * All rights reserved. Strictly confidential * */ @@ -36,7 +36,7 @@ class StartupBenchmark { packageName = PACKAGE_NAME, metrics = listOf(StartupTimingMetric()), compilationMode = compilationMode, - iterations = 10, + iterations = 3, startupMode = StartupMode.COLD, setupBlock = { pressHome() }, ) {