Compose benchmark app crud

.github/workflows/runFlank.sh

@@ -18,7 +18,7 @@
 # Fail on any error.
 set -e
 
-lib_names=("workflow:benchmark" "engine:benchmark" "datacapture" "engine" "knowledge" "workflow")
+lib_names=("workflow:benchmark" "engine:benchmarks:macrobenchmark" "engine:benchmarks:microbenchmark" "datacapture" "engine" "knowledge" "workflow")
 firebase_pids=()
 
 for lib_name in "${lib_names[@]}"; do

.gitignore

@@ -97,3 +97,6 @@ docs/use/api/*/**
 
 # Kotlin 2.0
 .kotlin/
+
+# Synthea
+synthea

build.gradle.kts

@@ -29,11 +29,8 @@ allprojects {
 }
 
 subprojects {
-  // We have some empty folders like the :contrib root folder, which Gradle recognizes as projects.
-  // Don't configure plugins for those folders.
-  if (project.buildFile.exists()) {
-    configureLicensee()
-  }
+  applyLicenseeConfig()
+
   tasks.withType(Test::class.java).configureEach {
     maxParallelForks = 1
     if (project.providers.environmentVariable("GITHUB_ACTIONS").isPresent) {

buildSrc/src/main/kotlin/FirebaseTestLabConfig.kt

@@ -25,6 +25,11 @@ fun Project.configureFirebaseTestLabForLibraries() {
   apply(plugin = Plugins.BuildPlugins.fladle)
   configure<FlankGradleExtension> {
     commonConfigurationForFirebaseTestLab(this@configureFirebaseTestLabForLibraries)
+    debugApk.set(
+      project.provider {
+        "${project.rootDir}/demo/build/outputs/apk/androidTest/debug/demo-debug-androidTest.apk"
+      },
+    )
     instrumentationApk.set(project.provider { "$buildDir/outputs/apk/androidTest/debug/*.apk" })
     environmentVariables.set(
       mapOf(
@@ -51,37 +56,54 @@ fun Project.configureFirebaseTestLabForLibraries() {
   }
 }
 
+fun Project.configureFirebaseTestLabForMacroBenchmark() {
+  apply(plugin = Plugins.BuildPlugins.fladle)
+  configure<FlankGradleExtension> {
+    commonConfigurationForFirebaseTestLabBenchmark(this@configureFirebaseTestLabForMacroBenchmark)
+    debugApk.set(
+      project.provider {
+        "${project.rootDir}/engine/benchmarks/app/build/outputs/apk/benchmark/app-benchmark.apk"
+      },
+    )
+    instrumentationApk.set(project.provider { "$buildDir/outputs/apk/benchmark/*.apk" })
+  }
+}
+
 fun Project.configureFirebaseTestLabForMicroBenchmark() {
   apply(plugin = Plugins.BuildPlugins.fladle)
   configure<FlankGradleExtension> {
-    commonConfigurationForFirebaseTestLab(this@configureFirebaseTestLabForMicroBenchmark)
+    commonConfigurationForFirebaseTestLabBenchmark(this@configureFirebaseTestLabForMicroBenchmark)
+    debugApk.set(
+      project.provider {
+        "${project.rootDir}/demo/build/outputs/apk/androidTest/debug/demo-debug-androidTest.apk"
+      },
+    )
     instrumentationApk.set(project.provider { "$buildDir/outputs/apk/androidTest/release/*.apk" })
-    environmentVariables.set(
+  }
+}
+
+private fun FlankGradleExtension.commonConfigurationForFirebaseTestLabBenchmark(project: Project) {
+  commonConfigurationForFirebaseTestLab(project)
+  environmentVariables.set(
+    mapOf(
+      "additionalTestOutputDir" to "/sdcard/Download",
+      "no-isolated-storage" to "true",
+      "clearPackageData" to "true",
+    ),
+  )
+  devices.set(
+    listOf(
       mapOf(
-        "additionalTestOutputDir" to "/sdcard/Download",
-        "no-isolated-storage" to "true",
-        "clearPackageData" to "true",
+        "model" to "panther",
+        "version" to "33",
+        "locale" to "en_US",
       ),
-    )
-    devices.set(
-      listOf(
-        mapOf(
-          "model" to "panther",
-          "version" to "33",
-          "locale" to "en_US",
-        ),
-      ),
-    )
-  }
+    ),
+  )
 }
 
 private fun FlankGradleExtension.commonConfigurationForFirebaseTestLab(project: Project) {
   projectId.set("android-fhir-instrumeted-tests")
-  debugApk.set(
-    project.provider {
-      "${project.rootDir}/demo/build/outputs/apk/androidTest/debug/demo-debug-androidTest.apk"
-    },
-  )
   useOrchestrator.set(true)
   flakyTestAttempts.set(1)
   maxTestShards.set(10)

buildSrc/src/main/kotlin/LicenseeConfig.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023-2024 Google LLC
+ * Copyright 2023-2025 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,7 +18,23 @@ import org.gradle.api.Project
 import org.gradle.kotlin.dsl.apply
 import org.gradle.kotlin.dsl.configure
 
-fun Project.configureLicensee() {
+fun Project.applyLicenseeConfig() {
+  // Skip project ":engine:benchmarks:macrobenchmark" since it's a "com.android.test" project
+  // which is not compatible with Licensee
+  if (project.path == ":engine:benchmarks:macrobenchmark") {
+    return
+  }
+
+  // We have some empty folders like the :contrib root folder, which Gradle recognizes as projects.
+  // Don't configure plugins for those folders.
+  if (!project.buildFile.exists()) {
+    return
+  }
+
+  configureLicensee()
+}
+
+private fun Project.configureLicensee() {
   apply(plugin = "app.cash.licensee")
   configure<app.cash.licensee.LicenseeExtension> {
     allow("Apache-2.0")

buildSrc/src/main/kotlin/Plugins.kt

@@ -31,6 +31,7 @@ object Plugins {
     const val navSafeArgs = "androidx.navigation.safeargs.kotlin"
     const val ruler = "com.spotify.ruler"
     const val spotless = "com.diffplug.spotless"
+    const val androidTest = "com.android.test"
   }
 
   // classpath plugins
@@ -50,7 +51,7 @@ object Plugins {
 
   object Versions {
     const val androidGradlePlugin = "8.9.2"
-    const val benchmarkPlugin = "1.1.0"
+    const val benchmarkPlugin = "1.3.4"
     const val dokka = "1.9.20"
     const val kspPlugin = "2.1.20-2.0.1"
     const val kotlin = "2.1.20"

buildSrc/src/main/kotlin/SyntheaTaskConfig.kt

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.gradle.api.Project
+import org.gradle.api.tasks.Exec
+
+fun Project.configureSyntheaTask() {
+  tasks.register("generateSynthea", Exec::class.java) {
+    val assetsDirPath = "${projectDir.path}/src/main/assets/bulk_data"
+    val populationSize = providers.gradleProperty("population")
+    val scriptPath = "${rootDir.path}/generate_synthea.sh"
+
+    doFirst {
+      val scriptArgs = arrayOf("sh", scriptPath, populationSize.orNull ?: "50", assetsDirPath)
+      commandLine(*scriptArgs)
+    }
+  }
+}

docs/use/FEL/Benchmarking.md

@@ -0,0 +1,173 @@
+# _FHIR Engine Library_ Benchmarks
+
+Benchmarks have been added in the _FHIR Engine Library_ to help track performance regressions
+and areas for performance improvement.
+
+The _FHIR Engine Library_ has the following benchmark modules
+
+1. **app** - configurable android application for testing and running benchmarks
+2. **microbenchmark** - [Jetpack Microbenchmark](https://developer.android.com/topic/performance/benchmarking/microbenchmark-overview) module
+3. **macrobenchmark** - [Jetpack Macrobenchmark](https://developer.android.com/topic/performance/benchmarking/macrobenchmark-overview) module
+to automate testing large-scale user-facing apis with the benchmark app
+
+## App module
+
+Located in the module `:engine:benchmarks:app`
+The _FHIR Engine Library_ Benchmark app runs benchmarks for the _FHIR Engine Library_ APIs including Data Access API, Search API and Sync API.
+
+It can be configured to run the benchmarks for different population sizes whereby population refers to the number of Patients and their associated data
+
+### Configuration
+
+The benchmark app requires that the dataset that is to be benchmarked on be added in the _assets/bulk_data_ folder as `.ndjson` formatted files; whereby each line refers to a single FHIR resource
+
+![Path to bulk data](path-bulk-data.png)
+
+The dataset can be from an external source. Within the repository, there is a script to generate [synthetic data](https://github.com/synthetichealth/synthea/wiki/Getting-Started) that could then be used for benchmarking.
+
+```shell
+./gradlew :engine:benchmarks:app:generateSynthea -Ppopulation=1000
+```
+
+It generates [synthetic data](https://github.com/synthetichealth/synthea/wiki/Getting-Started) with a population size of 1000.
+The `population` parameter determines the population size that would be used to generate the data
+
+### Running
+
+To run this app in Android Studio, [create a run/debug configuration](https://developer.android.com/studio/run/rundebugconfig) for the `:engine:benchmarks:app` module using the [Android App](https://developer.android.com/studio/run/rundebugconfig#android-application) template and run the app using the configuration.
+
+[Change the build variant](https://developer.android.com/studio/run#changing-variant) to `benchmark` for an optimised version of the app, for the best results
+
+Alternatively, run the following command to build and install the benchmark APK on your device/emulator:
+
+```shell
+./gradlew :engine:benchmarks:app:installBenchmark
+```
+
+## Microbenchmark module
+
+Contains test cases that evaluate the performance of individual tasks executed for the first time directly on hardware, located in the module `:engine:benchmarks:microbenchmark`.
+
+The test cases are designed to run in sequence of their alphabetic order to make sure larger tasks do not build cache for smaller ones. Their class names are prefixed by an extra letter to inform their position relative to others in the list.
+
+### Running
+
+In Android Studio, set your build variants to `release` and run your benchmark as you would any `@Test` using the gutter action next to your test class or method.
+
+![gutter test action](https://developer.android.com/static/topic/performance/images/benchmark_images/microbenchmark_run.png)
+
+The results will be similar to this:
+```
+1,297,374       ns        5345 allocs    trace    EngineDatabaseBenchmark.createAndGet
+1,114,474,793   ns     4922289 allocs    trace    FhirSyncWorkerBenchmark.oneTimeSync_50patients
+15,251,125      ns      100542 allocs    trace    FhirSyncWorkerBenchmark.oneTimeSync_1patient
+179,806,709     ns      986017 allocs    trace    FhirSyncWorkerBenchmark.oneTimeSync_10patients
+1,451,758       ns       11883 allocs    trace    GzipUploadInterceptorBenchmark.upload_10patientsWithGzip
+1,537,559       ns       11829 allocs    trace    GzipUploadInterceptorBenchmark.upload_10patientsWithoutGzip
+73,640,833      ns     1074360 allocs    trace    GzipUploadInterceptorBenchmark.upload_1000patientsWithGzip
+7,493,642       ns      108428 allocs    trace    GzipUploadInterceptorBenchmark.upload_100patientsWithoutGzip
+7,799,264       ns      108465 allocs    trace    GzipUploadInterceptorBenchmark.upload_100patientsWithGzip
+71,189,333      ns     1074466 allocs    trace    GzipUploadInterceptorBenchmark.upload_1000patientsWithoutGzip
+
+```
+
+Alternatively, from the command line, run the connectedCheck to run all of the tests from specified Gradle module:
+
+```bash
+./gradlew :engine:benchmarks:microbenchmark:connectedReleaseAndroidTest
+```
+
+In this case, results will be saved to the `outputs/androidTest-results/connected/<device>/test-result.pb`. To visualize on Android Studio, click Run / Import Tests From File and find the `.pb` file
+
+### Continuous Integration (CI)
+
+#### Configuration
+
+Microbenchmark tests are configured to run in Kokoro and use [Fladle](https://runningcode.github.io/fladle/) plugin, configured through `Project.configureFirebaseTestLabForMicroBenchmark` in file `buildSrc/src/main/kotlin/FirebaseTestLabConfig.kt`
+
+#### Accessing the benchmark results
+
+The Microbenchmark results can be accessed through the following steps
+
+1. Click to `View details` of the `Kokoro: Build and Device Tests`
+   ![PR Kokoro view details](pr-kokoro-view-details.png)
+
+   The details page would look similar to
+   ![Kokoro details](kokoro-details-page.png)
+2. Within the `Target Log` tab, locate for the section
+      ![Microbenchmark section](microbenchmark-section.png)
+      with the `TEST FILE NAME` `microbenchmark-release-androidTest.apk`
+3. Select and visit the Google Bucket url that looks as similar to
+      [https://console.developers.google.com/storage/browser/android-fhir-build-artifacts/prod/openhealthstack/android-fhir/gcp_ubuntu/presubmit/5404/20250618-172425/firebase/microbenchmark](ttps://console.developers.google.com/storage/browser/android-fhir-build-artifacts/prod/openhealthstack/android-fhir/gcp_ubuntu/presubmit/5404/20250618-172425/firebase/microbenchmark)
+   that navigates to the `android-fhir-build-artifacts` ![bucket](microbenchmark-bucket.png)
+4. Navigate to `matrix_0/panther-33-en_US-portrait-test_results_merged.xml` to download the benchmark .xml results file. The `panther-33-en_US-portrait` in the path refers to the Firebase Test Lab device used in running the benchmark tests.
+
+## Macrobenchmark module
+
+The _FHIR Engine Library_ macrobenchmark tests are located in the module `:engine:benchmarks:macrobenchmark`
+
+### Prerequisite
+
+Requires the _FHIR Engine Library_ Benchmark App configured with the relevant benchmark data described in the section for the _FHIR Engine Library_ Benchmark App
+
+### Running
+
+To run, use the command
+
+```shell
+./gradlew :engine:benchmarks:macrobenchmark:connectedCheck
+```
+
+### Continuous Integration (CI)
+
+#### Configuration
+
+The `FHIR Engine` Macrobenchmarks have been configured to run in Kokoro and use FirebaseTestLab physical devices
+
+Configuration for the Kokoro script are currently located in `kokoro/gcp_ubuntu/kokoro_build.sh` while the FirebaseTestLab testing is configured through the [Fladle](https://runningcode.github.io/fladle/) plugin in `buildSrc/src/main/kotlin/FirebaseTestLabConfig.kt`
+
+#### Accessing the benchmark results
+
+From a GitHub PR , the following steps could be used to download the benchmark results from a Kokoro run
+
+1. Click to `View details` of the `Kokoro: Build and Device Tests`
+   ![PR Kokoro view details](pr-kokoro-view-details.png)
+
+   The details page would look similar to
+   ![Kokoro details](kokoro-details-page.png)
+
+2. Within the `Target Log` tab, locate for the section
+   ![Macrobenchmark section](macrobenchmark-section.png)
+   with the `TEST FILE NAME` `macrobenchmark-benchmark.apk`
+
+3. Select and visit the url as referenced in image
+   ![Reference url](select-bucket-url.png)
+   representative of the Google Cloud Bucket containing the artifacts from the Kokoro run. From the image example, the url is [https://console.developers.google.com/storage/browser/android-fhir-build-artifacts/prod/openhealthstack/android-fhir/gcp_ubuntu/presubmit/5403/20250616-053647/firebase/macrobenchmark](https://console.developers.google.com/storage/browser/android-fhir-build-artifacts/prod/openhealthstack/android-fhir/gcp_ubuntu/presubmit/5403/20250616-053647/firebase/macrobenchmark)
+
+   The bucket page would look similar to
+   ![Artifacts bucket page](google-bucket-page.png)
+
+4. Navigate to `matrix_0/panther-33-en_US-portrait/artifacts/sdcard/Download/com.google.android.fhir.engine.macrobenchmark-benchmarkData.json` to download the benchmark results file. The `panther-33-en_US-portrait` in the path represents the Firebase Test Lab device that was used to run the benchmark tests.
+
+#### Sample Benchmark Results
+
+The results shared below are generated from running the _FHIR Engine Library_ Macrobenchmark tests in Kokoro
+
+##### [**_Panther - Google Pixel 7_**](https://wiki.lineageos.org/devices/panther/)
+
+**CPU** - Octa-core (2x2.85 GHz Cortex-X1 & 2x2.35 GHz Cortex-A78 & 4x1.80 GHz Cortex-A55)
+
+**RAM** - 8GB
+
+API 33
+
+###### Data Access API results
+
+Results were generated from execution of FhirEngineCrudBenchmark test in the `engine:benchmarks:macrobenchmark` module located at `engine/benchmarks/macrobenchmark/src/main/java/com/google/android/fhir/engine/macrobenchmark/FhirEngineCrudBenchmark.kt`
+
+| API    | Average duration (ms) | Notes                                 |
+|:-------|----------------------:|---------------------------------------|
+| create |                  ~4.7 | Takes ~47s for population size of 10k |
+| update |                ~12.29 |                                       |
+| get    |                 ~3.83 |                                       |
+| delete |                 ~8.08 |                                       |