dev: perf regression test configuration

.gitignore

@@ -93,4 +93,4 @@ storybook-static
 .jest-cache
 
 # E2E test reports
-tests/e2e/.results/
+tests/e2e/results/

package.json

@@ -38,7 +38,7 @@
     "analyze-packages": "ANALYZE_BUNDLE=true webpack --config config/webpack/webpack.common.js --env envFile=.env.production",
     "symbolicate:android": "npx metro-symbolicate android/app/build/generated/sourcemaps/react/release/index.android.bundle.map",
     "symbolicate:ios": "npx metro-symbolicate main.jsbundle.map",
-    "test:e2e": "node tests/e2e/testRunner.js"
+    "test:e2e": "node tests/e2e/testRunner.js --development"
   },
   "dependencies": {
     "@expensify/react-native-web": "0.18.9",

scripts/android-repackage-app-bundle-and-sign.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+###
+# Takes an android app that has been built with the debug keystore,
+# and re-packages it with an alternative JS bundle to run.
+# It then signs the APK again, so you can simply install the app on a device.
+# This is useful if you quickly want to test changes to the JS code with a
+# release app, without having to rebuild the whole app.
+#
+# There are many outdated resources on how to re-sign an app. The main
+# flow and commands have been taken from:
+# - https://gist.github.com/floyd-fuh/7f7408b560672ece3ea78348559d47b6#file-repackage_apk_for_burp-py-L276-L319
+#
+# This script uses `apktool` instead of manually unzipping and zipping the app.
+# Only with apktool it worked without any errors, so you need to install it.
+###
+
+BUILD_TOOLS=$ANDROID_SDK_ROOT/build-tools/31.0.0
+APK=$1
+NEW_BUNDLE_FILE=$2
+OUTPUT_APK=$3
+
+### Helper function to use echo but print text in bold
+echo_bold() {
+    echo -e "\033[1m$*\033[0m"
+}
+
+### Validating inputs
+
+if [ -z "$APK" ] || [ -z "$NEW_BUNDLE_FILE" ] || [ -z "$OUTPUT_APK" ]; then
+    echo "Usage: $0 <apk> <new-bundle-file> <output-apk>"
+    exit 1
+fi
+APK=$(realpath "$APK")
+if [ ! -f "$APK" ]; then
+    echo "APK not found: $APK"
+    exit 1
+fi
+NEW_BUNDLE_FILE=$(realpath "$NEW_BUNDLE_FILE")
+if [ ! -f "$NEW_BUNDLE_FILE" ]; then
+    echo "Bundle file not found: $NEW_BUNDLE_FILE"
+    exit 1
+fi
+OUTPUT_APK=$(realpath "$OUTPUT_APK")
+# check if "apktool" command is available
+if ! command -v apktool &> /dev/null
+then
+    echo "apktool could not be found. Please install it."
+    exit 1
+fi
+# check if "jarsigner" command is available
+if ! command -v jarsigner &> /dev/null
+then
+    echo "jarsigner could not be found. Please install it."
+    exit 1
+fi
+
+KEYSTORE="$(realpath ./android/app/debug.keystore)"
+ORIGINAL_WD=$(pwd)
+
+### Copy apk to a temp dir
+
+TMP_DIR=$(mktemp -d)
+cp "$APK" "$TMP_DIR"
+cd "$TMP_DIR" || exit
+
+### Dissemble app
+
+echo_bold "Dissembling app..."
+apktool d "$APK" -o app > /dev/null
+
+### Copy new bundle into assets
+
+echo_bold "Copying new bundle into assets..."
+rm app/assets/index.android.bundle
+cp "$NEW_BUNDLE_FILE" app/assets/index.android.bundle
+
+### Reassemble app
+
+echo_bold "Reassembling app..."
+apktool b app -o app.apk > /dev/null
+
+### Do jarsigner
+
+echo_bold "Signing app..."
+jarsigner -verbose -keystore "$KEYSTORE" -storepass android -keypass android app.apk androiddebugkey
+
+### Do zipalign
+
+echo_bold "Zipaligning app..."
+"$BUILD_TOOLS"/zipalign -p -v 4 app.apk app-aligned.apk
+
+### Do apksigner
+
+echo_bold "Signing app with apksigner..."
+"$BUILD_TOOLS"/apksigner sign  --v4-signing-enabled true --ks "$KEYSTORE" --ks-pass pass:android --ks-key-alias androiddebugkey --key-pass pass:android app-aligned.apk
+
+### Copy back to original location
+
+echo_bold "Copying back to original location..."
+cp app-aligned.apk "$OUTPUT_APK"
+echo "Done. Repacked app is at $OUTPUT_APK"
+rm -rf "$TMP_DIR"
+cd "$ORIGINAL_WD" || exit
+

tests/e2e/ADDING_TESTS.md

@@ -94,3 +94,9 @@ test file:
 
 Done! When you now start the test runner, your new test will be executed as well.
 
+## Quickly test your test
+
+To check your new test you can simply run `npm run test:e2e`, which uses the
+`--development` flag. This will run the tests on the branch you are currently on
+and will do fewer iterations.
+

tests/e2e/README.md

@@ -14,9 +14,36 @@ To run the e2e tests:
 2. Make sure Fastlane was initialized by running `bundle install`
 
 3. Run the tests with `npm run test:e2e`.
+  > 💡 Tip: To run the tests locally faster, and you are only making changes to JS, it's recommended to
+    build the app once with `npm run android-build-e2e` and from then on run the tests with
+    `npm run test:e2e -- --buildMode js-only`. This will only rebuild the JS code, and not the
+    whole native app!
 
 Ideally you want to run these tests on your branch before you want to merge your new feature to `main`.
 
+## Available CLI options
+
+The tests can be run with the following CLI options:
+
+- `--config`: Extend/Overwrite the default config with your values, e.g. `--config config.local.js`
+- `--includes`: Expects a string/regexp to filter the tests to run, e.g. `--includes "login|signup"`
+- `--skipInstallDeps`: Skips the `npm install` step, useful during development
+- `--development`: Applies some default configurations:
+  - Sets the config to `config.local.js`, which executes the tests with fewer iterations 
+  - Runs the tests only on the current branch
+- `--buildMode`: There are three build modes, the default is `full`:
+  1. **full**: rebuilds the full native app in (e2e) release mode
+  2. **js-only**: only rebuilds the js bundle, and then re-packages
+                   the existing native app with the new package. If there
+                   is no existing native app, it will fallback to mode "full"
+  3. **skip**: does not rebuild anything, and just runs the existing native app
+
+## Available environment variables
+
+The tests can be run with the following environment variables:
+
+- `baseline`: Change the baseline to run the tests again (default is `main`).
+
 ## Performance regression testing
 
 The output of the tests is a set of performance metrics (see video above).

tests/e2e/config.js

@@ -1,4 +1,4 @@
-const OUTPUT_DIR = process.env.WORKING_DIRECTORY || './results';
+const OUTPUT_DIR = process.env.WORKING_DIRECTORY || './tests/e2e/results';
 
 /**
  * @typedef TestConfig
@@ -10,9 +10,25 @@ const TEST_NAMES = {
     AppStartTime: 'App start time',
 };
 
+/**
+ * Default config, used by CI by default.
+ * You can modify these values for your test run by creating a
+ * separate config file and pass it to the test runner like this:
+ *
+ * ```bash
+ * npm run test:e2e -- --config ./path/to/your/config.js
+ * ```
+ */
 module.exports = {
     APP_PACKAGE: 'com.expensify.chat',
 
+    APP_PATHS: {
+        baseline: './app-e2eRelease-baseline.apk',
+        compare: './app-e2eRelease-compare.apk',
+    },
+
+    ENTRY_FILE: 'src/libs/E2E/reactNativeLaunchingTest.js',
+
     // The port of the testing server that communicates with the app
     SERVER_PORT: 4723,
 
@@ -21,9 +37,6 @@ module.exports = {
 
     DEFAULT_BASELINE_BRANCH: 'main',
 
-    // The amount of outliers to remove from a dataset before calculating the average
-    DROP_WORST: 8,
-
     // The amount of runs that should happen without counting test results
     WARM_UP_RUNS: 3,
 

tests/e2e/config.local.js

@@ -0,0 +1,8 @@
+module.exports = {
+    WARM_UP_RUNS: 1,
+    RUNS: 8,
+    APP_PATHS: {
+        baseline: './android/app/build/outputs/apk/e2eRelease/app-e2eRelease.apk',
+        compare: './android/app/build/outputs/apk/e2eRelease/app-e2eRelease.apk',
+    },
+};