Split out java/kotlin unit tests (#26897)

* Add JAVA_PATH to our dockerfile environment variables

* Use matter_enable_java_compilation instead of build_java_matter_controller

* Remove useless bit

* Fix unit tests once we have java available

* Fix seting up java deps if files already there ... this now runs in 5ms instead of 400ms

* Do not re-download java dependencies every time.

This moves java dependency download time from 500ms to 5ms after
the initial download.

If we get more dependencies, the savings should be even better.

* Restyled by shellharden

* Restyled by shfmt

* Prepare for unit tests

* Make output class directory unique for java

* Restyled by gn

* Undo unintentional change

* Allow separate unit test runs for java

* Restyled by gn

* Restyled by shellharden

* Restyled by shfmt

* Undo unintentional change in test name

* Fix yaml file

* Force java path

* Attempt to fix the run path ... again

* Update to not use run_in_build_env and switch to multi-line run script

* Do not attempt to compile pure java variant for android (keep only linux and mac generally)

* Fix required jar: tlv now needs copy as the tlv content is not part of controller anymore

* Ensure unit tests pretend java path is set

---------

Co-authored-by: Andrei Litvin <andreilitvin@google.com>
Co-authored-by: Restyled.io <commits@restyled.io>

.github/workflows/java-tests.yaml

@@ -38,13 +38,12 @@ jobs:
 
         env:
             TSAN_OPTIONS: "halt_on_error=1 suppressions=scripts/tests/chiptest/tsan-linux-suppressions.txt"
-            JAVA_PATH: /usr/lib/jvm/java-8-openjdk-amd64
 
         if: github.actor != 'restyled-io[bot]'
         runs-on: ubuntu-latest
 
         container:
-            image: connectedhomeip/chip-build-java:0.7.3
+            image: connectedhomeip/chip-build-java:0.7.15
             options: --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0
                 net.ipv4.conf.all.forwarding=0 net.ipv6.conf.all.forwarding=0"
 
@@ -79,7 +78,30 @@ jobs:
                   path: |
                       .environment/gn_out/.ninja_log
                       .environment/pigweed-venv/*.log
+            - name: Generate unit tests
+              timeout-minutes: 1
+              run: |
+                  scripts/run_in_build_env.sh \
+                  './scripts/build/build_examples.py \
+                     --target linux-x64-tests \
+                     gen \
+                  '
+            - name: Build unit tests
+              timeout-minutes: 25
+              run: scripts/run_in_build_env.sh 'ninja -C out/linux-x64-tests src:java_controller_tests'
 
+            - name: Run unit tests
+              timeout-minutes: 10
+              # TODO: this direct path loading is not maintainable. Our build system should define and
+              #       support test classes.
+              run: |
+                 $JAVA_PATH/bin/java \
+                   -cp 'third_party/java_deps/artifacts/*:out/linux-x64-tests/lib/src/controller/java/*' \
+                   org.junit.runner.JUnitCore       \
+                   chip.tlv.TlvWriterTest           \
+                   chip.tlv.TlvReadWriteTest        \
+                   chip.tlv.TlvReaderTest           \
+                   chip.jsontlv.JsonToTlvToJsonTest
             - name: Build Java Matter Controller and all clusters app
               timeout-minutes: 50
               run: |

build/chip/java/config.gni

@@ -15,7 +15,7 @@
 java_path = getenv("JAVA_PATH")
 declare_args() {
   java_matter_controller_dependent_paths = []
-  build_java_matter_controller = false
+  matter_enable_java_compilation = false
   if (java_path != "" && current_os != "android") {
     java_matter_controller_dependent_paths += [ "${java_path}/include/" ]
 
@@ -27,6 +27,6 @@ declare_args() {
           [ "${java_path}/include/linux/" ]
     }
 
-    build_java_matter_controller = true
+    matter_enable_java_compilation = true
   }
 }

build/chip/java/rules.gni

@@ -21,8 +21,8 @@ kotlinc_runner = "${chip_root}/build/chip/java/kotlinc_runner.py"
 jar_runner = "${chip_root}/build/chip/java/jar_runner.py"
 write_build_config = "${chip_root}/build/chip/java/write_build_config.py"
 
-assert(android_sdk_root != "" || build_java_matter_controller,
-       "android_sdk_root or java_path must be specified")
+assert(android_sdk_root != "" || matter_enable_java_compilation,
+       "android_sdk_root must be specified or JAVA_PATH must be set.")
 
 # Declare a java library target
 #

build/config/compiler/BUILD.gn

@@ -253,7 +253,7 @@ config("strict_warnings") {
     cflags += [ "-Wconversion" ]
   }
 
-  if (build_java_matter_controller) {
+  if (matter_enable_java_compilation) {
     cflags -= [ "-Wshadow" ]
   }
 

examples/java-matter-controller/BUILD.gn

@@ -37,11 +37,7 @@ kotlin_binary("java-matter-controller") {
   output_name = "java-matter-controller"
   deps = [
     ":java",
-    "${chip_root}/src/controller/java:json_to_tlv_to_json_test",
     "${chip_root}/src/controller/java:onboarding_payload",
-    "${chip_root}/src/controller/java:tlv_read_write_test",
-    "${chip_root}/src/controller/java:tlv_reader_test",
-    "${chip_root}/src/controller/java:tlv_writer_test",
     "${chip_root}/third_party/java_deps:kotlin-stdlib",
   ]
 

scripts/build/test.py

@@ -47,6 +47,7 @@ def build_actual_output(root: str, out: str, args: List[str]) -> List[str]:
         'NXP_K32W0_SDK_ROOT': 'TEST_NXP_K32W0_SDK_ROOT',
         'IMX_SDK_ROOT': 'IMX_SDK_ROOT',
         'TI_SYSCONFIG_ROOT': 'TEST_TI_SYSCONFIG_ROOT',
+        'JAVA_PATH': 'TEST_JAVA_PATH',
     })
 
     retval = subprocess.run([

scripts/build/testdata/dry_run_linux-arm64-chip-tool-ipv6only-clang.txt

@@ -6,6 +6,9 @@ bash -c '
 PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \
  gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/chip-tool '"'"'--args=chip_inet_config_enable_ipv4=false is_clang=true target_cpu="arm64" sysroot="SYSROOT_AARCH64"'"'"' {out}/linux-arm64-chip-tool-ipv6only-clang'
 
+# Setting up Java deps
+third_party/java_deps/set_up_java_deps.sh
+
 # Building linux-arm64-chip-tool-ipv6only-clang
 bash -c '
 PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \

scripts/build/testdata/dry_run_linux-arm64-ota-requestor-nodeps-ipv6only.txt

@@ -6,6 +6,9 @@ bash -c '
 PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \
  gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/ota-requestor-app/linux '"'"'--args=chip_inet_config_enable_ipv4=false chip_config_network_layer_ble=false chip_enable_wifi=false chip_enable_openthread=false is_clang=true chip_crypto="mbedtls" target_cpu="arm64" sysroot="SYSROOT_AARCH64"'"'"' {out}/linux-arm64-ota-requestor-nodeps-ipv6only'
 
+# Setting up Java deps
+third_party/java_deps/set_up_java_deps.sh
+
 # Building linux-arm64-ota-requestor-nodeps-ipv6only
 bash -c '
 PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \

scripts/build/testdata/dry_run_linux-x64-all-clusters-coverage.txt

@@ -4,5 +4,8 @@ cd "{root}"
 # Generating linux-x64-all-clusters-coverage
 gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/all-clusters-app/linux --args=use_coverage=true {out}/linux-x64-all-clusters-coverage
 
+# Setting up Java deps
+third_party/java_deps/set_up_java_deps.sh
+
 # Building linux-x64-all-clusters-coverage
 ninja -C {out}/linux-x64-all-clusters-coverage

src/BUILD.gn

@@ -16,6 +16,7 @@ import("//build_overrides/build.gni")
 import("//build_overrides/chip.gni")
 
 import("${build_root}/config/compiler/compiler.gni")
+import("${chip_root}/build/chip/java/config.gni")
 import("${chip_root}/build/chip/tests.gni")
 import("${chip_root}/src/ble/ble.gni")
 import("${chip_root}/src/lwip/lwip.gni")
@@ -147,4 +148,10 @@ if (chip_build_tests) {
       "${chip_root}/src/crypto/tests",
     ]
   }
+
+  if (matter_enable_java_compilation) {
+    group("java_controller_tests") {
+      deps = [ "${chip_root}/src/controller/java:unit_tests" ]
+    }
+  }
 }

src/controller/data_model/BUILD.gn

@@ -30,9 +30,9 @@ chip_data_model("data_model") {
   allow_circular_includes_from = [ "${chip_root}/src/controller" ]
 }
 
-if (current_os == "android" || build_java_matter_controller) {
+if (current_os == "android" || matter_enable_java_compilation) {
   config("java-build-config") {
-    if (build_java_matter_controller) {
+    if (matter_enable_java_compilation) {
       include_dirs = java_matter_controller_dependent_paths
     }
   }
@@ -324,7 +324,7 @@ if (current_os == "android" || build_java_matter_controller) {
       "${chip_root}/src/platform",
     ]
 
-    if (build_java_matter_controller) {
+    if (matter_enable_java_compilation) {
       if (current_os == "mac") {
         deps += [ "${chip_root}/src/platform/Darwin" ]
       } else {

src/controller/java/BUILD.gn

@@ -18,7 +18,7 @@ import("${chip_root}/build/chip/java/config.gni")
 import("${chip_root}/build/chip/java/rules.gni")
 import("${chip_root}/build/chip/tests.gni")
 
-if (!build_java_matter_controller) {
+if (!matter_enable_java_compilation) {
   import("${build_root}/config/android_abi.gni")
 }
 
@@ -76,7 +76,7 @@ shared_library("jni") {
     "${chip_root}/src/platform",
   ]
 
-  if (build_java_matter_controller) {
+  if (matter_enable_java_compilation) {
     defines = [ "JAVA_MATTER_CONTROLLER_TEST" ]
 
     sources += [
@@ -123,7 +123,7 @@ if (chip_link_tests) {
       "${chip_root}/src/messaging/tests:helpers",
     ]
 
-    if (build_java_matter_controller) {
+    if (matter_enable_java_compilation) {
       defines = [ "JAVA_MATTER_CONTROLLER_TEST" ]
       include_dirs = java_matter_controller_dependent_paths
       if (current_os == "mac") {
@@ -240,7 +240,7 @@ kotlin_library("json_to_tlv_to_json_test") {
 
 shared_library("jni_for_onboarding_payload") {
   output_name = "libOnboardingPayload"
-  if (build_java_matter_controller) {
+  if (matter_enable_java_compilation) {
     include_dirs = java_matter_controller_dependent_paths
     output_dir = "${root_out_dir}/lib/jni"
   } else {
@@ -260,7 +260,7 @@ kotlin_library("onboarding_payload") {
 
   data_deps = [ ":jni_for_onboarding_payload" ]
 
-  if (!build_java_matter_controller) {
+  if (!matter_enable_java_compilation) {
     data_deps += [ "${chip_root}/build/chip/java:shared_cpplib" ]
   }
 
@@ -272,6 +272,15 @@ kotlin_library("onboarding_payload") {
   ]
 }
 
+group("unit_tests") {
+  deps = [
+    ":json_to_tlv_to_json_test",
+    ":tlv_read_write_test",
+    ":tlv_reader_test",
+    ":tlv_writer_test",
+  ]
+}
+
 android_library("java") {
   output_name = "CHIPController.jar"
 
@@ -337,7 +346,7 @@ android_library("java") {
     "zap-generated/chip/devicecontroller/ClusterInfoMapping.java",
   ]
 
-  if (build_java_matter_controller) {
+  if (matter_enable_java_compilation) {
     deps += [
       "${chip_root}/third_party/java_deps:json",
       "${chip_root}/third_party/java_deps/stub_src",
@@ -372,7 +381,7 @@ if (chip_link_tests) {
       "src/chip/devicecontroller/GetConnectedDeviceCallbackForTestJni.java",
     ]
 
-    if (build_java_matter_controller) {
+    if (matter_enable_java_compilation) {
       deps += [
         "${chip_root}/third_party/java_deps:json",
         "${chip_root}/third_party/java_deps/stub_src",
@@ -407,7 +416,7 @@ if (chip_link_tests) {
     sources =
         [ "tests/chip/devicecontroller/GetConnectedDeviceCallbackJniTest.java" ]
 
-    if (build_java_matter_controller) {
+    if (matter_enable_java_compilation) {
       deps += [
         "${chip_root}/third_party/java_deps:json",
         "${chip_root}/third_party/java_deps/stub_src",
@@ -425,7 +434,7 @@ if (chip_link_tests) {
   }
 }
 
-if (!build_java_matter_controller) {
+if (!matter_enable_java_compilation) {
   java_prebuilt("android") {
     jar_path = "${android_sdk_root}/platforms/android-26/android.jar"
   }

src/lib/support/BUILD.gn

@@ -149,8 +149,8 @@ static_library("support") {
     "verhoeff/Verhoeff36.cpp",
   ]
 
-  if (current_os == "android" || build_java_matter_controller) {
-    if (build_java_matter_controller) {
+  if (current_os == "android" || matter_enable_java_compilation) {
+    if (matter_enable_java_compilation) {
       include_dirs = java_matter_controller_dependent_paths
     }
 

src/messaging/tests/java/BUILD.gn

@@ -17,7 +17,7 @@ import("//build_overrides/chip.gni")
 import("${chip_root}/build/chip/java/config.gni")
 import("${chip_root}/build/chip/java/rules.gni")
 
-if (!build_java_matter_controller) {
+if (!matter_enable_java_compilation) {
   import("${build_root}/config/android_abi.gni")
 }
 
@@ -29,7 +29,7 @@ shared_library("jni") {
 
   public_configs = [ "${chip_root}/src:includes" ]
 
-  if (build_java_matter_controller) {
+  if (matter_enable_java_compilation) {
     defines = [ "JAVA_MATTER_CONTROLLER_TEST" ]
     include_dirs = java_matter_controller_dependent_paths
 
@@ -56,7 +56,7 @@ android_library("java") {
 
   sources = [ "src/chip/testing/MessagingContext.java" ]
 
-  if (build_java_matter_controller) {
+  if (matter_enable_java_compilation) {
     deps += [
       "${chip_root}/third_party/java_deps:json",
       "${chip_root}/third_party/java_deps/stub_src",
@@ -73,7 +73,7 @@ android_library("java") {
   #  ..../platforms/android-26/android.jar to access BLE items)
 }
 
-if (!build_java_matter_controller) {
+if (!matter_enable_java_compilation) {
   java_prebuilt("android") {
     jar_path = "${android_sdk_root}/platforms/android-26/android.jar"
   }

src/setup_payload/java/BUILD.gn

@@ -17,13 +17,13 @@ import("//build_overrides/chip.gni")
 import("${chip_root}/build/chip/java/config.gni")
 import("${chip_root}/build/chip/java/rules.gni")
 
-if (!build_java_matter_controller) {
+if (!matter_enable_java_compilation) {
   import("${build_root}/config/android_abi.gni")
 }
 
 shared_library("jni") {
   output_name = "libSetupPayloadParser"
-  if (build_java_matter_controller) {
+  if (matter_enable_java_compilation) {
     include_dirs = java_matter_controller_dependent_paths
     output_dir = "${root_out_dir}/lib/jni"
   } else {
@@ -43,7 +43,7 @@ android_library("java") {
 
   data_deps = [ ":jni" ]
 
-  if (!build_java_matter_controller) {
+  if (!matter_enable_java_compilation) {
     data_deps += [ "${chip_root}/build/chip/java:shared_cpplib" ]
   }
 

third_party/java_deps/set_up_java_deps.sh

@@ -20,22 +20,27 @@ set -e
 
 mkdir -p third_party/java_deps/artifacts
 
-function download_maven_jar() {
-    _MAVEN_PATH=$1
-    _JAR_NAME=$2
+function download_jar() {
+    _HOST=$1
+    _MAVEN_PATH=$2
+    _JAR_NAME=$3
 
     if [ ! -f "third_party/java_deps/artifacts/$_JAR_NAME" ]; then
         curl --fail --location --silent --show-error \
-            "https://repo1.maven.org/maven2/$_MAVEN_PATH/$_JAR_NAME" \
+            "https://$_HOST/maven2/$_MAVEN_PATH/$_JAR_NAME" \
             -o "third_party/java_deps/artifacts/$_JAR_NAME"
     fi
 }
 
-download_maven_jar "com/google/code/findbugs/jsr305/3.0.2" "jsr305-3.0.2.jar"
-download_maven_jar "org/json/json/20220924" "json-20220924.jar"
-download_maven_jar "org/jetbrains/kotlin/kotlin-stdlib/1.8.10" "kotlin-stdlib-1.8.10.jar"
-download_maven_jar "org/jetbrains/kotlin/kotlin-test/1.8.10" "kotlin-test-1.8.10.jar"
-download_maven_jar "com/google/protobuf/protobuf-java/3.22.0" "protobuf-java-3.22.0.jar"
-download_maven_jar "com/google/truth/truth/1.1.3" "truth-1.1.3.jar"
-download_maven_jar "junit/junit/4.13.2" "junit-4.13.2.jar"
-download_maven_jar "com/google/code/gson/gson/2.9.1" "gson-2.9.1.jar"
+download_jar "repo1.maven.org" "com/google/code/findbugs/jsr305/3.0.2" "jsr305-3.0.2.jar"
+download_jar "repo1.maven.org" "com/google/code/gson/gson/2.9.1" "gson-2.9.1.jar"
+download_jar "repo1.maven.org" "com/google/protobuf/protobuf-java/3.22.0" "protobuf-java-3.22.0.jar"
+download_jar "repo1.maven.org" "com/google/truth/truth/1.1.3" "truth-1.1.3.jar"
+download_jar "repo1.maven.org" "junit/junit/4.13.2" "junit-4.13.2.jar"
+download_jar "repo1.maven.org" "org/jetbrains/kotlin/kotlin-stdlib/1.8.10" "kotlin-stdlib-1.8.10.jar"
+download_jar "repo1.maven.org" "org/jetbrains/kotlin/kotlin-test/1.8.10" "kotlin-test-1.8.10.jar"
+download_jar "repo1.maven.org" "org/json/json/20220924" "json-20220924.jar"
+
+# Unit test requirements
+download_jar "repo1.maven.org" "org/hamcrest/hamcrest-all/1.3" "hamcrest-all-1.3.jar"
+download_jar "dist.wso2.org" "com/google/common/google-collect/1.0-rc1/" "google-collect-1.0-rc1.jar"