odrefresh: regenerate artifacts after ART APEX update

Replaces compile_bcp.sh to check and compile boot class path
extensions and system server jars.

Enable ART to load those artifacts when present in the ART APEX data
directory.

Bug: 160683548
Test: art_libartbase_tests
Test: adb root && adb odrefresh {--force-check,--force-compile}
Test: adb install com.android.art && adb reboot && adb root && \
      adb shell odrefresh {--check,--compile} && adb reboot && \
      adb shell cat /proc/<zygote>/maps | grep apexdata
Change-Id: I81bf520d38f9dc0109c91f192bc6e728099049fd

Android.mk

@@ -349,6 +349,7 @@ LOCAL_REQUIRED_MODULES := \
     $(call art_module_lib,libart-compiler) \
     $(call art_module_lib,libopenjdkjvm) \
     $(call art_module_lib,libopenjdkjvmti) \
+    $(call art_module_lib,odrefresh) \
     $(call art_module_lib,profman) \
     $(call art_module_lib,libadbconnection) \
     $(call art_module_lib,libperfetto_hprof) \

build/Android.common_path.mk

@@ -88,6 +88,7 @@ ART_CORE_DEBUGGABLE_EXECUTABLES := \
     dexoptanalyzer \
     imgdiag \
     oatdump \
+    odrefresh \
     profman \
 
 ART_CORE_EXECUTABLES := \

build/Android.gtest.mk

@@ -136,6 +136,7 @@ ART_TEST_MODULES := \
     art_libdexfile_tests \
     art_libprofile_tests \
     art_oatdump_tests \
+    art_odrefresh_tests \
     art_profman_tests \
     art_runtime_compiler_tests \
     art_runtime_tests \

build/apex/Android.bp

@@ -24,6 +24,7 @@ art_runtime_base_binaries_both_on_device_first_on_host = [
 //   only the "first" (likely 64-bit) version is required on host).
 art_runtime_base_binaries_prefer32_on_device_first_on_host = [
     "dexoptanalyzer",
+    "odrefresh",
     "profman",
 ]
 
@@ -131,7 +132,6 @@ art_tools_debug_binaries_both = [
 
 // Tools exclusively for the device APEX derived from art-tools in art/Android.mk.
 art_tools_device_only_binaries = [
-    "compile_bcp.sh",
     // oatdump cannot link with host linux_bionic due to not using clang lld;
     // TODO: Make it work with clang lld.
     "oatdump",
@@ -333,6 +333,7 @@ art_gtests = [
     "art_libdexfile_support_tests",
     "art_libprofile_tests",
     "art_oatdump_tests",
+    "art_odrefresh_tests",
     "art_profman_tests",
     "art_runtime_compiler_tests",
     "art_runtime_tests",

build/apex/art_apex_test.py

@@ -619,9 +619,9 @@ def run(self):
     # removed in Android R.
 
     # Check binaries for ART.
-    self._checker.check_executable("compile_bcp.sh")
-    self._checker.check_executable('oatdump')
     self._checker.check_multilib_executable('dex2oat')
+    self._checker.check_executable('oatdump')
+    self._checker.check_executable("odrefresh")
 
     # Check internal libraries for ART.
     self._checker.check_native_library('libperfetto_hprof')
@@ -646,6 +646,7 @@ def run(self):
     self._checker.check_executable('hprof-conv')
     self._checker.check_symlinked_first_executable('dex2oatd')
     self._checker.check_symlinked_first_executable('dex2oat')
+    self._checker.check_executable("odrefresh")
 
     # Check exported native libraries for Managed Core Library.
     self._checker.check_native_library('libicu')
@@ -753,6 +754,7 @@ def run(self):
     self._checker.check_art_test_executable('art_libdexfile_tests')
     self._checker.check_art_test_executable('art_libprofile_tests')
     self._checker.check_art_test_executable('art_oatdump_tests')
+    self._checker.check_art_test_executable('art_odrefresh_tests')
     self._checker.check_art_test_executable('art_profman_tests')
     self._checker.check_art_test_executable('art_runtime_compiler_tests')
     self._checker.check_art_test_executable('art_runtime_tests')

libartbase/base/common_art_test.h

@@ -159,6 +159,34 @@ class FakeDexStorage {
   std::vector<std::unique_ptr<FakeDex>> fake_dex_files;
 };
 
+// Helper class that removes an environment variable whilst in scope.
+class ScopedUnsetEnvironmentVariable {
+ public:
+  explicit ScopedUnsetEnvironmentVariable(const char* variable)
+      : variable_{variable}, old_value_{GetOldValue(variable)} {
+    unsetenv(variable);
+  }
+
+  ~ScopedUnsetEnvironmentVariable() {
+    if (old_value_.has_value()) {
+      static constexpr int kReplace = 1;  // tidy-issue: replace argument has libc dependent name.
+      setenv(variable_, old_value_.value().c_str(), kReplace);
+    } else {
+      unsetenv(variable_);
+    }
+  }
+
+ private:
+  static std::optional<std::string> GetOldValue(const char* variable) {
+    const char* value = getenv(variable);
+    return value != nullptr ? std::optional<std::string>{value} : std::nullopt;
+  }
+
+  const char* variable_;
+  std::optional<std::string> old_value_;
+  DISALLOW_COPY_AND_ASSIGN(ScopedUnsetEnvironmentVariable);
+};
+
 class CommonArtTestImpl {
  public:
   CommonArtTestImpl() = default;

libartbase/base/file_utils.h

@@ -20,6 +20,7 @@
 #include <stdlib.h>
 
 #include <string>
+#include <string_view>
 
 #include <android-base/logging.h>
 

libartbase/base/file_utils_test.cc

@@ -28,34 +28,6 @@ namespace art {
 
 class FileUtilsTest : public CommonArtTest {};
 
-// Helper class that removes an environment variable whilst in scope.
-class ScopedUnsetEnvironmentVariable {
- public:
-  explicit ScopedUnsetEnvironmentVariable(const char* variable)
-      : variable_{variable}, old_value_{GetOldValue(variable)} {
-    unsetenv(variable);
-  }
-
-  ~ScopedUnsetEnvironmentVariable() {
-    if (old_value_.has_value()) {
-      static constexpr int kReplace = 1;  // tidy-issue: replace argument has libc dependent name.
-      setenv(variable_, old_value_.value().c_str(), kReplace);
-    } else {
-      unsetenv(variable_);
-    }
-  }
-
- private:
-  static std::optional<std::string> GetOldValue(const char* variable) {
-    const char* value = getenv(variable);
-    return value != nullptr ? std::optional<std::string>{value} : std::nullopt;
-  }
-
-  const char* variable_;
-  std::optional<std::string> old_value_;
-  DISALLOW_COPY_AND_ASSIGN(ScopedUnsetEnvironmentVariable);
-};
-
 TEST_F(FileUtilsTest, GetDalvikCacheFilename) {
   std::string name;
   std::string error;

odrefresh/Android.bp

@@ -0,0 +1,119 @@
+//
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+cc_defaults {
+    name: "odrefresh-defaults",
+    host_supported: true,
+    defaults: ["art_defaults"],
+    srcs: [
+        "odrefresh.cc",
+    ],
+    local_include_dirs: ["include"],
+    header_libs: ["dexoptanalyzer_headers"],
+    generated_sources: ["apex-info-list"],
+    shared_libs: [
+        "libartpalette",
+        "libbase",
+        "liblog",
+    ],
+    static_libs: ["libxml2"],
+    target: {
+        android: {
+            // Use the 32-bit version of odrefresh on devices.
+            compile_multilib: "prefer32",
+        },
+        linux: {
+            enabled: true,
+        },
+        host: {
+            shared_libs: [
+                // Both these libraries for libxml2 on host for code derived from apex-info-list.
+                "libicui18n",
+                "libicuuc",
+            ],
+        },
+    },
+    tidy: true,
+    tidy_flags: [
+        "-format-style='file'",
+        "--header-filter='system/apex/'",
+    ],
+}
+
+cc_library_headers {
+    name: "odrefresh_headers",
+    export_include_dirs: ["include"],
+    host_supported: true,
+    stl: "none",
+    system_shared_libs: [],
+    min_sdk_version: "29", // As part of mainline modules(APEX), it should support at least 29(Q).
+    sdk_version: "minimum", // The minimum sdk version required by users of this module.
+    apex_available: [
+        "//apex_available:platform", // For odsign.
+    ],
+    visibility: ["//visibility:public"],
+}
+
+art_cc_binary {
+    name: "odrefresh",
+    defaults: ["odrefresh-defaults"],
+    required: [
+        "dexoptanalyzer",
+        "dex2oat",
+    ],
+    shared_libs: [
+        "libart",
+        "libartbase",
+    ],
+    apex_available: [
+        "com.android.art",
+        "com.android.art.debug",
+    ],
+}
+
+art_cc_binary {
+    name: "odrefreshd",
+    defaults: [
+        "art_debug_defaults",
+        "odrefresh-defaults",
+    ],
+    required: [
+        "dexoptanalyzerd",
+        "dex2oatd",
+    ],
+    shared_libs: [
+        "libartd",
+        "libartbased",
+    ],
+    apex_available: [
+        "com.android.art.debug",
+    ],
+}
+
+art_cc_test {
+    name: "art_odrefresh_tests",
+    defaults: [
+        "art_gtest_defaults",
+    ],
+    header_libs: ["odrefresh_headers"],
+    srcs: [
+        "odr_artifacts_test.cc",
+        "odrefresh_test.cc",
+    ],
+    shared_libs: [
+        "libbase",
+    ],
+}

odrefresh/README.md

@@ -0,0 +1,7 @@
+# On-Device Refresh (odrefresh)
+
+This tool is part of the ART APEX and is used to refresh compilation artifacts following an
+ART APEX update. It provides two key features:
+
+* checking the freshness of compilation artifacts for boot class path extensions and system_server.
+* regenerating the compilation artifacts for boot class path extensions and system_server.

odrefresh/TODO.md

@@ -0,0 +1,22 @@
+# TODO Work Items
+
+## TODO (STOPSHIP until done)
+
+1. Add a log file that tracks status of recent compilation.
+2. Implement back off on trying compilation when previous attempt(s) failed.
+3. Free space calculation and only attempting compilation if sufficient space.
+4. Metrics for tracking issues:
+   - Successful compilation of all artifacts.
+   - Time limit exceeded (indicates a pathological issue, e.g. dex2oat bug, device driver bug, etc).
+   - Insufficient space for compilation.
+   - Compilation failure (boot extensions)
+   - Compilation failure (system server)
+   - Unexpected error (a setup or clean-up action failed).
+5. Metrics recording for subprocess timeouts.
+6. Decide and implement testing.
+
+## DONE
+
+1. <strike>Fix dexoptanalyzer so it can analyze boot extensions.</strike>
+2. <strike>Parse apex-info-list.xml into an apex_info (to make version and location available).</strike>
+3. <strike>Timeouts for pathological failures.</strike>

odrefresh/include/odrefresh/odrefresh.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_ODREFRESH_INCLUDE_ODREFRESH_ODREFRESH_H_
+#define ART_ODREFRESH_INCLUDE_ODREFRESH_ODREFRESH_H_
+
+#include <sysexits.h>
+
+namespace art {
+namespace odrefresh {
+
+static constexpr const char* kOdrefreshArtifactDirectory =
+    "/data/misc/apexdata/com.android.art/dalvik-cache";
+
+//
+// Exit codes from the odrefresh process (in addition to standard exit codes in sysexits.h).
+//
+// NB if odrefresh crashes, then the caller should not sign any artifacts and should remove any
+// unsigned artifacts under `kOdrefreshArtifactDirectory`.
+//
+enum ExitCode {
+  // No compilation required, all artifacts look good or there is insufficient space to compile.
+  // For ART APEX in the system image, there may be no artifacts present under
+  // `kOdrefreshArtifactDirectory`.
+  kOkay = EX_OK,
+
+  // Compilation required. Re-run program with --compile on the command-line to generate
+  // new artifacts under `kOdrefreshArtifactDirectory`.
+  kCompilationRequired = 1,
+
+  // Compilation failed. Artifacts under `kOdrefreshArtifactDirectory` will be valid. This may
+  // happen, for example, if compilation of boot extensions succeeds, but the compilation of the
+  // system_server jars fails due to lack of storage space.
+  kCompilationFailed = 2,
+};
+
+static_assert(EX_OK == 0);
+static_assert(ExitCode::kOkay < EX__BASE);
+static_assert(ExitCode::kCompilationFailed < EX__BASE);
+
+}  // namespace odrefresh
+}  // namespace art
+
+#endif  // ART_ODREFRESH_INCLUDE_ODREFRESH_ODREFRESH_H_