meson: Use provided directory for intermediate files

To avoid polluting build directories with files unknown to Meson, which
means it can't invalidate them when needed.

compat/build.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 import argparse
+import base64
 from collections import OrderedDict
 from dataclasses import dataclass
 import itertools
@@ -26,9 +27,10 @@ def main(argv):
     parser = argparse.ArgumentParser()
     subparsers = parser.add_subparsers()
 
-    command = subparsers.add_parser("setup", help="setup build directories")
+    command = subparsers.add_parser("setup", help="setup everything needed to compile")
     command.add_argument("role", help="project vs subproject", choices=["project", "subproject"])
     command.add_argument("builddir", help="build directory", type=Path)
+    command.add_argument("top_builddir", help="top build directory", type=Path)
     command.add_argument("host_os", help="operating system binaries are being built for")
     command.add_argument("host_arch", help="architecture binaries are being built for")
     command.add_argument("host_toolchain", help="the kind of toolchain being used",
@@ -40,17 +42,18 @@ def main(argv):
                          type=parse_array_option_value)
     command.set_defaults(func=lambda args: setup(args.role,
                                                  args.builddir,
+                                                 args.top_builddir,
                                                  args.host_os,
                                                  args.host_arch,
                                                  args.host_toolchain,
                                                  args.compat,
                                                  args.assets,
                                                  args.components))
 
-    command = subparsers.add_parser("compile", help="compile a specific build directory")
-    command.add_argument("builddir", help="build directory", type=Path)
-    command.add_argument("top_builddir", help="top build directory", type=Path)
-    command.set_defaults(func=lambda args: compile(args.builddir, args.top_builddir))
+    command = subparsers.add_parser("compile", help="compile compatibility assets")
+    command.add_argument("privdir", help="directory to store intermediate files", type=Path)
+    command.add_argument("state", help="opaque state from the setup step")
+    command.set_defaults(func=lambda args: compile(args.privdir, pickle.loads(base64.b64decode(args.state))))
 
     args = parser.parse_args()
     if "func" in args:
@@ -82,13 +85,16 @@ def parse_array_option_value(v: str) -> set[str]:
 
 def setup(role: Role,
           builddir: Path,
+          top_builddir: Path,
           host_os: str,
           host_arch: str,
           host_toolchain: str,
           compat: set[str],
           assets: str,
           components: set[str]):
-    outputs: Mapping[str, Sequence[Output]] = {}
+    outputs: Mapping[str, Sequence[Output]] = OrderedDict()
+
+    outputs["bundle"] = [Output("arch_support_bundle", "arch-support.bundle", Path("compat"), "")]
 
     releng_parentdir = query_releng_parentdir(role)
     ensure_submodules_checked_out(releng_parentdir)
@@ -218,22 +224,19 @@ def setup(role: Role,
                            target=AGENT_TARGET),
                 ]
 
-    if not outputs:
-        return
-
-    state = State(role, host_os, host_arch, host_toolchain, outputs)
-    privdir = compute_private_dir(builddir)
-    privdir.mkdir(exist_ok=True)
-    (privdir / STATE_FILENAME).write_bytes(pickle.dumps(state))
+    state = State(role, builddir, top_builddir, host_os, host_arch, host_toolchain, outputs)
+    serialized_state = base64.b64encode(pickle.dumps(state)).decode('ascii')
 
     variable_names, output_names = zip(*[(output.identifier, output.name) \
             for output in itertools.chain.from_iterable(outputs.values())])
-    print(f"{','.join(variable_names)} {','.join(output_names)} {DEPFILE_FILENAME}")
+    print(f"{','.join(variable_names)} {','.join(output_names)} {DEPFILE_FILENAME} {serialized_state}")
 
 
 @dataclass
 class State:
     role: Role
+    builddir: Path
+    top_builddir: Path
     host_os: str
     host_arch: str
     host_toolchain: str
@@ -248,9 +251,7 @@ class Output:
     target: str
 
 
-def compile(builddir: Path, top_builddir: Path):
-    state = pickle.loads((compute_private_dir(builddir) / STATE_FILENAME).read_bytes())
-
+def compile(privdir: Path, state: State):
     releng_parentdir = query_releng_parentdir(state.role)
     subprojects = detect_relevant_subprojects(releng_parentdir)
     if state.role == "subproject":
@@ -277,12 +278,18 @@ def call_internal_meson(argv, *args, **kwargs):
     source_paths: set[Path] = set()
     options: Optional[Sequence[str]] = None
     build_env = scrub_environment(os.environ)
-    for extra_arch, outputs in state.outputs.items():
-        workdir = compute_workdir_for_arch(extra_arch, builddir)
+    for key, outputs in state.outputs.items():
+        if key == "bundle":
+            for o in outputs:
+                (state.builddir / o.name).write_bytes(b"")
+            continue
+
+        extra_arch = key
+        workdir = (privdir / extra_arch).resolve()
 
         if not (workdir / "build.ninja").exists():
             if options is None:
-                options = load_meson_options(top_builddir, state.role, set(subprojects.keys()))
+                options = load_meson_options(state.top_builddir, state.role, set(subprojects.keys()))
 
             if state.host_os == "windows":
                 if state.host_toolchain == "microsoft":
@@ -316,24 +323,16 @@ def call_internal_meson(argv, *args, **kwargs):
              call_meson=call_internal_meson)
 
         for o in outputs:
-            shutil.copy(workdir / o.file, builddir / o.name)
+            shutil.copy(workdir / o.file, state.builddir / o.name)
 
         for cmd in json.loads((workdir / "compile_commands.json").read_text(encoding="utf-8")):
             source_paths.add((workdir / Path(cmd["file"])).absolute())
 
-    (builddir / DEPFILE_FILENAME).write_text(generate_depfile(itertools.chain.from_iterable(state.outputs.values()),
-                                                              source_paths,
-                                                              builddir,
-                                                              top_builddir),
-                                             encoding="utf-8")
-
-
-def compute_private_dir(builddir: Path) -> Path:
-    return builddir / "arch-support.p"
-
-
-def compute_workdir_for_arch(arch: str, builddir: Path) -> Path:
-    return compute_private_dir(builddir) / arch
+    (state.builddir / DEPFILE_FILENAME).write_text(generate_depfile(itertools.chain.from_iterable(state.outputs.values()),
+                                                                    source_paths,
+                                                                    state.builddir,
+                                                                    state.top_builddir),
+                                                   encoding="utf-8")
 
 
 def load_meson_options(top_builddir: Path,

compat/meson.build

@@ -36,6 +36,7 @@ if no_overridden_compat_bits
     'setup',
     meson.is_subproject() ? 'subproject' : 'project',
     meson.current_build_dir(),
+    meson.global_build_root(),
     host_os,
     host_abi,
     host_toolchain,
@@ -45,64 +46,63 @@ if no_overridden_compat_bits
     check: true
   )
   setup_output = setup_result.stdout().strip()
-  if setup_output != ''
-    tokens = setup_output.split(' ')
-    varnames = tokens[0].split(',')
-    outputs = tokens[1].split(',')
-    depfile = tokens[2]
+  tokens = setup_output.split(' ')
+  varnames = tokens[0].split(',')
+  outputs = tokens[1].split(',')
+  depfile = tokens[2]
+  state = tokens[3]
 
-    if host_os_family == 'darwin'
-      install = false
-      install_dir = asset_dir
-    else
-      install = true
-      install_dir = []
-      foreach varname : varnames
-        if get_option('assets') == 'installed' or varname.startswith('gadget_')
-          install_dir += varname.endswith('_modern') ? asset_dir_modern : asset_dir_legacy
-        else
-          install_dir += false
-        endif
-      endforeach
-    endif
+  if host_os_family == 'darwin'
+    install = false
+    install_dir = asset_dir
+  else
+    install = true
+    install_dir = []
+    foreach varname : varnames
+      if varname != 'arch_support_bundle' and (get_option('assets') == 'installed' or varname.startswith('gadget_'))
+        install_dir += varname.endswith('_modern') ? asset_dir_modern : asset_dir_legacy
+      else
+        install_dir += false
+      endif
+    endforeach
+  endif
 
-    arch_support = custom_target('arch-support',
-      output: outputs,
-      command: [
-        build_py,
-        'compile',
-        meson.current_build_dir(),
-        meson.global_build_root(),
-      ],
-      depfile: depfile,
-      install: install,
-      install_dir: install_dir,
-    )
+  arch_support = custom_target('arch-support',
+    output: outputs,
+    command: [
+      build_py,
+      'compile',
+      '@PRIVATE_DIR@',
+      state,
+    ],
+    depfile: depfile,
+    install: install,
+    install_dir: install_dir,
+  )
 
-    i = 0
-    foreach output : arch_support.to_list()
-      outpath = output.full_path()
-      varname = varnames[i]
+  i = 0
+  foreach output : arch_support.to_list()
+    outpath = output.full_path()
+    varname = varnames[i]
 
-      set_variable(varname, outpath)
+    set_variable(varname, outpath)
 
-      if varname.startswith('helper_')
-        helper_compat = outpath
-        helper_depends += arch_support
-      elif varname.startswith('agent_')
-        agent_compat = outpath
-        agent_depends += arch_support
-      elif varname.startswith('gadget_')
-        gadget_compat = outpath
-        gadget_depends += arch_support
-      elif varname.startswith('server_')
-        server_compat = outpath
-        server_depends += arch_support
-      else
-        assert(false, f'unexpected variable name: @varname@')
-      endif
+    if varname.startswith('helper_')
+      helper_compat = outpath
+      helper_depends += arch_support
+    elif varname.startswith('agent_')
+      agent_compat = outpath
+      agent_depends += arch_support
+    elif varname.startswith('gadget_')
+      gadget_compat = outpath
+      gadget_depends += arch_support
+    elif varname.startswith('server_')
+      server_compat = outpath
+      server_depends += arch_support
+    elif varname != 'arch_support_bundle'
+      assert(false, f'unexpected variable name: @varname@')
+    endif
 
-      i += 1
-    endforeach
-  endif
+    i += 1
+  endforeach
 endif

src/barebone/generate-script-runtime.py

@@ -8,22 +8,32 @@
 import sys
 
 
-def generate_runtime(input_dir, output_dir):
-    output_dir.mkdir(parents=True, exist_ok=True)
+def main(argv):
+    input_dir, output_dir, priv_dir = [Path(d).resolve() for d in sys.argv[1:]]
 
-    shutil.copy(input_dir / "package.json", output_dir)
-    shutil.copy(input_dir / "package-lock.json", output_dir)
+    try:
+        generate_runtime(input_dir, output_dir, priv_dir)
+    except Exception as e:
+        print(e, file=sys.stderr)
+        sys.exit(1)
+
+
+def generate_runtime(input_dir, output_dir, priv_dir):
+    priv_dir.mkdir(exist_ok=True)
+
+    shutil.copy(input_dir / "package.json", priv_dir)
+    shutil.copy(input_dir / "package-lock.json", priv_dir)
 
     runtime_reldir = Path("script-runtime")
     runtime_srcdir = input_dir / runtime_reldir
-    runtime_intdir = output_dir / runtime_reldir
+    runtime_intdir = priv_dir / runtime_reldir
     if runtime_intdir.exists():
         shutil.rmtree(runtime_intdir)
     shutil.copytree(runtime_srcdir, runtime_intdir)
 
     npm = os.environ.get("NPM", make_script_filename("npm"))
     try:
-        subprocess.run([npm, "install"], capture_output=True, cwd=output_dir, check=True)
+        subprocess.run([npm, "install"], capture_output=True, cwd=priv_dir, check=True)
     except Exception as e:
         message = "\n".join([
             "",
@@ -37,6 +47,8 @@ def generate_runtime(input_dir, output_dir):
         ])
         raise EnvironmentError(message)
 
+    shutil.copy(priv_dir / "script-runtime.js", output_dir)
+
 
 def make_script_filename(name):
     build_os = platform.system().lower()
@@ -45,10 +57,4 @@ def make_script_filename(name):
 
 
 if __name__ == "__main__":
-    input_dir, output_dir = [Path(d).resolve() for d in sys.argv[1:3]]
-
-    try:
-        generate_runtime(input_dir, output_dir)
-    except Exception as e:
-        print(e, file=sys.stderr)
-        sys.exit(1)
+    main(sys.argv)

src/barebone/meson.build

@@ -31,6 +31,7 @@ barebone_script_runtime = custom_target('frida-barebone-script-runtime',
     find_program('generate-script-runtime.py'),
     meson.current_source_dir(),
     meson.current_build_dir(),
+    '@PRIVATE_DIR@',
   ],
 )
 backend_sources += custom_target('frida-data-barebone',