Merge pull request #219 from nickel-lang/faster-file-regen

Make the file generation evaluation faster
nickel-lang · Jun 26, 2024 · 7f83a26 · 7f83a26
2 parents 5ec9c0b + 2fd0bd3
commit 7f83a26
Show file tree

Hide file tree

Showing 10 changed files with 226 additions and 211 deletions.
diff --git a/lib/files.ncl b/lib/files.ncl
@@ -11,12 +11,12 @@ let File = {
     | doc m%"
         The content of the file.
       "%
-    | nix.derivation.NullOr nix.derivation.NixString
+    | nix.derivation.NullOr nix.nix_string.NixString
     | default
     = null,
   file
     | doc "File from which to read the body of the script"
-    | nix.derivation.NullOr nix.derivation.NixString
+    | nix.derivation.NullOr nix.nix_string.NixString
     | default
     =
       if content == null then
@@ -44,44 +44,49 @@ let NormaliseTargets = fun label files =>
 in
 let regenerate_files | Files -> nix.derivation.Derivation
   = fun files_to_generate =>
-    let regnerate_one | String -> File -> nix.derivation.NixString
+    let regenerate_function | String
+      = m%"
+        regenerate_function () {
+          COPY_COMMAND="$1"
+          SOURCE="$2"
+          TARGET="$3"
+          if [[ ! -f "$TARGET" ]] || [[ $(cat "$TARGET") != $(cat "$SOURCE") ]]; then
+            rm -f "$TARGET"
+            echo "Regenerating $TARGET"
+            target_dir=$(dirname "$TARGET")
+            test "${target_dir}" != "." && mkdir -p "${target_dir}"
+            # XXX: If `source.file` is set explicitely to a relative path
+            # and `materialisation_method` is `'Symlink`, this will link to the
+            # original file, not one in the store. Not sure that's what we want.
+            $COPY_COMMAND "$SOURCE" "$TARGET"
+          fi
+        }
+      "%
+      in
+    let regnerate_one | String -> File -> nix.nix_string.NixString
       = fun key file_descr =>
-        let target = file_descr.target in
         let copy_command =
           match {
             'Symlink => "ln -s",
             'Copy => "cp",
           }
             file_descr.materialisation_method
         in
-        nix-s%"
-            if [[ ! -f "%{target}" ]] || [[ $(cat "%{target}") != $(cat "%{file_descr.file}") ]]; then
-              rm -f %{target}
-              echo "Regenerating %{target}"
-              target_dir=$(dirname "%{target}")
-              test "${target_dir}" != "." && mkdir -p "${target_dir}"
-              # XXX: If `source.file` is set explicitely to a relative path
-              # and `materialisation_method` is `'Symlink`, this will link to the
-              # original file, not one in the store. Not sure that's what we want.
-              %{copy_command} "%{file_descr.file}" "%{target}"
-            fi
-          "%
+        nix-s%"regenerate_function "%{copy_command}" "%{file_descr.file}" "%{file_descr.target}""%
       in
+    let regenerate_files = nix-s%"
+      %{regenerate_function}
+      %{
+        files_to_generate
+        |> std.record.map regnerate_one
+        |> std.record.values
+        |> nix.nix_string.join "\n"
+      }
+    "%
+    in
     {
       name = "regenerate-files",
-      content.text =
-        files_to_generate
-        |> std.record.to_array
-        |> std.array.map (fun { field, value } => regnerate_one field value)
-        |> std.array.fold_left
-          (
-            fun acc elt =>
-              nix-s%"
-                  %{acc}
-                  %{elt}
-                "%
-          )
-          "",
+      content.text = regenerate_files,
     }
       | nix.builders.ShellApplication
   in

diff --git a/lib/nix-interop/builders.ncl b/lib/nix-interop/builders.ncl
@@ -1,14 +1,8 @@
-let { NickelDerivation, Derivation, NixString, NixEnvironmentVariable, NullOr, .. } = import "derivation.ncl" in
+let { NickelDerivation, Derivation, NixEnvironmentVariable, NullOr, .. } = import "derivation.ncl" in
+let nix_string = import "nix-string.ncl" in
 
 let nix_builtins = import "builtins.ncl" in
 
-let concat_strings_sep = fun sep values =>
-  if std.array.length values == 0 then
-    ""
-  else
-    std.array.reduce_left (fun acc value => nix-s%"%{acc}%{sep}%{value}"%) values
-in
-
 let MutExclusiveWith = fun other name other_name label value =>
   if value == null && other == null then
     std.fail_with "You must specify either %{name} or %{other_name} field"
@@ -94,7 +88,7 @@ in
           echo "This derivation is not supposed to be built" 1>&2 1>/dev/null
           exit 1
         "%,
-        env.shellHook = concat_strings_sep "\n" (std.record.values hooks),
+        env.shellHook = nix_string.join "\n" (std.record.values hooks),
         structured_env.nativeBuildInputs = packages,
       }
         | NickelPkg,
@@ -121,12 +115,12 @@ in
           "%,
         content.text
           | doc "A string representing the body of the script"
-          | NullOr NixString
+          | NullOr nix_string.NixString
           | default
           = null,
         content.file
           | doc "File from which to read the body of the script"
-          | NullOr NixString
+          | NullOr nix_string.NixString
           | default
           =
             if content.text == null then
@@ -154,7 +148,7 @@ in
             The binary that will be run to execute the script.
             Needs to be bash-compatible.
           "%
-          | NixString
+          | nix_string.NixString
           | default
           = nix_builtins.import_nix "nixpkgs#runtimeShell",
 
@@ -176,7 +170,7 @@ in
                 runtime_inputs
                 |> std.record.values
                 |> std.array.map (fun s => nix-s%"%{s}/bin"%)
-                |> concat_strings_sep ":"
+                |> nix_string.join ":"
               in
               nix-s%"export PATH="%{paths}:$PATH""%
           in

diff --git a/lib/nix-interop/builtins.ncl b/lib/nix-interop/builtins.ncl
@@ -1,4 +1,5 @@
 let derivations = import "derivation.ncl" in
+let nix_string = import "nix-string.ncl" in
 {
   import_file
     | String -> derivations.NixPath
@@ -51,6 +52,6 @@ let derivations = import "derivation.ncl" in
       [toFile](https://nixos.org/manual/nix/stable/language/builtins.html#builtins-toFile)
       builtin.
     "%
-    | String -> derivations.NixString -> derivations.NixToFile
+    | String -> nix_string.NixString -> derivations.NixToFile
     = fun _name _text => { name = _name, text = _text },
 }
diff --git a/lib/nix-interop/derivation.ncl b/lib/nix-interop/derivation.ncl
@@ -1,54 +1,8 @@
 let type_field = "$__organist_type" in
 
-let predicate | doc "Various predicates used to define contracts"
-  = {
-    is_nix_path = fun x =>
-      std.is_record x
-      && std.record.has_field type_field x
-      && x."%{type_field}" == "nixPath",
-    is_nix_placeholder = fun x =>
-      std.is_record x
-      && std.record.has_field type_field x
-      && x."%{type_field}" == "nixPlaceholder",
-    is_nix_to_file = fun x =>
-      std.is_record x
-      && std.record.has_field type_field x
-      && x."%{type_field}" == "nixToFile",
-    is_nix_input = fun x =>
-      std.is_record x
-      && std.record.has_field type_field x
-      && x."%{type_field}" == "nixInput",
-    is_nix_string = fun value =>
-      std.is_record value
-      && std.record.has_field type_field value
-      && value."%{type_field}" == "nixString",
-    is_nickel_derivation = fun x =>
-      std.is_record x
-      && std.record.has_field type_field x
-      && x."%{type_field}" == "nickelDerivation",
-    is_derivation = fun x =>
-      is_nickel_derivation x
-      || is_nix_input x,
-    is_nix_call = fun value =>
-      std.is_record value
-      && std.record.has_field type_field value
-      && value."%{type_field}" == "callNix",
-    is_string_fragment = fun x =>
-      is_derivation x
-      || std.is_string x
-      || is_nix_path x
-      || is_nix_placeholder x
-      || is_nix_to_file x
-      || is_nix_call x
-  }
-  in
+let nix_string = import "./nix-string.ncl" in
 
-let mk_nix_string = fun fs =>
-  {
-    "%{type_field}" = "nixString",
-    fragments = fs,
-  }
-in
+let NixString = nix_string.NixString in
 
 {
   # Nix may require name, version, etc. to have a certain format, but we're not sure.
@@ -72,109 +26,6 @@ in
       "%
     = Dyn,
 
-  NixStringFragment | doc "A fragment of a Nix string (or a string with context). See `NixString`"
-    = std.contract.from_predicate predicate.is_string_fragment,
-
-  NixSymbolicString
-    | doc m%"
-      A symbolic string with the `'nix` prefix, as output by the Nickel
-      parser. Used as a subcontract for `NixString`.
-      "%
-    = {
-      prefix | [| 'nix |],
-      tag | [| 'SymbolicString |],
-      fragments | Array NixString,
-    },
-
-  NixString
-    | doc m%%"
-      Nix string with a
-      [context](https://shealevy.com/blog/2018/08/05/understanding-nixs-string-context/)
-      tracking the dependencies that need to be built before the string can make
-      sense.
-
-      Anything expecting a `NixString` accepts a pure Nickel string as well. A
-      `NixString` also accepts a Nix string fragment, which can be a Nickel
-      derivation, a Nickel derivation, a Nix path (built from `lib.import_file`), pure
-      Nickel strings, and maybe more in the future.
-
-      A `NixString` accepts any sequence of Nix string fragment as well.
-
-      A `NixString` is best constructed using the symbolic string syntax. See
-      the Nickel example below.
-
-      # Nix string context
-
-      In Nix, when one writes:
-
-      ```nix
-      shellHook = ''
-        echo "Development shell"
-        ${pkgs.hello}/bin/hello
-      ''
-      ```
-
-      Nix automatically deduces that this shell depends on the `hello`
-      package. Nickel doesn't have string contexts, and given the way values
-      are passed from and to Nix, this dependency information is just lost when
-      using bare strings.
-
-      Sometimes, you may not need the context: if `hello` is explicitly part
-      of the inputs, you can use a plain string in a Nickel
-      expression as well:
-
-      ```nickel
-      shellHook = m%"
-        echo "Development shell"
-        %{pkgs.hello.outputPath}/bin/hello
-      "%
-      ```
-
-      # Example
-
-      However, if you need the dependency to `hello` to be automatically
-      deduced, you can use symbolic strings whenever a field has a `NixString`
-      contract attached. The result will be elaborated as a richer structure,
-      carrying the context, and will be reconstructed on the Nix side.
-
-      To do so, juste use the multiline string syntax, but with an `s` prefix
-      instead (**Warning**: the `s` prefix is as of now temporary, and subject
-      to change in the future):
-
-      ```nickel
-      shellHook = nix-s%"
-        echo "Development shell"
-        %{pkgs.hello}/bin/hello
-      "%
-      ```
-
-      Note that:
-        - we've used the symbolic string syntax `nix-s%"`
-        - instead of `hello.outputPath`, we've interpolated `hello` directly,
-          which is a derivation, and not a string
-
-      Within a `NixString`, you can interpolate a Nix String, or a Nix string
-      fragment, that is a Nix derivation, a Nickel derivation, a Nix path (built from
-      `lib.import_file`), pure Nickel strings, and maybe more in the future.
-    "%%
-    = fun label value =>
-      # A contract must always be idempotent (be a no-op if applied a second
-      # time), so we accept something that is already a NixString
-      if predicate.is_nix_string value then
-        value
-        # We accept a single string fragment (a plain string, a derivation or a
-        # Nix path). We normalize it by wrapping it as a one-element array
-      else if predicate.is_string_fragment value then
-        mk_nix_string [value]
-      else
-        let { fragments, .. } = std.contract.apply NixSymbolicString label value in
-        mk_nix_string
-          (
-            std.array.flat_map
-              (fun elt => elt.fragments)
-              fragments
-          ),
-
   NixDerivation
     | doc m%"
         The basic, low-level interface for a symbolic derivation. A