diff --git a/examples/c-hello-world/nickel.lock.ncl b/examples/c-hello-world/nickel.lock.ncl index d85c324e..059540c7 100644 --- a/examples/c-hello-world/nickel.lock.ncl +++ b/examples/c-hello-world/nickel.lock.ncl @@ -1,3 +1,3 @@ { - organist = import "../../lib/nix.ncl", + organist = import "../../lib/organist.ncl", } diff --git a/examples/c-hello-world/project.ncl b/examples/c-hello-world/project.ncl index d96e803a..4e2fc31b 100644 --- a/examples/c-hello-world/project.ncl +++ b/examples/c-hello-world/project.ncl @@ -4,21 +4,21 @@ let organist = inputs.organist in { packages."default" = packages.hello, packages.hello = - organist.builders.NixpkgsPkg + organist.nix.builders.NixpkgsPkg & { name = "hello", version = "0.1", nix_drv = { - buildInputs.gcc = organist.lib.import_nix "nixpkgs#gcc", - buildInputs.coreutils = organist.lib.import_nix "nixpkgs#coreutils", - buildInputs.bash = organist.lib.import_nix "nixpkgs#bash", + buildInputs.gcc = organist.import_nix "nixpkgs#gcc", + buildInputs.coreutils = organist.import_nix "nixpkgs#coreutils", + buildInputs.bash = organist.import_nix "nixpkgs#bash", buildCommand = nix-s%" - gcc %{organist.lib.import_file "hello.c"} -o hello + gcc %{organist.nix.builtins.import_file "hello.c"} -o hello mkdir -p $out/bin cp hello $out/bin/hello "% - | organist.contracts.NixString, + | organist.nix.derivation.NixString, }, } } diff --git a/flake.nix b/flake.nix index f4c5c711..94bb45df 100644 --- a/flake.nix +++ b/flake.nix @@ -29,7 +29,7 @@ outputsFromNickel = baseDir: flakeInputs: { systems ? flake-utils.lib.defaultSystems, lockFileContents ? { - organist = "${self}/lib/nix.ncl"; + organist = "${self}/lib/organist.ncl"; }, }: flake-utils.lib.eachSystem systems (system: let @@ -56,7 +56,7 @@ }); computedOutputs = outputsFromNickel ./. inputs { - lockFileContents.organist = "./lib/nix.ncl"; + lockFileContents.organist = "./lib/organist.ncl"; }; in { diff --git a/lib/lib.nix b/lib/lib.nix index 77f21c1f..4dc558fa 100644 --- a/lib/lib.nix +++ b/lib/lib.nix @@ -46,7 +46,7 @@ # organist = { # builders = "/nix/store/...-source/builders.ncl"; # contracts = "/nix/store/...-source/contracts.ncl"; - # nix = "/nix/store/...-source/nix.ncl"; + # nix = "/nix/store/...-source/organist.ncl"; # }; # } # Result: @@ -54,7 +54,7 @@ # organist = { # builders = import "/nix/store/...-source/builders.ncl", # contracts = import "/nix/store/...-source/contracts.ncl", - # nix = import "/nix/store/...-source/nix.ncl", + # nix = import "/nix/store/...-source/organist.ncl", # }, # } buildLockFile = contents: @@ -147,9 +147,9 @@ system = "${system}", } in - let nix = (import "${src}/nickel.lock.ncl").organist in + let organist = (import "${src}/nickel.lock.ncl").organist in - let nickel_expr | nix.contracts.OrganistExpression = + let nickel_expr | organist.OrganistExpression = import "${src}/${nickelFile}" in nickel_expr & params @@ -197,7 +197,7 @@ nixpkgs = pkgs; }, lockFileContents ? { - organist = "${organistSrc}/lib/nix.ncl"; + organist = "${organistSrc}/lib/organist.ncl"; }, }: let nickelResult = callNickel { diff --git a/lib/builders.ncl b/lib/nix-interop/builders.ncl similarity index 98% rename from lib/builders.ncl rename to lib/nix-interop/builders.ncl index e4d79d2f..49d78901 100644 --- a/lib/builders.ncl +++ b/lib/nix-interop/builders.ncl @@ -1,6 +1,6 @@ -let { NickelDerivation, Derivation, NixString, NixEnvironmentVariable, .. } = import "contracts.ncl" in +let { NickelDerivation, Derivation, NixString, NixEnvironmentVariable, .. } = import "derivation.ncl" in -let lib = import "lib.ncl" in +let lib = import "builtins.ncl" in let concat_strings_sep = fun sep values => if std.array.length values == 0 then diff --git a/lib/lib.ncl b/lib/nix-interop/builtins.ncl similarity index 100% rename from lib/lib.ncl rename to lib/nix-interop/builtins.ncl diff --git a/lib/contracts.ncl b/lib/nix-interop/contracts.ncl similarity index 100% rename from lib/contracts.ncl rename to lib/nix-interop/contracts.ncl diff --git a/lib/nix-interop/derivation.ncl b/lib/nix-interop/derivation.ncl new file mode 100644 index 00000000..5b50102a --- /dev/null +++ b/lib/nix-interop/derivation.ncl @@ -0,0 +1,347 @@ +let type_field = "$__organist_type" in + +let predicate | doc "Various predicates used to define contracts" + = { + is_nix_path = fun x => + std.is_record x + && std.record.has_field type_field x + && x."%{type_field}" == "nixPath", + is_nix_placeholder = fun x => + std.is_record x + && std.record.has_field type_field x + && x."%{type_field}" == "nixPlaceholder", + is_nix_to_file = fun x => + std.is_record x + && std.record.has_field type_field x + && x."%{type_field}" == "nixToFile", + is_nix_input = fun x => + std.is_record x + && std.record.has_field type_field x + && x."%{type_field}" == "nixInput", + is_nix_string = fun value => + std.is_record value + && std.record.has_field type_field value + && value."%{type_field}" == "nixString", + is_nickel_derivation = fun x => + std.is_record x + && std.record.has_field type_field x + && x."%{type_field}" == "nickelDerivation", + is_derivation = fun x => + is_nickel_derivation x + || is_nix_input x, + is_string_fragment = fun x => + is_derivation x + || std.is_string x + || is_nix_path x + || is_nix_placeholder x + || is_nix_to_file x + } + in + +let mk_nix_string = fun fs => + { + "%{type_field}" = "nixString", + fragments = fs, + } +in + +{ + # Nix may require name, version, etc. to have a certain format, but we're not sure. + # TODO: refine those contracts + Name = String, + Version = String, + # TODO: For now, we use String, but should we have enums tags for arch and os + # instead? + System = { arch | String, os | String }, + + NullOr + | doc "Make a contract nullable" + = fun contract label value => + if value == null then value else std.contract.apply contract label value, + + # TODO: more precise contract + Derivation + | doc m%" + Contract representing either a Nix derivation (evaluated and imported +from the Nix world) or a derivation defined in Nickel. + "% + = Dyn, + + NixStringFragment | doc "A fragment of a Nix string (or a string with context). See `NixString`" + = std.contract.from_predicate predicate.is_string_fragment, + + NixSymbolicString + | doc m%" + A symbolic string with the `'nix` prefix, as output by the Nickel + parser. Used as a subcontract for `NixString`. + "% + = { + prefix | [| 'nix |], + tag | [| 'SymbolicString |], + fragments | Array NixString, + }, + + NixString + | doc m%%" + Nix string with a + [context](https://shealevy.com/blog/2018/08/05/understanding-nixs-string-context/) + tracking the dependencies that need to be built before the string can make + sense. + + Anything expecting a `NixString` accepts a pure Nickel string as well. A + `NixString` also accepts a Nix string fragment, which can be a Nickel + derivation, a Nickel derivation, a Nix path (built from `lib.import_file`), pure + Nickel strings, and maybe more in the future. + + A `NixString` accepts any sequence of Nix string fragment as well. + + A `NixString` is best constructed using the symbolic string syntax. See + the Nickel example below. + + # Nix string context + + In Nix, when one writes: + + ```nix + shellHook = '' + echo "Development shell" + ${pkgs.hello}/bin/hello + '' + ``` + + Nix automatically deduces that this shell depends on the `hello` + package. Nickel doesn't have string contexts, and given the way values + are passed from and to Nix, this dependency information is just lost when + using bare strings. + + Sometimes, you may not need the context: if `hello` is explicitly part + of the inputs, you can use a plain string in a Nickel + expression as well: + + ```nickel + shellHook = m%" + echo "Development shell" + %{pkgs.hello.outputPath}/bin/hello + "% + ``` + + # Example + + However, if you need the dependency to `hello` to be automatically + deduced, you can use symbolic strings whenever a field has a `NixString` + contract attached. The result will be elaborated as a richer structure, + carrying the context, and will be reconstructed on the Nix side. + + To do so, juste use the multiline string syntax, but with an `s` prefix + instead (**Warning**: the `s` prefix is as of now temporary, and subject + to change in the future): + + ```nickel + shellHook = nix-s%" + echo "Development shell" + %{pkgs.hello}/bin/hello + "% + ``` + + Note that: + - we've used the symbolic string syntax `nix-s%"` + - instead of `hello.outputPath`, we've interpolated `hello` directly, + which is a derivation, and not a string + + Within a `NixString`, you can interpolate a Nix String, or a Nix string + fragment, that is a Nix derivation, a Nickel derivation, a Nix path (built from + `lib.import_file`), pure Nickel strings, and maybe more in the future. + "%% + = fun label value => + # A contract must always be idempotent (be a no-op if applied a second + # time), so we accept something that is already a NixString + if predicate.is_nix_string value then + value + # We accept a single string fragment (a plain string, a derivation or a + # Nix path). We normalize it by wrapping it as a one-element array + else if predicate.is_string_fragment value then + mk_nix_string [std.contract.apply NixStringFragment label value] + else + # TODO: it's for debugging, but we should remove the serializing at some + # point. + let label = std.contract.label.append_note (std.serialize 'Json value) label in + let { fragments, .. } = std.contract.apply NixSymbolicString label value in + mk_nix_string fragments, + + NixDerivation + | doc m%" + The basic, low-level interface for a symbolic derivation. A + NixDerivations is intended to be passed (exported) to the Nix side, + which will take care of actually building it. + + The fields directly map to the corresponding + [builtins.derivation](https://nixos.org/manual/nix/stable/language/derivations.html) + attribute on the Nix side + "% + = { + name | Name, + builder | NixString, + args | Array NixString, + system | NullOr String, + outputs | Array String | optional, + "__structuredAttrs" | Bool | default = true, + .. + }, + + NickelDerivation + | doc m%" + The representation of a symbolic derivation on the Nickel side. + This record is extensible as the different layers involved might add new attributes to this derivation. + The important part eventually is the `nix_drv` field which is computed from the rest and sent to Nix +"% + = { + "%{type_field}" | force = "nickelDerivation", + nix_drv + | doc "The raw derivation sent to Nix" + | NixDerivation + = + let _name = name in + let _system = system in + let _version = version in + { + name = _name, + system = _system, + version = _version, + builder = build_command.cmd, + args = build_command.args + }, + name + | doc "The name of the package." + | Name, + version + | doc "The version of the package." + | optional + | Version, + system + | doc "The system to build the package on. Defaults to the system used by importNcl." + | NullOr System + | default + = null, + build_command + | doc "The build command to execute." + | { + cmd | NixString, + args | Array NixString + }, + .. + }, + + Params | doc "The parameters provided to the Nickel expression" + = { + system | System, + }, + + InputPath + | doc " The path of a package in an input (usually nixpkgs)" + = Array String, + + NixInput + | doc m%" + The specification of a Nix input in a Nickel expression. + "% + = { + "%{type_field}" | force = "nixInput", + input + | doc "The flake input from which we'll resolve this input" + | String + | default + = "nixpkgs", + attr_path + | doc m%" + The path to look for in the given flake input. + + This follows the same search rules as the `nix build` cli, namely + that the library will consider the first valid values within: + - InputPath + - "packages".system.InputPath + - "legacyPackages".system.InputPath + "% + | InputPath + | optional, + }, + + NixInputSugar + | doc m%" + Syntactic sugar for defining a `NixInput` to allow writing inputs + directly as strings of the form `{inputName}#{path}` + "% + = fun label value => + if std.is_string value then + let hashPosition = (std.string.find "#" value).index in + let value' = + if hashPosition == -1 then + { input = value, attr_path = [] } + else + { + input = std.string.substring 0 hashPosition value, + attr_path = + std.string.split + "." + ( + std.string.substring + (hashPosition + 1) + (std.string.length value) + value + ), + } + in + value' |> std.contract.apply NixInput label + else + std.contract.apply NixInput label value, + + NixPath + | doc "A path to be imported in the Nix store" + = { + "%{type_field}" | force = "nixPath", + path | String, + }, + + NixPlaceholder + | doc "A path to the given output resolved later in the Nix store" + = { + "%{type_field}" | force = "nixPlaceholder", + output | String, + }, + + NixEnvironmentVariable + | doc m%" + Covers all types that are allowed in Nix derivation's environment variables: + - strings + - arrays of strings + - records with string values + "% + = fun label value => + let Contract = + if std.is_string value then + NixString + else if std.is_record value then + if std.record.has_field type_field value + || ( + std.record.has_field "tag" value + && value.tag == 'SymbolicString + && std.record.has_field "prefix" value + && value.prefix == 'nix + ) then + NixString + else + { _ | NixString } + else if std.is_array value then + Array NixString + else + std.contract.blame_with_message "Must be string, array of strings or record with string values" label + in + std.contract.apply Contract label value, + + NixToFile + | doc "A path to the given output resolved later in the Nix store" + = { + "%{type_field}" | force = "nixToFile", + name | String, + text | NixString, + }, +} diff --git a/lib/nix-interop/nix.ncl b/lib/nix-interop/nix.ncl new file mode 100644 index 00000000..18bfa233 --- /dev/null +++ b/lib/nix-interop/nix.ncl @@ -0,0 +1,8 @@ +{ + derivation = import "derivation.ncl", + builders = import "builders.ncl", + shells = import "shells.ncl", + builtins = import "builtins.ncl", + + import_nix = builtins.import_nix, +} diff --git a/lib/shells.ncl b/lib/nix-interop/shells.ncl similarity index 99% rename from lib/shells.ncl rename to lib/nix-interop/shells.ncl index 7fe0688f..408781fd 100644 --- a/lib/shells.ncl +++ b/lib/nix-interop/shells.ncl @@ -1,6 +1,6 @@ let builders = import "builders.ncl" in let contracts = import "contracts.ncl" in -let lib = import "lib.ncl" in +let lib = import "builtins.ncl" in let concat_strings_sep = fun sep values => if std.array.length values == 0 then diff --git a/lib/nix.ncl b/lib/organist.ncl similarity index 71% rename from lib/nix.ncl rename to lib/organist.ncl index a1c3a91a..c0333d89 100644 --- a/lib/nix.ncl +++ b/lib/organist.ncl @@ -1,8 +1,9 @@ { - lib = import "lib.ncl", - builders = import "builders.ncl", - contracts = import "contracts.ncl", - shells = import "shells.ncl", + nix = import "./nix-interop/nix.ncl", + shells = nix.shells, + schema = import "./schema.ncl", + OrganistExpression = schema.OrganistExpression, + import_nix = nix.builtins.import_nix, } #TODO: currently, Nickel forbids doc at the toplevel. It's most definitely # temporary, as the implementation of RFC005 is ongoing. Once the capability is diff --git a/lib/schema.ncl b/lib/schema.ncl new file mode 100644 index 00000000..d227178d --- /dev/null +++ b/lib/schema.ncl @@ -0,0 +1,30 @@ +let nix = import "./nix-interop/nix.ncl" in +{ + OrganistShells = { + dev | nix.derivation.NickelDerivation = build, + build | nix.derivation.NickelDerivation, + "default" | nix.derivation.NickelDerivation = dev, + }, + + FlakeOutputs = { + packages | { _ | nix.derivation.Derivation } | optional, + checks | { _ | nix.derivation.Derivation } | optional, + devShells | { _ | nix.derivation.Derivation } | optional, + apps | { _ | { type = "app", program | nix.derivation.NixString } } | optional, + }, + + # TODO: have the actual contract for the result of an expression. It's pretty + # open (could be an integer, a derivation, a record of derivations, etc.) but + # it still obeys some rules: if the `type` field is set to a known predefined + # value, then the record must have a certain shape. + # + # The contract must be: what the Nix side of the code can "parse" without + # erroring out. + OrganistExpression = { + shells + | OrganistShells + | optional, + flake | FlakeOutputs | optional, + .. + }, +} diff --git a/lib/shell-tests.ncl b/lib/shell-tests.ncl index 5ddb94cc..360891d9 100644 --- a/lib/shell-tests.ncl +++ b/lib/shell-tests.ncl @@ -11,7 +11,7 @@ std.record.map |> std.record.merge_all, } ) - (import "./shells.ncl") + (import "./organist.ncl").shells & { # Override all cases where `--version` does not work or is not enough Go.tests = { diff --git a/nickel.lock.ncl b/nickel.lock.ncl index 89c9b7ef..cd5abd46 100644 --- a/nickel.lock.ncl +++ b/nickel.lock.ncl @@ -1,3 +1,3 @@ { - organist = import "./lib/nix.ncl", + organist = import "./lib/organist.ncl", } diff --git a/project.ncl b/project.ncl index d2a0bd9d..a03c1c94 100644 --- a/project.ncl +++ b/project.ncl @@ -1,6 +1,6 @@ let inputs = import "./nickel.lock.ncl" in let organist = inputs.organist in -let import_nix = organist.lib.import_nix in +let import_nix = organist.nix.import_nix in { shells = @@ -17,11 +17,11 @@ let import_nix = organist.lib.import_nix in }, flake.apps.run-test = - let testScript | organist.builders.ShellApplication = { + let testScript | organist.nix.builders.ShellApplication = { name = "run-test.sh", - content.file = organist.lib.import_file "run-test.sh", + content.file = organist.nix.builtins.import_file "run-test.sh", runtime_inputs = { - nickel = organist.lib.import_nix "nickel#nickel-lang-cli", + nickel = import_nix "nickel#nickel-lang-cli", parallel = import_nix "nixpkgs#parallel", gnused = import_nix "nixpkgs#gnused", }, @@ -29,7 +29,7 @@ let import_nix = organist.lib.import_nix in in { type = "app", - program | organist.contracts.NixString = nix-s%"%{testScript}/bin/run-test.sh"% + program | organist.nix.derivation.NixString = nix-s%"%{testScript}/bin/run-test.sh"% }, flake.checks @@ -42,7 +42,7 @@ let import_nix = organist.lib.import_nix in .. } } - | { _ | organist.builders.NixpkgsPkg } + | { _ | organist.nix.builders.NixpkgsPkg } = { alejandra = { name = "check-alejandra", @@ -74,4 +74,4 @@ let import_nix = organist.lib.import_nix in flake.checks = import "tests/main.ncl", } - | organist.contracts.OrganistExpression + | organist.OrganistExpression diff --git a/run-test.sh b/run-test.sh index ca2b105a..5eab718d 100755 --- a/run-test.sh +++ b/run-test.sh @@ -91,7 +91,7 @@ test_template () { if [[ -n ${1+x} ]]; then test_one_template "$1" else - all_targets=$(nickel export --format raw <<<'std.record.fields ((import "lib/nix.ncl").shells) |> std.string.join "\n"') + all_targets=$(nickel export --format raw <<<'std.record.fields ((import "lib/organist.ncl").shells) |> std.string.join "\n"') # --line-buffer outputs one line at a time, as opposed to dumping all output at once when job finishes # --keep-order makes sure that the order of the output corresponds to the job order, keeping output for each job together # --tag prepends each line with the name of the job diff --git a/templates/default/nickel.lock.ncl b/templates/default/nickel.lock.ncl index d85c324e..059540c7 100644 --- a/templates/default/nickel.lock.ncl +++ b/templates/default/nickel.lock.ncl @@ -1,3 +1,3 @@ { - organist = import "../../lib/nix.ncl", + organist = import "../../lib/organist.ncl", } diff --git a/templates/default/project.ncl b/templates/default/project.ncl index 5b060e78..e7cb8bc4 100644 --- a/templates/default/project.ncl +++ b/templates/default/project.ncl @@ -9,7 +9,7 @@ let organist = inputs.organist in }, shells.dev = { - packages.hello = organist.lib.import_nix "nixpkgs#hello", + packages.hello = organist.import_nix "nixpkgs#hello", }, } - | organist.contracts.OrganistExpression + | organist.OrganistExpression diff --git a/tests/ShellApplication.ncl b/tests/ShellApplication.ncl index 99fa4f34..30882b90 100644 --- a/tests/ShellApplication.ncl +++ b/tests/ShellApplication.ncl @@ -1,4 +1,4 @@ -let organist = import "../lib/nix.ncl" in +let organist = import "../lib/organist.ncl" in let helloScriptFromText = { name = "hello", @@ -8,14 +8,14 @@ let helloScriptFromText = echo "Hello World" "%, } - | organist.builders.ShellApplication + | organist.nix.builders.ShellApplication in let helloScriptFromFile = { name = "hello", - content.file = organist.lib.import_file "tests/hello.sh", + content.file = organist.nix.builtins.import_file "tests/hello.sh", } - | organist.builders.ShellApplication + | organist.nix.builders.ShellApplication in { name = "test-shellapplication", diff --git a/tests/to_file.ncl b/tests/to_file.ncl index 5cd64aa5..0d34d985 100644 --- a/tests/to_file.ncl +++ b/tests/to_file.ncl @@ -1,7 +1,7 @@ -let organist = import "../lib/nix.ncl" in -let file1 = organist.lib.to_file "file1" "important data" in -let file2 = organist.lib.to_file "file2" nix-s%"see %{file1}"% in -organist.builders.NixpkgsPkg +let organist = import "../lib/organist.ncl" in +let file1 = organist.nix.builtins.to_file "file1" "important data" in +let file2 = organist.nix.builtins.to_file "file2" nix-s%"see %{file1}"% in +organist.nix.builders.NixpkgsPkg & { name = "test-to_path", env.buildCommand = nix-s%"