Allow online installation of packages (#56)

## Registries

_See the newly added registry.nuon file for more specific info about the
registry format._

This PR adds a new concept: **registry**. Registries are .nuon files
containing a list of packages, their versions, and how they should be
installed. Currently, registry can contain two kinds of packages:
* `local` -- installed from local file system
* `git` -- installed from remote Git repository

The list of recognized registries is stored in `$env.NUPM_REGISTRIES` as
a record of name: path pairs. That way, users can maintain their
collection of sources from where they want to install packages. Not sure
if this will be the final way to pass available registries to nupm, but
should be good enough for early testing.

## Online Installation

If a package is to be installed from a Git repository, the repository is
cached in `$env.NUPM_CACHE` to avoid re-cloning the same repository all
the time. Once cloned, the local path of the cloned repository is passed
to the installer, as if you called `nupm install --path=...`.

One important missing feature is some kind of file integrity check. One
step at a time...

## Versions

If a registry contains multiple versions (common scenario), by default,
the newest one is installed. "Newest" currently means sorting the
versions alphabetically and taking the last one. If `--pkg-version` is
specified, nupm tries to install the exact version passed to this
option. This needs better version matching (e.g., version 0.2 matching
0.2.1 etc.), but again, one step at a time...

## New command

`nupm search` searches for packages in registries without installing
them. I thought this would be useful to have early on.

## Try it out

From the repository root:

```nushell
overlay use nupm/
nupm install nu-git-manager
# use nu-git-manager *
# ...
```

## TODO

- [x] Remove debug stuff
- [x] Nicer / more informative prints
- [x] (can be done over time) Rewrite
https://github.com/nushell/nupm/blob/main/index.nuon to fit the registry
format, move it to its own repo
	- It would be a single-file repository
- Package authors would put PRs there whenever they update their package
- [x] Tests

## After
- better version matching => `0.2` shouldn't match `0.2.1`
- file integrity checks

nupm/install.nu

@@ -1,9 +1,19 @@
 use std log
 
-use utils/dirs.nu [ nupm-home-prompt script-dir module-dir tmp-dir ]
+use utils/completions.nu complete-registries
+use utils/dirs.nu [ nupm-home-prompt cache-dir module-dir script-dir tmp-dir ]
 use utils/log.nu throw-error
+use utils/misc.nu check-cols
+use utils/registry.nu search-package
+use utils/version.nu filter-by-version
 
 def open-package-file [dir: path] {
+    if not ($dir | path exists) {
+        throw-error "package_dir_does_not_exist" (
+            $"Package directory ($dir) does not exist"
+        )
+    }
+
     let package_file = $dir | path join "nupm.nuon"
 
     if not ($package_file | path exists) {
@@ -92,8 +102,8 @@ def install-path [
 
             if ($destination | path type) == dir {
                 throw-error "package_already_installed" (
-                    $"Package ($package.name) is already installed."
-                    + "Use `--force` to override the package"
+                    $"Package ($package.name) is already installed in"
+                    + $" ($destination). Use `--force` to override the package"
                 )
             }
 
@@ -144,20 +154,129 @@ def install-path [
     }
 }
 
+
+# Downloads a package and returns its downloaded path
+def download-pkg [
+    pkg: record<
+        name: string,
+        version: string,
+        url: string,
+        revision: string,
+        path: string,
+        type: string,
+    >
+]: nothing -> path {
+    # TODO: Add some kind of hashing to check that files really match
+
+    if ($pkg.type != 'git') {
+        throw-error 'Downloading non-git packages is not supported yet'
+    }
+
+    let cache_dir = cache-dir --ensure
+    cd $cache_dir
+
+    let git_dir = $cache_dir | path join git
+    mkdir $git_dir
+    cd $git_dir
+
+    let repo_name = $pkg.url | url parse | get path | path parse | get stem
+    let url_hash = $pkg.url | hash md5 # in case of git repo name collision
+    let clone_dir = $'($repo_name)-($url_hash)-($pkg.revision)'
+
+    let pkg_dir = $env.PWD | path join $clone_dir $pkg.path
+
+    if ($pkg_dir | path exists) {
+        print $'Package ($pkg.name) found in cache'
+        return $pkg_dir
+    }
+
+    try {
+        git clone $pkg.url $clone_dir
+    } catch {
+        throw-error $'Error cloning repository ($pkg.url)'
+    }
+
+    cd $clone_dir
+
+    try {
+        git checkout $pkg.revision
+    } catch {
+        throw-error $'Error checking out revision ($pkg.revision)'
+    }
+
+    if not ($pkg_dir | path exists) {
+        throw-error $'Path ($pkg.path) does not exist'
+    }
+
+    $pkg_dir
+}
+
+# Fetch a package from a registry
+def fetch-package [
+    package: string  # Name of the package
+    --registry: string  # Which registry to use
+    --version: string  # Package version to install (string or null)
+]: nothing -> path {
+    let regs = (search-package $package
+        --registry $registry
+        --version $version
+        --exact-match)
+
+    if ($regs | is-empty) {
+        throw-error $'Package ($package) not found in any registry'
+    } else if ($regs | length) > 1 {
+        # TODO: Here could be interactive prompt
+        throw-error $'Multiple registries contain package ($package)'
+    }
+
+    # Now, only one registry contains the package
+    let reg = $regs | first
+    let pkgs = $reg.pkgs | filter-by-version $version
+
+    let pkg = try {
+        $pkgs | last
+    } catch {
+        throw-error $'No package matching version `($version)`'
+    }
+
+    print $pkg
+
+    if $pkg.type == 'git' {
+        download-pkg $pkg
+    } else {
+        # local package path is relative to the registry file (absolute paths
+        # are discouraged but work)
+        $reg.path | path dirname | path join $pkg.path
+    }
+}
+
 # Install a nupm package
+#
+# Installation consists of two parts:
+# 1. Fetching the package (if the package is online)
+# 2. Installing the package (build action, if any; copy files to install location)
 export def main [
-    package # Name, path, or link to the package
+    package  # Name, path, or link to the package
+    --registry: string@complete-registries  # Which registry to use (either a name
+                                            # in $env.NUPM_REGISTRIES or a path)
+    --pkg-version(-v): string  # Package version to install
     --path  # Install package from a directory with nupm.nuon given by 'name'
     --force(-f)  # Overwrite already installed package
-    --no-confirm # Allows to bypass the interactive confirmation, useful for scripting
+    --no-confirm  # Allows to bypass the interactive confirmation, useful for scripting
 ]: nothing -> nothing {
     if not (nupm-home-prompt --no-confirm=$no_confirm) {
         return
     }
 
-    if not $path {
-        throw-error "missing_required_option" "`nupm install` currently requires a `--path` flag"
+    let pkg: path = if not $path {
+        fetch-package $package --registry $registry --version $pkg_version
+    } else {
+        if $pkg_version != null {
+            throw-error "Use only --path or --pkg-version, not both"
+        }
+
+        $package
     }
 
-    install-path $package --force=$force
+    install-path $pkg --force=$force
 }

nupm/mod.nu

@@ -1,7 +1,10 @@
-use utils/dirs.nu [ DEFAULT_NUPM_HOME DEFAULT_NUPM_TEMP nupm-home-prompt ]
+use utils/dirs.nu [
+    DEFAULT_NUPM_HOME DEFAULT_NUPM_TEMP DEFAULT_NUPM_CACHE  nupm-home-prompt
+]
 
 export module install.nu
 export module test.nu
+export module search.nu
 
 export-env {
     # Ensure that $env.NUPM_HOME is always set when running nupm. Any missing
@@ -10,6 +13,17 @@ export-env {
 
     # Ensure temporary path is set.
     $env.NUPM_TEMP = ($env.NUPM_TEMP? | default $DEFAULT_NUPM_TEMP)
+
+    # Ensure install cache is set
+    $env.NUPM_CACHE = ($env.NUPM_CACHE? | default $DEFAULT_NUPM_CACHE)
+
+    # TODO: Maybe this is not the best way to set registries, but should be
+    #       good enough for now.
+    # TODO: Add `nupm registry` for showing info about registries
+    # TODO: Add `nupm registry add/remove` to add/remove registry from the env?
+    $env.NUPM_REGISTRIES = {
+        nupm_test: 'https://raw.githubusercontent.com/nushell/nupm/main/registry.nuon'
+    }
 }
 
 # Nushell Package Manager

nupm/search.nu

@@ -0,0 +1,12 @@
+use utils/completions.nu complete-registries
+use utils/registry.nu search-package
+
+# Search for a package
+export def main [
+    package  # Name, path, or link to the package
+    --registry: string@complete-registries  # Which registry to use (either a name
+                                            # in $env.NUPM_REGISTRIES or a path)
+    --pkg-version(-v): string  # Package version to install
+]: nothing -> table {
+    search-package $package --registry $registry --version $pkg_version
+}

nupm/utils/completions.nu

@@ -0,0 +1,3 @@
+export def complete-registries [] {
+    $env.NUPM_REGISTRIES? | default {} | columns
+}

nupm/utils/dirs.nu

@@ -3,6 +3,10 @@
 # Default installation path for nupm packages
 export const DEFAULT_NUPM_HOME = ($nu.default-config-dir | path join "nupm")
 
+# Default path for installation cache
+export const DEFAULT_NUPM_CACHE = ($nu.default-config-dir
+    | path join nupm cache)
+
 # Default temporary path for various nupm purposes
 export const DEFAULT_NUPM_TEMP = ($nu.temp-path | path join "nupm")
 
@@ -70,6 +74,16 @@ export def module-dir [--ensure]: nothing -> path {
     $d
 }
 
+export def cache-dir [--ensure]: nothing -> path {
+    let d = $env.NUPM_CACHE
+
+    if $ensure {
+        mkdir $d
+    }
+
+    $d
+}
+
 export def tmp-dir [subdir: string, --ensure]: nothing -> path {
     let d = $env.NUPM_TEMP
         | path join $subdir

nupm/utils/misc.nu

@@ -0,0 +1,38 @@
+# Misc unsorted helpers
+
+# Make sure input has requested columns and no extra columns
+export def check-cols [
+    what: string, 
+    required_cols: list<string>
+    --extra-ok
+    --missing-ok
+]: [ table -> table, record -> record ]  {
+    let inp = $in
+
+    if ($inp | is-empty) {
+        return $inp
+    }
+
+    let cols = $inp | columns
+    if not $missing_ok {
+        let missing_cols = $required_cols | where {|req_col| $req_col not-in $cols }
+
+        if not ($missing_cols | is-empty) {
+            throw-error ($"Missing the following required columns in ($what):"
+                + $" ($missing_cols | str join ', ')")
+            )
+        }
+    }
+
+    if not $extra_ok {
+        let extra_cols = $cols | where {|col| $col not-in $required_cols }
+
+        if not ($extra_cols | is-empty) {
+            throw-error ($"Got the following extra columns in ($what):"
+                + $" ($extra_cols | str join ', ')")
+            )
+        }
+    }
+
+    $inp
+}

nupm/utils/registry.nu

@@ -0,0 +1,77 @@
+# Utilities related to nupm registries
+
+# Search for a package in a registry
+export def search-package [
+    package: string  # Name of the package
+    --registry: string  # Which registry to use
+    --version: any  # Package version to install (string or null)
+    --exact-match  # Searched package name must match exactly
+] -> table {
+    let registries = if (not ($registry | is-empty)) and ($registry in $env.NUPM_REGISTRIES) {
+        # If $registry is a valid column in $env.NUPM_REGISTRIES, use that
+        { $registry : ($env.NUPM_REGISTRIES | get $registry) }
+    } else if (not ($registry | is-empty)) and ($registry | path exists) {
+        # If $registry is a path, use that
+        let reg_name = $registry | path parse | get stem
+        { $reg_name: $registry }
+    } else {
+        # Otherwise use $env.NUPM_REGISTRIES
+        $env.NUPM_REGISTRIES
+    }
+
+    let name_matcher: closure = if $exact_match {
+        {|row| $row.name == $package }
+    } else {
+        {|row| $package in $row.name }
+    }
+
+    # Collect all registries matching the package and all matching packages
+    let regs = $registries
+        | items {|name, path|
+            # Open registry (online or offline)
+            let registry = if ($path | path type) == file {
+                open $path
+            } else {
+                try {
+                    let reg = http get $path
+
+                    if local in $reg {
+                        throw-error ("Can't have local packages in online registry"
+                            + $" '($path)'.")
+                    }
+
+                    $reg
+                } catch {
+                    throw-error $"Cannot open '($path)' as a file or URL."
+                }
+            }
+
+            $registry | check-cols --missing-ok "registry" [ git local ] | ignore
+
+            # Find all packages matching $package in the registry
+            let pkgs_local = $registry.local?
+                | default []
+                | check-cols "local packages" [ name version path ]
+                | filter $name_matcher
+
+            let pkgs_git = $registry.git?
+                | default []
+                | check-cols "git packages" [ name version url revision path ]
+                | filter $name_matcher
+
+            let pkgs = $pkgs_local
+                | insert type local
+                | insert url null
+                | insert revision null
+                | append ($pkgs_git | insert type git)
+
+            {
+                name: $name
+                path: $path
+                pkgs: $pkgs
+            }
+        }
+        | compact
+
+    $regs | where not ($it.pkgs | is-empty)
+}

nupm/utils/version.nu

@@ -0,0 +1,24 @@
+# Commands related to handling versions
+#
+# We might move some of this to Nushell builtins
+
+# Sort packages by version
+def sort-by-version []: table<version: string> -> table<version: string> {
+    sort-by version
+}
+
+# Check if the target version is equal or higher than the target version
+def matches-version [version: string]: string -> bool {
+    # TODO: Add proper version matching
+    $in == $version
+}
+
+# Filter packages by version and sort them by version
+export def filter-by-version [version: any]: table<version: string> -> table<version: string> {
+    if $version == null {
+        $in
+    } else {
+        $in | filter {|row| $row.version | matches-version $version}
+    }
+    | sort-by-version
+}