🎉 Initial commit

.gitignore

@@ -0,0 +1,48 @@
+
+# Created by https://www.gitignore.io/api/go,linux,code
+# Edit at https://www.gitignore.io/?templates=go,linux,code
+
+### Code ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+### Go ###
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+### Go Patch ###
+/vendor/
+/Godeps/
+
+### Linux ###
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+# End of https://www.gitignore.io/api/go,linux,code

README.md

@@ -0,0 +1,27 @@
+# gemget
+
+A command line downloader for the [Gemini protocol](https://gemini.circumlunar.space/).
+
+```
+gemget [option]... URL...
+
+Usage of gemget:
+  -e, --add-extension     Add .gmi extensions to gemini files that don't have it, like directories.
+  -d, --directory string   The directory where downloads go (default ".")
+      --follow             Follow redirects, up to 5. (default true)
+      --insecure           Skip checking the cert
+  -o, --output string      Output file, for when there is only one URL.
+                           '-' means stdout.
+  -q, --quiet              No output except for errors.
+      --skip               Move to the next URL when one fails. (default true)
+```
+
+# Features to add
+- Support TOFU with a certificate fingerprint cache, and option to disable it
+- Support client certificates
+  - This requires forking the [go-gemini](https://git.sr.ht/~yotam/go-gemini) library this project uses, as it doesn't support that
+- Support interactive input for status code 10
+- Read URLs from file
+
+## License
+This project is under the [MIT License](./LICENSE).

gemget.go

@@ -0,0 +1,179 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	gemini "git.sr.ht/~yotam/go-gemini"
+	"github.com/schollz/progressbar/v3"
+	flag "github.com/spf13/pflag"
+)
+
+//var cert = flag.String("cert", "", "Not implemented.")
+//var key = flag.String("key", "", "Not implemented.")
+
+var insecure = flag.Bool("insecure", false, "Skip checking the cert")
+var dir = flag.StringP("directory", "d", ".", "The directory where downloads go")
+var output = flag.StringP("output", "o", "", "Output file, for when there is only one URL.\n'-' means stdout.")
+var errorSkip = flag.Bool("skip", true, "Move to the next URL when one fails.")
+var follow = flag.Bool("follow", true, "Follow redirects, up to 5.")
+var exts = flag.BoolP("add-extension", "e", false, "Add .gmi extensions to gemini files that don't have it, like directories.")
+var quiet = flag.BoolP("quiet", "q", false, "No output except for errors.")
+
+func fatal(format string, a ...interface{}) {
+	urlError(format, a...)
+	os.Exit(1)
+}
+
+func urlError(format string, a ...interface{}) {
+	if strings.HasPrefix(format, "\n") {
+		format = "*** " + format[:len(format)-1] + " ***\n"
+	} else {
+		format = "*** " + format + " ***\n"
+	}
+	fmt.Fprintf(os.Stderr, format, a...)
+	if !*errorSkip {
+		os.Exit(1)
+	}
+}
+
+func saveFile(resp *gemini.Response, u *url.URL) {
+	var name string
+	if *output != "" {
+		name = *output
+	} else {
+		name = path.Base(u.Path) // Filename from URL
+		if name == "/" || name == "." {
+			// Domain is being downloaded, so there's no path/file
+			name = u.Hostname()
+		}
+		if *exts && !(strings.HasSuffix(name, ".gmi") && strings.HasSuffix(name, ".gemini")) && (resp.Meta == "" || strings.HasPrefix(resp.Meta, "text/gemini")) {
+			// It's a gemini file, but it doesn't have that extension - and the user wants them added
+			name += ".gmi"
+		}
+	}
+
+	f, err := os.OpenFile(filepath.Join(*dir, name), os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		fatal("Error saving file %s", name)
+	}
+	defer f.Close()
+
+	var written int64
+	if *quiet {
+		written, err = io.Copy(f, resp.Body)
+	} else {
+		bar := progressbar.DefaultBytes(-1, "downloading")
+		written, err = io.Copy(io.MultiWriter(f, bar), resp.Body)
+		fmt.Println()
+	}
+	if err != nil {
+		fatal("Error saving file %s, %d bytes saved.", name, written)
+	}
+}
+
+func _fetch(n int, u *url.URL, client *gemini.Client) {
+	uStr := u.String()
+	resp, err := client.Fetch(uStr)
+	if err != nil {
+		urlError(err.Error())
+		return
+	}
+	defer resp.Body.Close()
+
+	// Validate status
+	if resp.Status >= 60 {
+		urlError("%s needs a certificate, which is not implemented yet.", uStr)
+		return
+	} else if gemini.SimplifyStatus(resp.Status) == 30 {
+		if !*follow {
+			urlError("%s redirects.", uStr)
+			return
+		}
+		// Redirect
+		if n == 5 {
+			urlError("%s redirected too many times.", uStr)
+			return
+		}
+		// Follow the recursion
+		redirect, err := url.Parse(resp.Meta)
+		if err != nil {
+			urlError("Redirect url %s couldn't be parsed.", resp.Meta)
+			return
+		}
+		fmt.Printf("*** Redirected to %s ***\n", resp.Meta)
+		_fetch(n+1, u.ResolveReference(redirect), client)
+	} else if resp.Status == 10 {
+		urlError("%s needs input, which is not implemented yet. You should make the request manually with a URL query.", uStr)
+	} else if gemini.SimplifyStatus(resp.Status) == 20 {
+		// Output to stdout, otherwise save it to a file
+		if *output == "-" {
+			io.Copy(os.Stdout, resp.Body)
+			return
+		}
+		saveFile(&resp, u)
+		return
+	} else {
+		urlError("%s returned status %d, skipping.", u, resp.Status)
+	}
+}
+
+func fetch(u *url.URL, client *gemini.Client) {
+	_fetch(1, u, client)
+}
+
+func main() {
+	flag.Parse()
+
+	// Validate urls
+	if len(flag.Args()) == 0 {
+		flag.Usage()
+		os.Exit(0)
+	}
+	urls := make([]*url.URL, len(flag.Args()))
+	for i, u := range flag.Args() {
+		parsed, err := url.Parse(u)
+		if err != nil {
+			urlError("%s could not be parsed.", u)
+			continue
+		}
+		if len(parsed.Scheme) != 0 && parsed.Scheme != "gemini" {
+			urlError("%s is not a gemini URL.", u)
+			continue
+		}
+		if !strings.HasPrefix(u, "gemini://") {
+			// Have to reparse due to the way the lib works
+			parsed, err = url.Parse("gemini://" + u)
+			if err != nil {
+				urlError("Adding gemini:// to %s failed.", u)
+			}
+		}
+		if parsed.Port() == "" {
+			// Add port, gemini library requires it
+			parsed.Host = parsed.Hostname() + ":" + "1965"
+		}
+		if parsed.Path == "" {
+			// Add slash to the end of domains to prevent redirects
+			parsed.Path = "/"
+		}
+		urls[i] = parsed
+	}
+
+	// Validate flags
+	if len(flag.Args()) > 1 && *output != "" && *output != "-" {
+		fatal("The output flag cannot be specified when there are multiple URLs, unless it is '-'.")
+	}
+	// Fetch each URL
+	client := &gemini.Client{InsecureSkipVerify: *insecure}
+	for _, u := range urls {
+		if !*quiet {
+			fmt.Printf("*** Started %s ***\n", u)
+		}
+		fetch(u, client)
+	}
+}

go.mod

@@ -0,0 +1,9 @@
+module github.com/makeworld-the-better-one/gemget
+
+go 1.14
+
+require (
+	git.sr.ht/~yotam/go-gemini v0.0.0-20191116204306-8ebb75240eef
+	github.com/schollz/progressbar/v3 v3.3.2
+	github.com/spf13/pflag v1.0.5
+)

go.sum

@@ -0,0 +1,23 @@
+git.sr.ht/~yotam/go-gemini v0.0.0-20191116204306-8ebb75240eef h1:rHPrfUoN0cIwxf5PaSDQgDd9r1kBab0OiEu2akI12dU=
+git.sr.ht/~yotam/go-gemini v0.0.0-20191116204306-8ebb75240eef/go.mod h1:KxQlipD0Ti7MfV3itYJfuvgcvd+SOlRTtbOK+A0DCCE=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/schollz/progressbar/v3 v3.3.2 h1:/jzin2rOEWdXUufUR48PVIWcPrSdsCvmy40o+RlZ+z0=
+github.com/schollz/progressbar/v3 v3.3.2/go.mod h1:N/820QRS3ua9DhrVnLShsNgAEKNYFd89Cf5syXfqeyQ=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=