Ws fetch (#45)

simple python cli to fetch shared files

wsFetch/README.md

@@ -0,0 +1,31 @@
+# wsFetch: simple python script to retrieve files using bearer token
+
+```
+usage: wsFetch [-h] [--token-file TOKEN_FILE] URL [URL ...]
+
+Fetch files from WildStore using an access token.
+
+positional arguments:
+  URL                   the URL to download or the name of a file containing a list of URLs to download
+
+options:
+  -h, --help            show this help message and exit
+  --token-file TOKEN_FILE
+                        location of ini file (default is /<homedir>/.config/wildstore.ini) with access token. (token=XXXX)
+```
+
+This script simplifies the downloading of multiple URLs using an access token, also called a bearer token.
+The token can be generated on the wildstore website for the user who would like to do the download.
+
+To use wsFetch create a "token file" with the line
+
+```
+token=XXXX
+```
+
+where XXXX is your token.
+If you store this file in the default location (shown by running `python3 wsFetch.py --help`), you do not need to specify the `--token`
+argument when running the script.
+
+The `URL` parameter of the script can be a URL you wish to download, or if the parameter does not start with `http`, `wsFetch` will interpret the parameter as a file of URLs.
+The files should have a URL on each line, and `wsFetch` will download each URL in order.

wsFetch/wsFetch.py

@@ -0,0 +1,74 @@
+import argparse
+import os
+import pathlib
+import re
+import sys
+import urllib.request
+
+READ_CHUNK_SIZE = 1024*1024
+spinner = ['-', '/', '|', '\\', '-', '/', '|', '\\']
+def fetch(URL, token):
+    request = urllib.request.Request(URL)
+    request.add_header("Authorization", "Bearer " + token)
+    print(request.full_url)
+    print(f"Fetching {URL} ", end="", flush=True)
+    with (urllib.request.urlopen(request) as response):
+        filename = None
+        disposition = response.getheader('Content-Disposition')
+        if disposition:
+            filename = re.findall("filename=(.+)", disposition)[0]
+        if filename:
+            filename = os.path.basename(filename)
+        else:
+            filename = "downloaded.nc"
+        print(f"to {filename} ", end="", flush=True)
+        spin_index = 0
+        with open(filename, "wb") as fd:
+            while True:
+                print(f"\b{spinner[spin_index]}", end="", flush=True)
+                spin_index = (spin_index + 1) % len(spinner)
+
+                data = response.read(READ_CHUNK_SIZE)
+                if data:
+                    fd.write(data)
+                else:
+                    break
+        print("\b.")
+
+
+# Press the green button in the gutter to run the script.
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        prog="wsFetch",
+        description="Fetch files from WildStore using an access token."
+    )
+    parser.add_argument("URL", nargs="+", help="the URL to download or the name of a file containing a list of URLs to download")
+    cf = str(pathlib.Path.home() / ".config" / "wildstore.ini")
+    parser.add_argument("--token-file", default=cf,
+                        help=f"location of ini file (default is {cf}) with access token. (token=XXXX)")
+    args=parser.parse_args()
+    token = None
+    try:
+        with open(args.token_file, "r") as f:
+            for line in f.readlines():
+                parts = line.split("=", 1)
+                if (len(parts) == 2 and parts[0].strip() == "token"):
+                    token = parts[1].strip()
+    except Exception as e:
+        print(e, file=sys.stderr)
+
+    if not token:
+        print(f"no token defined in {args.token_file}. make sure there is a line of the form token=XXXXX")
+        exit(2)
+
+    for U in args.URL:
+        if not U.startswith("http"):
+            print(f"Using {U} as a file with URLS:")
+            try:
+                with open(U, "r") as f:
+                    for line in f.readlines():
+                        fetch(line.strip(), token)
+            except Exception as e:
+                print(e, file=sys.stderr)
+        else:
+            fetch(U, token)