diff --git a/wsFetch/README.md b/wsFetch/README.md new file mode 100644 index 0000000..3b179be --- /dev/null +++ b/wsFetch/README.md @@ -0,0 +1,31 @@ +# wsFetch: simple python script to retrieve files using bearer token + +``` +usage: wsFetch [-h] [--token-file TOKEN_FILE] URL [URL ...] + +Fetch files from WildStore using an access token. + +positional arguments: + URL the URL to download or the name of a file containing a list of URLs to download + +options: + -h, --help show this help message and exit + --token-file TOKEN_FILE + location of ini file (default is //.config/wildstore.ini) with access token. (token=XXXX) +``` + +This script simplifies the downloading of multiple URLs using an access token, also called a bearer token. +The token can be generated on the wildstore website for the user who would like to do the download. + +To use wsFetch create a "token file" with the line + +``` +token=XXXX +``` + +where XXXX is your token. +If you store this file in the default location (shown by running `python3 wsFetch.py --help`), you do not need to specify the `--token` +argument when running the script. + +The `URL` parameter of the script can be a URL you wish to download, or if the parameter does not start with `http`, `wsFetch` will interpret the parameter as a file of URLs. +The files should have a URL on each line, and `wsFetch` will download each URL in order. diff --git a/wsFetch/wsFetch.py b/wsFetch/wsFetch.py new file mode 100644 index 0000000..751c12e --- /dev/null +++ b/wsFetch/wsFetch.py @@ -0,0 +1,74 @@ +import argparse +import os +import pathlib +import re +import sys +import urllib.request + +READ_CHUNK_SIZE = 1024*1024 +spinner = ['-', '/', '|', '\\', '-', '/', '|', '\\'] +def fetch(URL, token): + request = urllib.request.Request(URL) + request.add_header("Authorization", "Bearer " + token) + print(request.full_url) + print(f"Fetching {URL} ", end="", flush=True) + with (urllib.request.urlopen(request) as response): + filename = None + disposition = response.getheader('Content-Disposition') + if disposition: + filename = re.findall("filename=(.+)", disposition)[0] + if filename: + filename = os.path.basename(filename) + else: + filename = "downloaded.nc" + print(f"to {filename} ", end="", flush=True) + spin_index = 0 + with open(filename, "wb") as fd: + while True: + print(f"\b{spinner[spin_index]}", end="", flush=True) + spin_index = (spin_index + 1) % len(spinner) + + data = response.read(READ_CHUNK_SIZE) + if data: + fd.write(data) + else: + break + print("\b.") + + +# Press the green button in the gutter to run the script. +if __name__ == '__main__': + parser = argparse.ArgumentParser( + prog="wsFetch", + description="Fetch files from WildStore using an access token." + ) + parser.add_argument("URL", nargs="+", help="the URL to download or the name of a file containing a list of URLs to download") + cf = str(pathlib.Path.home() / ".config" / "wildstore.ini") + parser.add_argument("--token-file", default=cf, + help=f"location of ini file (default is {cf}) with access token. (token=XXXX)") + args=parser.parse_args() + token = None + try: + with open(args.token_file, "r") as f: + for line in f.readlines(): + parts = line.split("=", 1) + if (len(parts) == 2 and parts[0].strip() == "token"): + token = parts[1].strip() + except Exception as e: + print(e, file=sys.stderr) + + if not token: + print(f"no token defined in {args.token_file}. make sure there is a line of the form token=XXXXX") + exit(2) + + for U in args.URL: + if not U.startswith("http"): + print(f"Using {U} as a file with URLS:") + try: + with open(U, "r") as f: + for line in f.readlines(): + fetch(line.strip(), token) + except Exception as e: + print(e, file=sys.stderr) + else: + fetch(U, token)