Merge pull request #11 from program--/jsm-patch-3

Update subsetting code & initial terraform configuration

.dockerignore

@@ -1,3 +1,5 @@
 cli
 .git
-.github
+.github
+.vscode
+terraform

.github/workflows/Generate-subset-app.yaml

@@ -1,5 +1,7 @@
 on:
   push:
+    branches:
+      - main
     paths:
       - cli/**.go
       - '.github/workflows/Generate-subset-app.yaml'
@@ -25,21 +27,21 @@ jobs:
       - name: Package MacOS executable
         working-directory: ./cli
         run: |
-          GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build
+          GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build -o hfsubset
           tar -czvf hfsubset-macos_amd64.tar.gz hfsubset
           rm hfsubset
 
       - name: Package Linux executable
         working-directory: ./cli
         run: |
-          GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build
+          GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o hfsubset
           tar -czvf hfsubset-linux_amd64.tar.gz hfsubset
           rm hfsubset
   
       - name: Package Windows executable
         working-directory: ./cli
         run: |
-          GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build
+          GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o hfsubset
           zip hfsubset-windows_amd64.zip hfsubset.exe
           rm hfsubset.exe
 

.gitignore

@@ -4,3 +4,6 @@
 .Ruserdata
 hfsubset
 *.gpkg
+
+terraform/.terraform
+terraform/.infracost

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "hydrofabric"]
+	path = hydrofabric
+	url = https://github.com/NOAA-OWP/hydrofabric.git

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "go.toolsEnvVars": {
+        "GO111MODULE": "on"
+    }
+}

Dockerfile → Dockerfile.lambda

@@ -25,9 +25,7 @@ RUN apt update \
 RUN mkdir -p /hydrofabric/subset
 
 # Install CRAN Packages
-# && echo "options(Ncpus = $(nproc --all), repos=c(CRAN = 'https://packagemanager.rstudio.com/cran/__linux__/${DISTRIB_CODENAME}/latest'))" >> .Rprofile \
-ENV HF_CRAN_R_PKGS="pak box zlib cli arrow crayon dplyr DBI RSQLite sf \
-                    terra lambdr glue rstudioapi purrr magrittr nhdplusTools aws.s3"
+ENV HF_CRAN_R_PKGS="arrow aws.s3 base64enc box DBI dplyr glue lambdr logger nhdplusTools pak readr RSQLite sf"
 RUN cd /hydrofabric \
     && . /etc/lsb-release \
     && echo "options(ncpus = $(nproc --all))" >> .Rprofile \
@@ -38,6 +36,7 @@ RUN cd /hydrofabric \
                   ${HF_CRAN_R_PKGS}
 
 COPY . /hydrofabric/subset
+COPY hydrofabric/inst/qml /hydrofabric/qml
 
 RUN cd /hydrofabric \
     && chmod 755 subset/runtime.R \

cli/go.mod

@@ -1,13 +1,3 @@
 module hfsubset
 
 go 1.20
-
-require github.com/schollz/progressbar/v3 v3.13.1
-
-require (
-	github.com/mattn/go-runewidth v0.0.14 // indirect
-	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
-	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/term v0.8.0 // indirect
-)

cli/go.sum

@@ -1,26 +0,0 @@
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
-github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
-github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
-github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
-github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
-golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=

cli/main.go

@@ -26,11 +26,11 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
-
-	"github.com/schollz/progressbar/v3"
 )
 
 const usage string = `hfsubset - Hydrofabric Subsetter
@@ -48,6 +48,15 @@ Examples:
 
   hfsubset -o ./poudre.gpkg -t hl_uri "Gages-06752260"
 
+  # Using network-linked data index identifiers
+  hfsubset -o ./poudre.gpkg -t nldi_feature "nwis:USGS-08279500"
+  
+  # Specifying layers and hydrofabric version
+  hfsubset -l divides,nexus -o ./divides_nexus.gpkg -r "v20" -t hl_uri "Gages-06752260"
+  
+  # Finding data around a POI
+  hfsubset -l flowpaths,reference_flowpaths -o ./sacramento_flowpaths.gpkg -t xy -121.494400,38.581573
+
 Options:
 `
 
@@ -63,6 +72,7 @@ type SubsetResponse struct {
 	data []byte
 }
 
+// Parse comma-delimited layers string
 func (opts *SubsetRequest) Layers() []string {
 	split := strings.Split(*opts.layers, ",")
 	for i, v := range split {
@@ -71,10 +81,53 @@ func (opts *SubsetRequest) Layers() []string {
 	return split
 }
 
-func (opts *SubsetRequest) IDs() []string {
+// Parse IDs format, i.e. trim spaces
+func (opts *SubsetRequest) IDs(key string) []string {
 	for i, v := range opts.id {
 		opts.id[i] = strings.TrimSpace(v)
 	}
+
+	if key == "nldi_feature" {
+		var feat struct {
+			FeatureSource string `json:"featureSource"`
+			FeatureId     string `json:"featureId"`
+		}
+
+		feat.FeatureSource = ""
+		feat.FeatureId = ""
+		for i, v := range opts.id {
+			f := strings.Split(v, ":")
+
+			feat.FeatureSource = f[0]
+			feat.FeatureId = f[1]
+			fstr, _ := json.Marshal(feat)
+			opts.id[i] = string(fstr)
+			feat.FeatureSource = ""
+			feat.FeatureId = ""
+		}
+	}
+
+	if key == "xy" {
+		var xy struct {
+			X float64
+			Y float64
+		}
+
+		xy.X = -1
+		xy.Y = -1
+		for i, v := range opts.id {
+			f := strings.Split(v, ",")
+
+			xy.X, _ = strconv.ParseFloat(f[0], 64)
+			xy.Y, _ = strconv.ParseFloat(f[1], 64)
+
+			fstr, _ := json.Marshal(xy)
+			opts.id[i] = string(fstr)
+			xy.X = -1
+			xy.Y = -1
+		}
+	}
+
 	return opts.id
 }
 
@@ -85,33 +138,25 @@ func (opts *SubsetRequest) MarshalJSON() ([]byte, error) {
 	switch *opts.id_type {
 	case "id":
 		key = "id"
-		break
 	case "hl_uri":
 		key = "hl_uri"
-		break
 	case "comid":
 		key = "comid"
-		break
 	case "nldi_feature":
-		// key = "nldi"
-		// break
-		fallthrough
+		key = "nldi_feature"
 	case "xy":
-		// key = "loc"
-		// break
-		panic("-nldi_feature and -xy support are not implemented currently")
+		key = "xy"
 	default:
 		panic("type " + *opts.id_type + " not supported; only one of: id, hl_uri, comid, nldi_feature, xy")
 	}
 
 	jsonmap["layers"] = opts.Layers()
-	jsonmap[key] = opts.IDs()
-    // TODO: use opts.version
-	jsonmap["version"] = "v20" // v20 is v2.0
+	jsonmap[key] = opts.IDs(key)
+	jsonmap["version"] = *opts.version
 	return json.Marshal(jsonmap)
 }
 
-func makeRequest(lambda_endpoint string, opts *SubsetRequest, bar *progressbar.ProgressBar) *SubsetResponse {
+func makeRequest(lambda_endpoint string, opts *SubsetRequest, logger *log.Logger) *SubsetResponse {
 	var uri string = lambda_endpoint + "/2015-03-31/functions/function/invocations"
 	payload, err := opts.MarshalJSON()
 	if err != nil {
@@ -120,14 +165,14 @@ func makeRequest(lambda_endpoint string, opts *SubsetRequest, bar *progressbar.P
 
 	reader := bytes.NewReader(payload)
 
-	bar.Describe("[1/4] waiting for response")
+	logger.Println("[1/4] waiting for response")
 	req, err := http.Post(uri, "application/json", reader)
 	if err != nil {
 		panic(err)
 	}
 	defer req.Body.Close()
 
-	bar.Describe("[2/4] reading hydrofabric subset")
+	logger.Println("[2/4] reading hydrofabric subset")
 	resp := new(SubsetResponse)
 	b := new(bytes.Buffer)
 	buffer := bufio.NewWriter(b)
@@ -137,15 +182,15 @@ func makeRequest(lambda_endpoint string, opts *SubsetRequest, bar *progressbar.P
 	}
 
 	r := b.Bytes()
+
+	// Trim quotes if returned
 	if r[0] == '"' && r[len(r)-1] == '"' {
 		r = r[1 : len(r)-1]
 	}
 
-	bar.Describe("[3/4] decoding gzip")
+	logger.Println("[3/4] parsing base64 response")
 	rr := bytes.NewReader(r)
 	gpkg := base64.NewDecoder(base64.StdEncoding, rr)
-	// gpkg, _ := gzip.NewReader(rr)
-	// defer gpkg.Close()
 	resp.data, err = io.ReadAll(gpkg)
 	if err != nil {
 		panic(err)
@@ -154,16 +199,15 @@ func makeRequest(lambda_endpoint string, opts *SubsetRequest, bar *progressbar.P
 	return resp
 }
 
-func writeToFile(request *SubsetRequest, response *SubsetResponse, bar *progressbar.ProgressBar) int {
+func writeToFile(request *SubsetRequest, response *SubsetResponse, logger *log.Logger) int {
 	f, err := os.Create(*request.output)
 	if err != nil {
 		panic(err)
 	}
 
-	bar.Describe(fmt.Sprintf("[4/4] writing to %s", *request.output))
+	logger.Printf("[4/4] writing to %s", *request.output)
 	w := bufio.NewWriter(f)
-	mw := io.MultiWriter(w, bar)
-	n, err := mw.Write(response.data)
+	n, err := w.Write(response.data)
 	if err != nil {
 		panic(err)
 	}
@@ -186,7 +230,7 @@ Either "all" or "core", or one or more of:
 	opts := new(SubsetRequest)
 	opts.id_type = flag.String("t", "id", `One of: "id", "hl_uri", "comid", "xy", or "nldi_feature"`)
 	opts.layers = flag.String("l", "core", layers_help)
-	opts.version = flag.String("r", "pre-release", "Hydrofabric version")
+	opts.version = flag.String("r", "v20", "Hydrofabric version")
 	opts.output = flag.String("o", "hydrofabric.gpkg", "Output file name")
 	quiet := flag.Bool("quiet", false, "Disable progress bar")
 	flag.Parse()
@@ -205,12 +249,11 @@ Either "all" or "core", or one or more of:
 		*opts.layers = "divides,nexus,flowpaths,network,hydrolocations"
 	}
 
-	bar := progressbar.NewOptions(3,
-		progressbar.OptionSetWidth(15),
-		progressbar.OptionSetDescription("[0/4] sending http request"),
-		progressbar.OptionShowBytes(false),
-		progressbar.OptionSetVisibility(!*quiet),
-	)
+	logger := log.New(os.Stdout, "hfsubset ==> ", log.Ltime)
+	if *quiet {
+		logger.SetOutput(io.Discard)
+	}
+	logger.Println("[0/4] sending http request")
 
 	var endpoint string
 	if v, ok := os.LookupEnv("HFSUBSET_ENDPOINT"); ok {
@@ -220,11 +263,9 @@ Either "all" or "core", or one or more of:
 		endpoint = "https://hfsubset-e9kvx.ondigitalocean.app"
 	}
 
-	resp := makeRequest(endpoint, opts, bar)
+	resp := makeRequest(endpoint, opts, logger)
 	response_size := len(resp.data)
-	bytes_written := writeToFile(opts, resp, bar)
-	bar.Finish()
-	println() // so progress bar doesn't show up
+	bytes_written := writeToFile(opts, resp, logger)
 
 	if bytes_written != response_size {
 		panic(fmt.Sprintf("wrote %d bytes out of %d bytes to %s", bytes_written, response_size, *opts.output))