Merge pull request #127 from voxel51/merge/v1.6.0-to-main

Merge v1.6.0 to main
voxel51 · Apr 30, 2024 · 3c063ba · 3c063ba
2 parents c7b1414 + cf6819a
commit 3c063ba
Show file tree

Hide file tree

Showing 55 changed files with 6,023 additions and 956 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,11 +1,13 @@
 *.tgz
-compose.override.yaml
+*~
+.DS_Store
 .env
-helm/gke-example/voxel51-docker.json
+compose.override*yaml
 helm/gke-example/values-*.yaml
+helm/gke-example/voxel51-docker.json
+tests/integration/*/test_output*.log
+tests/integration/*/test_output*/*
+tests/integration/*/test_reports/*
 tests/unit/*/test_output.log
 tests/unit/*/test_output/*
 tests/unit/*/test_reports/*
-tests/integration/*/test_output.log
-tests/integration/*/test_output/*
-tests/integration/*/test_reports/*
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -27,6 +27,16 @@ repos:
           - go-mod
           - go-sum
           - makefile
+  - repo: https://github.com/norwoodj/helm-docs
+    rev: v1.11.3
+    hooks:
+      - id: helm-docs
+        args:
+          # Make the tool search for charts only under the `helm/fiftyone-teams-app` directory
+          - --chart-search-root=helm/fiftyone-teams-app
+
+          # A base filename makes it relative to each chart directory found
+          - --template-files=README.md.gotmpl
   - repo: https://github.com/igorshubovych/markdownlint-cli
     rev: v0.36.0
     hooks:
@@ -44,16 +54,6 @@ repos:
         entry: yamllint --config-file .yamllint.yaml
         # We exclude helm template files since in raw form, they are not valid yaml
         exclude: helm/fiftyone-teams-app/templates
-  - repo: https://github.com/norwoodj/helm-docs
-    rev: v1.11.3
-    hooks:
-      - id: helm-docs
-        args:
-          # Make the tool search for charts only under the `helm/fiftyone-teams-app` directory
-          - --chart-search-root=helm/fiftyone-teams-app
-
-          # A base filename makes it relative to each chart directory found
-          - --template-files=README.md.gotmpl
   - repo: https://github.com/Lucas-C/pre-commit-hooks-nodejs
     rev: v1.1.2
     hooks:
@@ -66,13 +66,17 @@ repos:
           - helm/fiftyone-teams-app/README.md.gotmpl
           - helm/README.md
           - CONTRIBUTING.md
+  - repo: https://github.com/gruntwork-io/pre-commit
+    rev: v0.1.23
+    hooks:
+      - id: helmlint
   - repo: https://github.com/Yelp/detect-secrets
     rev: v1.4.0
     hooks:
       - id: detect-secrets
         args:
           - --exclude-secrets
-          - '(password|REPLACEME|fiftyone-teams-tls-secret|btv8BiFCaPIayWU3IU3a_Lm_EMIIk-t6H_yN1ORV45o=|5b32118032bfd50b64b3cc7c0e0821f4e84f63ad517a9687ac2b6ce6ab261976|test-*|/api/proxy/fiftyone-teams|/opt/plugins)'
+          - '(password|REPLACEME|fiftyone-teams-tls-secret|3-9XjJ-gUV?vp\^e\(WUk>LD&lAjh7yEji|btv8BiFCaPIayWU3IU3a_Lm_EMIIk-t6H_yN1ORV45o=|5b32118032bfd50b64b3cc7c0e0821f4e84f63ad517a9687ac2b6ce6ab261976|test-*|/api/proxy/fiftyone-teams|/opt/plugins)'
   - repo: https://github.com/dnephin/pre-commit-golang
     rev: v0.5.1
     hooks:

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -34,10 +34,10 @@
 
 ## Quickstart to Skaffold in Minikube
 
-1. Auth with gcloud
+1. Auth with gcloud for the project `computer-vision-team`
 
     ```shell
-    gcloud auth application-default login
+    make auth
     ```
 
 1. Install the asdf tools
@@ -53,7 +53,8 @@
     ```
 
 1. Set skaffold secrets.
-   See [skaffold](#skaffold)
+   See
+   [skaffold](#skaffold)
 
 1. Run skaffold
 
@@ -265,7 +266,7 @@ skaffold dev --profile only-fiftyone
 Our FiftyOne Teams container images are stored in the private repositories
 
 - [Google Artifact Repository (Docker)](#google-artifact-repository)
-  - Contains private development images created by our
+  - Contains private development images created by our private repository
     [Google Cloud Build](https://github.com/voxel51/cloud-build-and-deploy/)
     CI/CD runs
     - Development
@@ -293,7 +294,8 @@ GCP project `computer-vision-team`, configure minikube and skaffold
     minikube addons enable gcp-auth
     ```
 
-1. In [skaffold.yaml](./skaffold.yaml)
+1. In
+   [skaffold.yaml](./skaffold.yaml)
    comment `imagePullSecrets` for the helm release named `fiftyone-teams-app`
    in `setValueTemplates.imagePullSecrets[0].name=regcred`
 
@@ -407,7 +409,8 @@ To run released images from Docker hub, configure minikube and Skaffold
       --namespace "${NAMESPACE}" \
     ```
 
-1. In [skaffold.yaml](./skaffold.yaml)
+1. In
+   [skaffold.yaml](./skaffold.yaml)
    set `imagePullSecrets` for the helm release named `fiftyone-teams-app`
    in `setValueTemplates.imagePullSecrets[0].name=regcred`
 
@@ -482,7 +485,9 @@ This section assumes the use of TLS certificates and the `https` protocol.
 > _Note:_ For local development with, we use the
 > Auth0 Tenant `dev-fiftyone` and the Auth0 Application `local-dev`.
 > The `local-dev` app contains the setting Allowed Callback URLs
-> (aka Redirect URLs) with [https://local.fiftyone.ai](https://local.fiftyone.ai).
+> (aka Redirect URLs) with
+> [https://local.fiftyone.ai](https://local.fiftyone.ai)
+> .
 > In `skaffold.yaml`, in both `appSettings.env` and `teamsAppSettings.env`,
 > either omit `APP_USE_HTTPS=false` or set `APP_USE_HTTPS=true`
 > for the app to set the Redirect URL's protocol to `https`.
@@ -536,10 +541,11 @@ With the port forward running,
 
 > _Note:_ For local development, we use the Auth0 Tenant `dev-fiftyone` and
 > the Auth0 Application `local-dev` contains the setting Allowed Callback URLs
-> (aka Redirect URLs) with [http://localhost:3000](http://localhost:3000).
+> (aka Redirect URLs) with
+> [http://localhost:3000](http://localhost:3000).
 > In `skaffold.yaml` we set `APP_USE_HTTPS=false`
 > to prohibit the app from setting the Redirect URL protocol to `https`.
 > Must be set in both `appSettings.env` and `teamsAppSettings.env`.
 > Without this setting, the app code makes the callback URL
-> [https://localhost:3000](https://localhost:3000) and Auth0
-> throws a Callback URL mismatch error.
+> [https://localhost:3000](https://localhost:3000)
+> and Auth0 throws a Callback URL mismatch error.
diff --git a/Makefile b/Makefile
@@ -27,6 +27,9 @@ asdf:  ## Update plugins, add plugins, install plugins, set local, reshim
 	@echo "Reshimming.."
 	@asdf reshim
 
+auth:
+	gcloud auth application-default login --project computer-vision-team
+	gcloud auth configure-docker us-central1-docker.pkg.dev
 
 hooks:  ## Install git hooks (pre-commit)
 	@pre-commit install
@@ -39,7 +42,7 @@ hooks:  ## Install git hooks (pre-commit)
 pre-commit:  ## Run pre-commit against all files
 	@pre-commit run -a
 
-start:   ## Run minikube with ingress and gcp-auth
+start:  ## Run minikube with ingress and gcp-auth
 	# to persist mongodb data, we may want to start minikube with a volume mount
 	# minikube start --mount=true \
 	#   --mount-string=/var/tmp/mongodb_data:/tmp/hostpath-provisioner/fiftyone-teams/mongodb
@@ -89,12 +92,40 @@ port-forward-api:  ## port forward to service `teams-api` on the host port 8000
 port-forward-mongo:  ## port forward to service `mongodb` on the host port 27017
 	kubectl port-forward --namespace fiftyone-teams svc/mongodb 27017:27017 --context minikube
 
+tunnel:  ## run minikube tunnel to access the k8s ingress via localhost ()
+	minikube tunnel
+
 helm-repos:  ## add helm repos for the project
 	helm repo add bitnami https://charts.bitnami.com/bitnami
 	helm repo add jetstack https://charts.jetstack.io
 
-tunnel:  ## run minikube tunnel to access the k8s ingress via localhost ()
-	minikube tunnel
+clean: clean-unit-compose clean-unit-helm clean-integration-compose clean-integration-helm  ## delete all test output and reports
+
+clean-integration-compose:  ## delete docker compose integration test output and reports
+	rm -rf tests/integration/compose/test_output_internal || true
+	rm -rf tests/integration/compose/test_output_legacy || true
+	rm tests/integration/compose/test_output.log || true
+
+clean-integration-helm:  ## delete helm integration test output and reports
+	rm -rf tests/integration/helm/test_output_internal || true
+	rm -rf tests/integration/helm/test_output_legacy || true
+	rm tests/integration/helm/test_output.log || true
+
+clean-unit-compose:  ## delete docker compose unit test output and reports
+	rm -rf tests/unit/compose/test_output || true
+	rm tests/unit/compose/test_output.log || true
+
+clean-unit-helm:  ## delete helm unit test output and reports
+	rm -rf tests/unit/helm/test_output || true
+	rm tests/unit/helm/test_output.log || true
+
+dependencies-integration-compose:  ## create a (temporary) directory for mongodb container
+	mkdir -p /tmp/mongodb
+
+login:  ## Docker login to Google Artifact Registry (for accessing internal gcr.io container images)
+	gcloud auth print-access-token | \
+	  docker login -u oauth2accesstoken \
+	    --password-stdin https://us-central1-docker.pkg.dev
 
 test-unit-compose:  ## run go test on the tests/unit/compose directory
 	@cd tests/unit/compose; \
@@ -106,17 +137,39 @@ test-unit-helm:  ## run go test on the tests/unit/helm directory
 
 test-unit-compose-interleaved: install-terratest-log-parser  ## run go test on the tests/unit/compose directory and run the terratest_log_parser for reports
 	@cd tests/unit/compose; \
-	rm -rf test_reports; \
-	mkdir test_reports; \
+	rm -rf test_output/*; \
 	go test -count=1 -timeout=10m -v -tags unit | tee test_output.log; \
 	${ASDF}/packages/bin/terratest_log_parser -testlog test_output.log -outputdir test_output
 
 test-unit-helm-interleaved: install-terratest-log-parser  ## run go test on the tests/unit/helm directory and run the terratest_log_parser for reports
 	@cd tests/unit/helm; \
-	rm -rf test_reports; \
-	mkdir test_reports; \
+	rm -rf test_output/*; \
 	go test -count=1 -timeout=10m -v -tags unit | tee test_output.log; \
 	${ASDF}/packages/bin/terratest_log_parser -testlog test_output.log -outputdir test_output
 
+test-integration-compose: test-integration-compose-internal test-integration-compose-legacy ## run go test on the tests/integration/compose directory for both internal and legacy auth modes
+
+test-integration-compose-internal: dependencies-integration-compose ## run go test on the tests/integration/compose directory for internal auth mode
+	@cd tests/integration/compose; \
+	go test -count=1 -timeout=10m -v -tags integrationComposeInternalAuth
+
+test-integration-compose-legacy: dependencies-integration-compose ## run go test on the tests/integration/compose directory for legacy auth mode
+	@cd tests/integration/compose; \
+	go test -count=1 -timeout=10m -v -tags integrationComposeLegacyAuth
+
+test-integration-compose-interleaved:  test-integration-compose-interleaved-internal test-integration-compose-interleaved-legacy  ## run go test on the tests/integration/compose directory and run the terratest_log_parser for reports
+
+test-integration-compose-interleaved-internal: install-terratest-log-parser dependencies-integration-compose clean-integration-compose ## run go test on the tests/integration/compose directory for internal auth mode and run the terratest_log_parser for reports
+	@cd tests/integration/compose; \
+	rm -rf test_output_internal/*; \
+	go test -count=1 -timeout=10m -v -tags integrationComposeInternalAuth | tee test_output_internal.log; \
+	${ASDF}/packages/bin/terratest_log_parser -testlog test_output_internal.log -outputdir test_output_internal
+
+test-integration-compose-interleaved-legacy: install-terratest-log-parser dependencies-integration-compose clean-integration-compose ## run go test on the tests/integration/compose directory for legacy auth mode and run the terratest_log_parser for reports
+	@cd tests/integration/compose; \
+	rm -rf test_output_legacy/*; \
+	go test -count=1 -timeout=10m -v -tags integrationComposeLegacyAuth | tee test_output_legacy.log; \
+	${ASDF}/packages/bin/terratest_log_parser -testlog test_output_legacy.log -outputdir test_output_legacy
+
 install-terratest-log-parser:  ## install terratest_log_parser
 	go install github.com/gruntwork-io/terratest/cmd/terratest_log_parser@latest