Circleci project setup

.circleci/.gitignore

@@ -0,0 +1 @@
+process.yml

.circleci/config.yml

@@ -0,0 +1,118 @@
+# Use the latest 2.1 version of CircleCI pipeline process engine.
+# See: https://circleci.com/docs/configuration-reference
+version: 2.1
+
+<<<<<<< HEAD
+commands:
+  apt-packages:
+    description: Performs a common install in each job
+    steps:
+      - run:
+          name: "common apt packages"
+          command: |
+            sudo apt-get update
+            sudo apt-get -y install build-essential \
+              curl \
+              libpoppler-dev \
+              libpoppler-glib-dev \
+              poppler-utils \
+              software-properties-common \
+              tree \
+              wget
+  install-linter:
+    description: Installs the golangci linter
+    parameters:
+      linter-version: 
+        default: 1.63.4
+        type: string
+    steps:
+      - run:
+          name: "install golangci-lint"
+          command: |
+            curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v<< parameters.linter-version >>
+  install-go-utilities:
+    description: Installs the golang utilities
+    steps:
+      - run: 
+          name: "install jsonnet"
+          command: |
+            go install golang.org/x/tools/cmd/goimports@latest
+            go install golang.org/x/tools/cmd/stringer@latest
+            go install github.com/google/go-jsonnet/cmd/jsonnet@latest
+            go install github.com/google/go-jsonnet/cmd/jsonnet-lint@latest
+            go install github.com/google/go-jsonnet/cmd/jsonnetfmt@latest
+      - run: 
+          name: "install sqlc"
+          command: |
+            wget https://downloads.sqlc.dev/sqlc_1.27.0_linux_amd64.tar.gz
+            tar xvzf sqlc_1.27.0_linux_amd64.tar.gz
+            chmod 755 sqlc
+            sudo mv sqlc /bin/sqlc
+  prep-the-build:
+    description: Generates files required for the build
+    steps:
+      - run:
+          name: "generate sqlc models"
+          command: make generate
+jobs:
+  lint:
+    docker:
+      - image: cimg/go:1.23.3
+    steps:
+      - checkout
+      - apt-packages
+      - install-go-utilities
+      - install-linter
+      - prep-the-build
+      - run: 
+          name: "find the lint"
+          command: golangci-lint run -v
+          no_output_timeout: 5m
+
+  setup-and-build:
+    docker:
+      - image: cimg/go:1.23.3
+    steps:
+      # Checkout the code as the first step.
+      - checkout
+      - apt-packages
+      - install-go-utilities
+      - run: make build
+
+=======
+# Define a job to be invoked later in a workflow.
+# See: https://circleci.com/docs/jobs-steps/#jobs-overview & https://circleci.com/docs/configuration-reference/#jobs
+jobs:
+  say-hello:
+    # Specify the execution environment. You can specify an image from Docker Hub or use one of our convenience images from CircleCI's Developer Hub.
+    # See: https://circleci.com/docs/executor-intro/ & https://circleci.com/docs/configuration-reference/#executor-job
+    docker:
+      # Specify the version you desire here
+      # See: https://circleci.com/developer/images/image/cimg/base
+      - image: cimg/base:current
+
+    # Add steps to the job
+    # See: https://circleci.com/docs/jobs-steps/#steps-overview & https://circleci.com/docs/configuration-reference/#steps
+    steps:
+      # Checkout the code as the first step.
+      - checkout
+      - run:
+          name: "Say hello"
+          command: "echo Hello, World!"
+>>>>>>> db97d998cf4d8b6c042529d8c655c3ea8a750117
+
+# Orchestrate jobs using workflows
+# See: https://circleci.com/docs/workflows/ & https://circleci.com/docs/configuration-reference/#workflows
+workflows:
+<<<<<<< HEAD
+  build-and-deploy:
+    jobs:
+      - lint
+      - setup-and-build:
+          requires: [ lint ]
+=======
+  say-hello-workflow: # This is the name of the workflow, feel free to change it to better match your workflow.
+    # Inside the workflow, you define the jobs you want to run.
+    jobs:
+      - say-hello
+>>>>>>> db97d998cf4d8b6c042529d8c655c3ea8a750117

.golangci.yml

@@ -0,0 +1,80 @@
+linters:
+  enable-all: true
+  # Some things here we may want to leave disabled.
+  # Some we may want to enable, and then pay down as debt.
+  # Anything included here represented a more invasive set of changes to 
+  # satisfy the linter than was desired during initial cleanup.
+  disable:
+    - canonicalheader
+    - contextcheck
+    - depguard
+    - err113
+    - errchkjson
+    - exhaustruct
+    - exportloopref
+    - forbidigo
+    - gochecknoglobals
+    - gocritic
+    - intrange
+    - ireturn
+    - musttag
+    - nilerr
+    - noctx
+    - paralleltest
+    - perfsprint
+    - tagalign
+    - tagliatelle
+    - tenv
+    - testifylint
+    - unconvert
+    - unparam
+    - varnamelen
+    - wastedassign
+  depguard:
+    # Rules to apply.
+    #
+    # Variables:
+    # - File Variables
+    #   you can still use and exclamation mark ! in front of a variable to say not to use it.
+    #   Example !$test will match any file that is not a go test file.
+    #
+    #   `$all` - matches all go files
+    #   `$test` - matches all go test files
+    #
+    # - Package Variables
+    #
+    #  `$gostd` - matches all of go's standard library (Pulled from `GOROOT`)
+    #
+    # Default: Only allow $gostd in all files.
+    rules:
+      main:
+        list-mode: lax
+        # List of file globs that will match this list of settings to compare against.
+        # Default: $all
+        files:
+          - $all
+        # List of allowed packages.
+        allow:
+          - $gostd
+          - github.com/GSA-TTS/jemison/config
+        # Packages that are not allowed where the value is a suggestion.
+        deny:
+          - pkg: "github.com/sirupsen/logrus"
+            desc: not allowed
+          - pkg: "github.com/pkg/errors"
+            desc: Should be replaced by standard lib errors package
+
+run:
+  timeout: 5m
+  issues-exit-code: 2
+  concurrency: 4
+  allow-parallel-runners: true
+
+output:
+  formats:
+    - format: json
+      path: stderr
+    - format: checkstyle
+      path: report.xml
+    - format: colored-line-number
+  show-stats: true

Dockerfile.build

@@ -2,4 +2,5 @@ FROM jemison/dev
 
 WORKDIR /app
 
-ENTRYPOINT ["make", "build"]
+ENTRYPOINT ["make"]
+CMD ["build"]

Dockerfile.dev

@@ -64,7 +64,7 @@ RUN cp /root/go/bin/stringer /usr/local/go/bin/stringer
 
 WORKDIR /golanglint
 # binary will be $(go env GOPATH)/bin/golangci-lint
-RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /usr/local/go/bin v1.62.0
+RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /usr/local/go/bin v1.63.4
 
 
 ENTRYPOINT ["tree /"]

Makefile

@@ -9,21 +9,19 @@ clean:
 	rm -f cmd/*/service.exe
 
 .PHONY: generate
-generate:
+generate: config
 	cd internal/postgres ; make generate
-	# cd internal/postgres/search_db ; make generate
 
 .PHONY: config
-config:
+config: clean
 	cd config ; make all || exit 1
 
 docker: 
 	docker build -t jemison/dev -f Dockerfile.dev .
 	docker build -t jemison/build -f Dockerfile.build .
 
 .PHONY: build
-# lint
-build: clean config generate 
+build: generate 
 	echo "build migrate"
 	cd cmd/migrate ; make build
 	echo "build admin"
@@ -42,8 +40,14 @@ build: clean config generate
 	# cd cmd/validate ; make build
 	echo "build walk"
 	cd cmd/walk ; make build
-	echo "copy assets"
-	cd assets ; rm -rf static/assets ; unzip -qq static.zip
+
+.PHONY: lint
+lint: generate
+	golangci-lint run -v
+
+.PHONY: containerlint
+containerlint:
+	docker run -v ${PWD}:/app -t jemison/build lint
 
 .PHONY: up
 up: build
@@ -111,7 +115,3 @@ terraform: delete_all
 docker_full_clean:
 	-docker stop $(docker ps -a -q)
 	-docker rm $(docker ps -a -q)
-
-.PHONY: lint
-lint:
-	-golangci-lint run -v

cmd/admin/main.go

@@ -9,18 +9,18 @@ import (
 	"github.com/GSA-TTS/jemison/internal/env"
 	"github.com/GSA-TTS/jemison/internal/queueing"
 	"github.com/gin-gonic/gin"
-
 	"go.uber.org/zap"
 )
 
 var ThisServiceName = "admin"
+
 var ChQSHP = make(chan queueing.QSHP)
 
 type FetchRequestInput struct {
 	Scheme string `json:"scheme" maxLength:"10" doc:"Resource scheme"`
 	Host   string `json:"host" maxLength:"500" doc:"Host of resource"`
 	Path   string `json:"path" maxLength:"1500" doc:"Path to resource"`
-	ApiKey string `json:"api-key"`
+	APIKey string `json:"api-key"`
 }
 
 // https://dev.to/kashifsoofi/rest-api-with-go-chi-and-inmemory-store-43ag
@@ -29,9 +29,12 @@ func FetchRequestHandler(c *gin.Context) {
 	if err := c.BindJSON(&fri); err != nil {
 		return
 	}
-	if fri.ApiKey == os.Getenv("API_KEY") || true {
-		zap.L().Debug("fetch enqueue", zap.String("host", fri.Host), zap.String("path", fri.Path))
-		//queueing.InsertFetch(fri.Scheme, fri.Host, fri.Path)
+
+	if fri.APIKey == os.Getenv("API_KEY") || true {
+		zap.L().Debug("fetch enqueue",
+			zap.String("host", fri.Host),
+			zap.String("path", fri.Path))
+
 		ChQSHP <- queueing.QSHP{
 			Queue:  "fetch",
 			Scheme: fri.Scheme,
@@ -46,19 +49,22 @@ func FetchRequestHandler(c *gin.Context) {
 
 func EntreeRequestHandler(c *gin.Context) {
 	var fri FetchRequestInput
+
 	full := c.Param("fullorone")
 	hallPass := c.Param("hallpass")
 
 	if err := c.BindJSON(&fri); err != nil {
 		return
 	}
-	if fri.ApiKey == os.Getenv("API_KEY") || true {
+
+	if fri.APIKey == os.Getenv("API_KEY") || true {
 		hallPassB := false
 		fullB := false
 
 		if hallPass == "pass" {
 			hallPassB = true
 		}
+
 		if full == "full" {
 			fullB = true
 		}
@@ -89,8 +95,8 @@ func PackRequestHandler(c *gin.Context) {
 	if err := c.BindJSON(&fri); err != nil {
 		return
 	}
-	if fri.ApiKey == os.Getenv("API_KEY") || true {
 
+	if fri.APIKey == os.Getenv("API_KEY") || true {
 		zap.L().Debug("pack enqueue",
 			zap.String("host", fri.Host))
 
@@ -118,10 +124,10 @@ func main() {
 		v1.PUT("/fetch", FetchRequestHandler)
 		v1.PUT("/entree/:fullorone/:hallpass", EntreeRequestHandler)
 		v1.PUT("/pack", PackRequestHandler)
-		// v1.GET("/jobs", JobCountHandler)
 	}
 
 	log.Println("environment initialized")
+
 	go queueing.Enqueue(ChQSHP)
 
 	// // Init a cache for the workers
@@ -130,6 +136,9 @@ func main() {
 	zap.L().Info("listening to the music of the spheres",
 		zap.String("port", env.Env.Port))
 	// Local and Cloud should both get this from the environment.
-	http.ListenAndServe(":"+env.Env.Port, engine)
-
+	//nolint:gosec
+	err := http.ListenAndServe(":"+env.Env.Port, engine)
+	if err != nil {
+		zap.Error(err)
+	}
 }

cmd/admin/queues.go

@@ -1,28 +1,9 @@
 package main
 
 import (
-	"os"
-
-	common "github.com/GSA-TTS/jemison/internal/common"
 	"github.com/GSA-TTS/jemison/internal/queueing"
-	"github.com/jackc/pgx/v5"
-	"github.com/riverqueue/river"
-	"github.com/riverqueue/river/riverdriver/riverpgxv5"
-	"go.uber.org/zap"
 )
 
-// GLOBAL TO THE APP
-var insertClient *river.Client[pgx.Tx]
-
 func InitializeQueues() {
 	queueing.InitializeRiverQueues()
-
-	// Insert-only client
-	_, pool, _ := common.CommonQueueInit()
-	ic, err := river.NewClient(riverpgxv5.New(pool), &river.Config{})
-	if err != nil {
-		zap.L().Error("could not establish insert-only client")
-		os.Exit(1)
-	}
-	insertClient = ic
 }