Skip to content

Commit

Permalink
Merge pull request #348 from hearchco/as/feat/dynamodb
Browse files Browse the repository at this point in the history
feat(cache): dynamodb
  • Loading branch information
aleksasiriski authored Jul 10, 2024
2 parents c7178cf + da42afe commit c3faa03
Show file tree
Hide file tree
Showing 13 changed files with 503 additions and 6 deletions.
59 changes: 58 additions & 1 deletion .github/workflows/ci_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,15 @@ jobs:
--health-interval 10s
--health-timeout 5s
--health-retries 5
dynamodb:
image: amazon/dynamodb-local
ports:
- 8000:8000
options: >-
--health-cmd "curl -s http://localhost:8000/shell/ -o /dev/null"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Checkout repository
Expand All @@ -62,7 +71,7 @@ jobs:
- name: Generate go code from go:generate comments
run: make install

- name: Test units (w/o engines & redis)
- name: Test units (w/o engines & cache)
run: make test

- name: Test units (only redis)
Expand All @@ -71,6 +80,54 @@ jobs:
REDIS_HOST: localhost
REDIS_PORT: "6379"

- name: Install AWS CLI
run: |
sudo apt-get update
sudo apt-get install -y awscli
- name: Configure AWS CLI for Local DynamoDB
run: |
aws configure set aws_access_key_id dummy
aws configure set aws_secret_access_key dummy
aws configure set region eu-central-1
- name: Setup DynamoDB table KV
env:
AWS_REGION: "eu-central-1"
AWS_ACCESS_KEY_ID: "hearchco"
AWS_SECRET_ACCESS_KEY: "hearchco"
DYNAMODB_TABLE: "hearchco"
DYNAMODB_ENDPOINT: "http://localhost:8000"
run: >-
aws dynamodb create-table
--table-name $DYNAMODB_TABLE
--attribute-definitions AttributeName=Key,AttributeType=S
--key-schema AttributeName=Key,KeyType=HASH
--billing-mode PAY_PER_REQUEST
--endpoint-url $DYNAMODB_ENDPOINT
- name: Setup DynamoDB table TTL
env:
AWS_REGION: "eu-central-1"
AWS_ACCESS_KEY_ID: "hearchco"
AWS_SECRET_ACCESS_KEY: "hearchco"
DYNAMODB_TABLE: "hearchco"
DYNAMODB_ENDPOINT: "http://localhost:8000"
run: >-
aws dynamodb update-time-to-live
--table-name $DYNAMODB_TABLE
--time-to-live-specification "Enabled=true, AttributeName=TTL"
--endpoint-url $DYNAMODB_ENDPOINT
- name: Test units (only dynamodb)
run: make test-dynamodb
env:
AWS_REGION: "eu-central-1"
AWS_ACCESS_KEY_ID: "hearchco"
AWS_SECRET_ACCESS_KEY: "hearchco"
DYNAMODB_TABLE: "hearchco"
DYNAMODB_ENDPOINT: "http://localhost:8000"

test-engines:
runs-on: ubuntu-latest
permissions:
Expand Down
15 changes: 12 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,24 @@ test:
sh ./scripts/test.sh
test-engines:
sh ./scripts/test-engines.sh

test-redis:
sh ./scripts/test-redis.sh
test-redis-podman:
sh ./scripts/test-redis-podman.sh
test-redis-docker:
sh ./scripts/test-redis-docker.sh
test-all: test test-redis test-engines
test-all-podman: test test-redis-podman test-engines
test-all-docker: test test-redis-docker test-engines

test-dynamodb:
sh ./scripts/test-dynamodb.sh
test-dynamodb-podman:
sh ./scripts/test-dynamodb-podman.sh
test-dynamodb-docker:
sh ./scripts/test-dynamodb-docker.sh

test-all: test test-redis test-dynamodb test-engines
test-all-podman: test test-redis-podman test-dynamodb-podman test-engines
test-all-docker: test test-redis-docker test-dynamodb-docker test-engines

update:
go get -u ./...
Expand Down
20 changes: 19 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,30 @@ require (
golang.org/x/tools v0.23.0
)

require github.com/klauspost/compress v1.17.9
require (
github.com/aws/aws-sdk-go-v2 v1.30.1
github.com/aws/aws-sdk-go-v2/config v1.27.24
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.1
github.com/klauspost/compress v1.17.9
)

require (
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/antchfx/htmlquery v1.3.2 // indirect
github.com/antchfx/xmlquery v1.4.1 // indirect
github.com/antchfx/xpath v1.3.1 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.24 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.14 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 // indirect
github.com/aws/smithy-go v1.20.3 // indirect
github.com/bits-and-blooms/bitset v1.13.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
Expand All @@ -43,6 +60,7 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/pprof v0.0.0-20240625030939-27f56978b8b0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/kennygrant/sanitize v1.2.4 // indirect
github.com/knadh/koanf/maps v0.1.1 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
Expand Down
36 changes: 36 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,36 @@ github.com/antchfx/xpath v1.3.1 h1:PNbFuUqHwWl0xRjvUPjJ95Agbmdj2uzzIwmQKgu4oCk=
github.com/antchfx/xpath v1.3.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI=
github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A=
github.com/aws/aws-sdk-go-v2 v1.30.1 h1:4y/5Dvfrhd1MxRDD77SrfsDaj8kUkkljU7XE83NPV+o=
github.com/aws/aws-sdk-go-v2 v1.30.1/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
github.com/aws/aws-sdk-go-v2/config v1.27.24 h1:NM9XicZ5o1CBU/MZaHwFtimRpWx9ohAUAqkG6AqSqPo=
github.com/aws/aws-sdk-go-v2/config v1.27.24/go.mod h1:aXzi6QJTuQRVVusAO8/NxpdTeTyr/wRcybdDtfUwJSs=
github.com/aws/aws-sdk-go-v2/credentials v1.17.24 h1:YclAsrnb1/GTQNt2nzv+756Iw4mF8AOzcDfweWwwm/M=
github.com/aws/aws-sdk-go-v2/credentials v1.17.24/go.mod h1:Hld7tmnAkoBQdTMNYZGzztzKRdA4fCdn9L83LOoigac=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 h1:Aznqksmd6Rfv2HQN9cpqIV/lQRMaIpJkLLaJ1ZI76no=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9/go.mod h1:WQr3MY7AxGNxaqAtsDWn+fBxmd4XvLkzeqQ8P1VM0/w=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 h1:5SAoZ4jYpGH4721ZNoS1znQrhOfZinOhc4XuTXx/nVc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13/go.mod h1:+rdA6ZLpaSeM7tSg/B0IEDinCIBJGmW8rKDFkYpP04g=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 h1:WIijqeaAO7TYFLbhsZmi2rgLEAtWOC1LhxCAVTJlSKw=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13/go.mod h1:i+kbfa76PQbWw/ULoWnp51EYVWH4ENln76fLQE3lXT8=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.1 h1:Szwz1vpZkvfhFMJ0X5uUECgHeUmPAxk1UGqAVs/pARw=
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.1/go.mod h1:b4wouGyJlzkr2HAvPrDGgYNp1EtmlXOkzhEOvl0c0FQ=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI=
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.14 h1:X1J0Kd17n1PeXeoArNXlvnKewCyMvhVQh7iNMy6oi3s=
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.14/go.mod h1:VYMN7l7dxp6xtQRjqIau6d7QAbmPG+yJ75GtCy70f18=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 h1:I9zMeF107l0rJrpnHpjEiiTSCKYAIw8mALiXcPsGBiA=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15/go.mod h1:9xWJ3Q/S6Ojusz1UIkfycgD1mGirJfLLKqq3LPT7WN8=
github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 h1:p1GahKIjyMDZtiKoIn0/jAj/TkMzfzndDv5+zi2Mhgc=
github.com/aws/aws-sdk-go-v2/service/sso v1.22.1/go.mod h1:/vWdhoIoYA5hYoPZ6fm7Sv4d8701PiG5VKe8/pPJL60=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 h1:ORnrOK0C4WmYV/uYt3koHEWBLYsRDwk2Np+eEoyV4Z0=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2/go.mod h1:xyFHA4zGxgYkdD73VeezHt3vSKEG9EmFnGwoKlP00u4=
github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 h1:+woJ607dllHJQtsnJLi52ycuqHMwlW+Wqm2Ppsfp4nQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.30.1/go.mod h1:jiNR3JqT15Dm+QWq2SRgh0x0bCNSRP2L25+CqPNpJlQ=
github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/awslabs/aws-lambda-go-api-proxy v0.16.2 h1:CJyGEyO1CIwOnXTU40urf0mchf6t3voxpvUDikOU9LY=
github.com/awslabs/aws-lambda-go-api-proxy v0.16.2/go.mod h1:vxxjwBHe/KbgFeNlAP/Tvp4SsVRL3WQamcWRxqVh0z0=
github.com/bits-and-blooms/bitset v1.2.2-0.20220111210104-dfa3e347c392/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
Expand Down Expand Up @@ -113,6 +143,10 @@ github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSo
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw=
github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
Expand Down Expand Up @@ -308,6 +342,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Expand Down
23 changes: 23 additions & 0 deletions scripts/test-dynamodb-docker.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/usr/bin/env bash

export AWS_REGION=hearchco-test-1
export AWS_ACCESS_KEY_ID=hearchco
export AWS_SECRET_ACCESS_KEY=hearchco
export DYNAMODB_TABLE=hearchco_test
export DYNAMODB_ENDPOINT=http://localhost:8000

docker run --rm --name hearchco-dynamodb -d -p 8000:8000 docker.io/amazon/dynamodb-local && \
sleep 5 && \
aws dynamodb create-table \
--table-name $DYNAMODB_TABLE \
--attribute-definitions AttributeName=Key,AttributeType=S \
--key-schema AttributeName=Key,KeyType=HASH \
--billing-mode PAY_PER_REQUEST \
--endpoint-url $DYNAMODB_ENDPOINT && \
aws dynamodb update-time-to-live \
--table-name $DYNAMODB_TABLE \
--time-to-live-specification "Enabled=true, AttributeName=TTL" \
--endpoint-url $DYNAMODB_ENDPOINT && \
go test $(go list ./... | grep /dynamodb) -count=1

docker stop hearchco-dynamodb
23 changes: 23 additions & 0 deletions scripts/test-dynamodb-podman.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/usr/bin/env bash

export AWS_REGION=hearchco-test-1
export AWS_ACCESS_KEY_ID=hearchco
export AWS_SECRET_ACCESS_KEY=hearchco
export DYNAMODB_TABLE=hearchco_test
export DYNAMODB_ENDPOINT=http://localhost:8000

podman run --rm --name hearchco-dynamodb -d -p 8000:8000 docker.io/amazon/dynamodb-local && \
sleep 5 && \
aws dynamodb create-table \
--table-name $DYNAMODB_TABLE \
--attribute-definitions AttributeName=Key,AttributeType=S \
--key-schema AttributeName=Key,KeyType=HASH \
--billing-mode PAY_PER_REQUEST \
--endpoint-url $DYNAMODB_ENDPOINT && \
aws dynamodb update-time-to-live \
--table-name $DYNAMODB_TABLE \
--time-to-live-specification "Enabled=true, AttributeName=TTL" \
--endpoint-url $DYNAMODB_ENDPOINT && \
go test $(go list ./... | grep /dynamodb) -count=1

podman stop hearchco-dynamodb
2 changes: 2 additions & 0 deletions scripts/test-dynamodb.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
#!/usr/bin/env bash
go test $(go list ./... | grep /dynamodb) -count=1
2 changes: 1 addition & 1 deletion scripts/test.sh
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
#!/usr/bin/env bash
go test $(go list ./... | grep -v /engines/ | grep -v /redis)
go test $(go list ./... | grep -v /engines/ | grep -v /redis | grep -v /dynamodb)
6 changes: 6 additions & 0 deletions src/cache/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/rs/zerolog/log"

"github.com/hearchco/agent/src/cache/dynamodb"
"github.com/hearchco/agent/src/cache/nocache"
"github.com/hearchco/agent/src/cache/redis"
"github.com/hearchco/agent/src/config"
Expand All @@ -25,6 +26,11 @@ func New(ctx context.Context, cacheConf config.Cache) (DB, error) {
if err != nil {
err = fmt.Errorf("failed creating a redis cache: %w", err)
}
case "dynamodb":
drv, err = dynamodb.New(ctx, cacheConf.KeyPrefix, cacheConf.DynamoDB)
if err != nil {
err = fmt.Errorf("failed creating a dynamodb cache: %w", err)
}
default:
drv, err = nocache.New()
if err != nil {
Expand Down
Loading

0 comments on commit c3faa03

Please sign in to comment.