From a3f44f1797e4e0f99aa96081d605705a4735d1a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Herna=CC=81ndez?= Date: Mon, 1 Apr 2024 17:34:16 +0200 Subject: [PATCH 1/9] Clean the repository --- .editorconfig | 10 - .github/workflows/ci.yml | 188 --------- .github/workflows/clean.yml | 59 --- .gitignore | 27 -- .scalafmt.conf | 32 -- CODE-OF-CONDUCT.md | 44 -- LICENSE | 201 --------- README.md | 382 ------------------ RELEASE.md | 26 -- build.sbt | 50 --- .../consumer/grpc/PubsubGoogleConsumer.scala | 129 ------ .../grpc/PubsubGoogleConsumerConfig.scala | 40 -- .../pubsub/consumer/grpc/internal/Model.scala | 23 -- .../grpc/internal/PubsubSubscriber.scala | 168 -------- .../producer/grpc/GooglePubsubProducer.scala | 34 -- .../producer/grpc/PubsubProducerConfig.scala | 33 -- .../grpc/internal/DefaultPublisher.scala | 53 --- .../grpc/internal/FutureInterop.scala | 74 ---- .../grpc/internal/PubsubPublisher.scala | 71 ---- .../permutive/pubsub/GrpcPingPongSpec.scala | 180 --------- .../com/permutive/pubsub/PubSubSpec.scala | 204 ---------- .../com/permutive/pubsub/ValueHolder.scala | 32 -- .../pubsub/consumer/grpc/SimpleDriver.scala | 47 --- .../producer/grpc/PubsubProducerExample.scala | 51 --- .../consumer/http/PubsubHttpConsumer.scala | 157 ------- .../http/PubsubHttpConsumerConfig.scala | 62 --- .../pubsub/consumer/http/PubsubMessage.scala | 32 -- .../http/internal/HttpPubsubReader.scala | 201 --------- .../pubsub/consumer/http/internal/Model.scala | 95 ----- .../consumer/http/internal/PubsubReader.scala | 44 -- .../http/internal/PubsubSubscriber.scala | 98 ----- .../http/crypto/GoogleAccountParser.scala | 67 --- .../pubsub/http/oauth/AccessToken.scala | 29 -- .../http/oauth/CachedTokenProvider.scala | 53 --- .../http/oauth/DefaultTokenProvider.scala | 68 ---- .../pubsub/http/oauth/GoogleOAuth.scala | 95 ----- .../http/oauth/InstanceMetadataOAuth.scala | 66 --- .../pubsub/http/oauth/NoopOAuth.scala | 30 -- .../permutive/pubsub/http/oauth/OAuth.scala | 40 -- .../pubsub/http/oauth/RequestAuthorizer.scala | 44 -- .../pubsub/http/oauth/TokenProvider.scala | 35 -- .../permutive/pubsub/http/util/RefCache.scala | 90 ----- .../pubsub/http/util/RefreshableEffect.scala | 121 ------ .../http/BatchingHttpProducerConfig.scala | 27 -- .../http/BatchingHttpPubsubProducer.scala | 62 --- .../producer/http/HttpPubsubProducer.scala | 55 --- .../http/PubsubHttpProducerConfig.scala | 52 --- .../http/internal/BatchingHttpPublisher.scala | 102 ----- .../http/internal/DefaultHttpPublisher.scala | 205 ---------- .../src/test/resources/logback.xml | 21 - .../permutive/pubsub/HttpPingPongSpec.scala | 221 ---------- .../com/permutive/pubsub/PubSubSpec.scala | 198 --------- .../pubsub/consumer/http/Example.scala | 70 ---- .../com/permutive/pubsub/http/package.scala | 19 - .../producer/http/ExampleBatching.scala | 105 ----- .../producer/http/ExampleEmulator.scala | 75 ---- .../pubsub/producer/http/ExampleGoogle.scala | 74 ---- .../pubsub/consumer/ConsumerRecord.scala | 53 --- .../com/permutive/pubsub/consumer/Model.scala | 31 -- .../consumer/decoder/MessageDecoder.scala | 34 -- .../pubsub/producer/AsyncPubsubProducer.scala | 44 -- .../com/permutive/pubsub/producer/Model.scala | 44 -- .../pubsub/producer/PubsubProducer.scala | 32 -- .../producer/encoder/MessageEncoder.scala | 25 -- project/Dependencies.scala | 79 ---- project/build.properties | 1 - project/plugins.sbt | 2 - 67 files changed, 5216 deletions(-) delete mode 100644 .editorconfig delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/clean.yml delete mode 100644 .gitignore delete mode 100644 .scalafmt.conf delete mode 100644 CODE-OF-CONDUCT.md delete mode 100644 LICENSE delete mode 100644 README.md delete mode 100644 RELEASE.md delete mode 100644 build.sbt delete mode 100644 fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/PubsubGoogleConsumer.scala delete mode 100644 fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/PubsubGoogleConsumerConfig.scala delete mode 100644 fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/internal/Model.scala delete mode 100644 fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/internal/PubsubSubscriber.scala delete mode 100644 fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/GooglePubsubProducer.scala delete mode 100644 fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/PubsubProducerConfig.scala delete mode 100644 fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/internal/DefaultPublisher.scala delete mode 100644 fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/internal/FutureInterop.scala delete mode 100644 fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/internal/PubsubPublisher.scala delete mode 100644 fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/GrpcPingPongSpec.scala delete mode 100644 fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/PubSubSpec.scala delete mode 100644 fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/ValueHolder.scala delete mode 100644 fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/consumer/grpc/SimpleDriver.scala delete mode 100644 fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/producer/grpc/PubsubProducerExample.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubHttpConsumer.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubHttpConsumerConfig.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubMessage.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/HttpPubsubReader.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/Model.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/PubsubReader.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/PubsubSubscriber.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/crypto/GoogleAccountParser.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/AccessToken.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/CachedTokenProvider.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/DefaultTokenProvider.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/GoogleOAuth.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/InstanceMetadataOAuth.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/NoopOAuth.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/OAuth.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/RequestAuthorizer.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/TokenProvider.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/util/RefCache.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/util/RefreshableEffect.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/BatchingHttpProducerConfig.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/BatchingHttpPubsubProducer.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/HttpPubsubProducer.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/PubsubHttpProducerConfig.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/internal/BatchingHttpPublisher.scala delete mode 100644 fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/internal/DefaultHttpPublisher.scala delete mode 100644 fs2-google-pubsub-http/src/test/resources/logback.xml delete mode 100644 fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/HttpPingPongSpec.scala delete mode 100644 fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/PubSubSpec.scala delete mode 100644 fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/consumer/http/Example.scala delete mode 100644 fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/http/package.scala delete mode 100644 fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/producer/http/ExampleBatching.scala delete mode 100644 fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/producer/http/ExampleEmulator.scala delete mode 100644 fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/producer/http/ExampleGoogle.scala delete mode 100644 fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/ConsumerRecord.scala delete mode 100644 fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/Model.scala delete mode 100644 fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/decoder/MessageDecoder.scala delete mode 100644 fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/AsyncPubsubProducer.scala delete mode 100644 fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/Model.scala delete mode 100644 fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/PubsubProducer.scala delete mode 100644 fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/encoder/MessageEncoder.scala delete mode 100644 project/Dependencies.scala delete mode 100644 project/build.properties delete mode 100644 project/plugins.sbt diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index d1d8a417..00000000 --- a/.editorconfig +++ /dev/null @@ -1,10 +0,0 @@ -# http://editorconfig.org -root = true - -[*] -indent_style = space -indent_size = 2 -end_of_line = lf -charset = utf-8 -trim_trailing_whitespace = true -insert_final_newline = true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 298c813d..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,188 +0,0 @@ -# This file was automatically generated by sbt-github-actions using the -# githubWorkflowGenerate task. You should add and commit this file to -# your git repository. It goes without saying that you shouldn't edit -# this file by hand! Instead, if you wish to make changes, you should -# change your sbt build configuration to revise the workflow description -# to meet your needs, then regenerate this file. - -name: Continuous Integration - -on: - pull_request: - branches: ['**', '!update/**', '!pr/**'] - push: - branches: ['**', '!update/**', '!pr/**'] - tags: [v*] - -env: - PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} - SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} - SONATYPE_CREDENTIAL_HOST: ${{ secrets.SONATYPE_CREDENTIAL_HOST }} - SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} - PGP_SECRET: ${{ secrets.PGP_SECRET }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - -jobs: - build: - name: Build and Test - strategy: - matrix: - os: [ubuntu-latest] - scala: [2.13.10, 2.12.17, 3.2.0] - java: [temurin@11] - project: [rootJVM] - runs-on: ${{ matrix.os }} - steps: - - name: Checkout current branch (full) - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Download Java (temurin@11) - id: download-java-temurin-11 - if: matrix.java == 'temurin@11' - uses: typelevel/download-java@v1 - with: - distribution: temurin - java-version: 11 - - - name: Setup Java (temurin@11) - if: matrix.java == 'temurin@11' - uses: actions/setup-java@v2 - with: - distribution: jdkfile - java-version: 11 - jdkFile: ${{ steps.download-java-temurin-11.outputs.jdkFile }} - - - name: Cache sbt - uses: actions/cache@v2 - with: - path: | - ~/.sbt - ~/.ivy2/cache - ~/.coursier/cache/v1 - ~/.cache/coursier/v1 - ~/AppData/Local/Coursier/Cache/v1 - ~/Library/Caches/Coursier/v1 - key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - - name: Check that workflows are up to date - run: sbt githubWorkflowCheck - - - name: Check headers and formatting - if: matrix.java == 'temurin@11' - run: sbt 'project ${{ matrix.project }}' '++${{ matrix.scala }}' headerCheckAll scalafmtCheckAll 'project /' scalafmtSbtCheck - - - name: Test - run: sbt 'project ${{ matrix.project }}' '++${{ matrix.scala }}' test - - - name: Check binary compatibility - if: matrix.java == 'temurin@11' - run: sbt 'project ${{ matrix.project }}' '++${{ matrix.scala }}' mimaReportBinaryIssues - - - name: Generate API documentation - if: matrix.java == 'temurin@11' - run: sbt 'project ${{ matrix.project }}' '++${{ matrix.scala }}' doc - - - name: Make target directories - if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') - run: mkdir -p fs2-google-pubsub-grpc/target target .js/target fs2-google-pubsub/target .jvm/target .native/target fs2-google-pubsub-http/target project/target - - - name: Compress target directories - if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') - run: tar cf targets.tar fs2-google-pubsub-grpc/target target .js/target fs2-google-pubsub/target .jvm/target .native/target fs2-google-pubsub-http/target project/target - - - name: Upload target directories - if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') - uses: actions/upload-artifact@v2 - with: - name: target-${{ matrix.os }}-${{ matrix.java }}-${{ matrix.scala }}-${{ matrix.project }} - path: targets.tar - - publish: - name: Publish Artifacts - needs: [build] - if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') - strategy: - matrix: - os: [ubuntu-latest] - scala: [2.13.10] - java: [temurin@11] - runs-on: ${{ matrix.os }} - steps: - - name: Checkout current branch (full) - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Download Java (temurin@11) - id: download-java-temurin-11 - if: matrix.java == 'temurin@11' - uses: typelevel/download-java@v1 - with: - distribution: temurin - java-version: 11 - - - name: Setup Java (temurin@11) - if: matrix.java == 'temurin@11' - uses: actions/setup-java@v2 - with: - distribution: jdkfile - java-version: 11 - jdkFile: ${{ steps.download-java-temurin-11.outputs.jdkFile }} - - - name: Cache sbt - uses: actions/cache@v2 - with: - path: | - ~/.sbt - ~/.ivy2/cache - ~/.coursier/cache/v1 - ~/.cache/coursier/v1 - ~/AppData/Local/Coursier/Cache/v1 - ~/Library/Caches/Coursier/v1 - key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - - name: Download target directories (2.13.10, rootJVM) - uses: actions/download-artifact@v2 - with: - name: target-${{ matrix.os }}-${{ matrix.java }}-2.13.10-rootJVM - - - name: Inflate target directories (2.13.10, rootJVM) - run: | - tar xf targets.tar - rm targets.tar - - - name: Download target directories (2.12.17, rootJVM) - uses: actions/download-artifact@v2 - with: - name: target-${{ matrix.os }}-${{ matrix.java }}-2.12.17-rootJVM - - - name: Inflate target directories (2.12.17, rootJVM) - run: | - tar xf targets.tar - rm targets.tar - - - name: Download target directories (3.2.0, rootJVM) - uses: actions/download-artifact@v2 - with: - name: target-${{ matrix.os }}-${{ matrix.java }}-3.2.0-rootJVM - - - name: Inflate target directories (3.2.0, rootJVM) - run: | - tar xf targets.tar - rm targets.tar - - - name: Import signing key - if: env.PGP_SECRET != '' && env.PGP_PASSPHRASE == '' - run: echo $PGP_SECRET | base64 -di | gpg --import - - - name: Import signing key and strip passphrase - if: env.PGP_SECRET != '' && env.PGP_PASSPHRASE != '' - run: | - echo "$PGP_SECRET" | base64 -di > /tmp/signing-key.gpg - echo "$PGP_PASSPHRASE" | gpg --pinentry-mode loopback --passphrase-fd 0 --import /tmp/signing-key.gpg - (echo "$PGP_PASSPHRASE"; echo; echo) | gpg --command-fd 0 --pinentry-mode loopback --change-passphrase $(gpg --list-secret-keys --with-colons 2> /dev/null | grep '^sec:' | cut --delimiter ':' --fields 5 | tail -n 1) - - - name: Publish - run: sbt '++${{ matrix.scala }}' tlRelease diff --git a/.github/workflows/clean.yml b/.github/workflows/clean.yml deleted file mode 100644 index 547aaa43..00000000 --- a/.github/workflows/clean.yml +++ /dev/null @@ -1,59 +0,0 @@ -# This file was automatically generated by sbt-github-actions using the -# githubWorkflowGenerate task. You should add and commit this file to -# your git repository. It goes without saying that you shouldn't edit -# this file by hand! Instead, if you wish to make changes, you should -# change your sbt build configuration to revise the workflow description -# to meet your needs, then regenerate this file. - -name: Clean - -on: push - -jobs: - delete-artifacts: - name: Delete Artifacts - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - name: Delete artifacts - run: | - # Customize those three lines with your repository and credentials: - REPO=${GITHUB_API_URL}/repos/${{ github.repository }} - - # A shortcut to call GitHub API. - ghapi() { curl --silent --location --user _:$GITHUB_TOKEN "$@"; } - - # A temporary file which receives HTTP response headers. - TMPFILE=/tmp/tmp.$$ - - # An associative array, key: artifact name, value: number of artifacts of that name. - declare -A ARTCOUNT - - # Process all artifacts on this repository, loop on returned "pages". - URL=$REPO/actions/artifacts - while [[ -n "$URL" ]]; do - - # Get current page, get response headers in a temporary file. - JSON=$(ghapi --dump-header $TMPFILE "$URL") - - # Get URL of next page. Will be empty if we are at the last page. - URL=$(grep '^Link:' "$TMPFILE" | tr ',' '\n' | grep 'rel="next"' | head -1 | sed -e 's/.*.*//') - rm -f $TMPFILE - - # Number of artifacts on this page: - COUNT=$(( $(jq <<<$JSON -r '.artifacts | length') )) - - # Loop on all artifacts on this page. - for ((i=0; $i < $COUNT; i++)); do - - # Get name of artifact and count instances of this name. - name=$(jq <<<$JSON -r ".artifacts[$i].name?") - ARTCOUNT[$name]=$(( $(( ${ARTCOUNT[$name]} )) + 1)) - - id=$(jq <<<$JSON -r ".artifacts[$i].id?") - size=$(( $(jq <<<$JSON -r ".artifacts[$i].size_in_bytes?") )) - printf "Deleting '%s' #%d, %'d bytes\n" $name ${ARTCOUNT[$name]} $size - ghapi -X DELETE $REPO/actions/artifacts/$id - done - done diff --git a/.gitignore b/.gitignore deleted file mode 100644 index e5209cbe..00000000 --- a/.gitignore +++ /dev/null @@ -1,27 +0,0 @@ -*.class -*.log - -# mill specific -out -target/ - -# Scala-IDE specific -.scala_dependencies -.worksheet - -# OS X specific -.DS_Store - -# IntelliJ -.idea -.idea_modules - -# bsp -.bsp - -.bloop -.metals -.vscode -project/.bloop -project/metals.sbt -project/project diff --git a/.scalafmt.conf b/.scalafmt.conf deleted file mode 100644 index 0894de25..00000000 --- a/.scalafmt.conf +++ /dev/null @@ -1,32 +0,0 @@ -version = 2.7.1 - -align.preset = more -assumeStandardLibraryStripMargin = true -trailingCommas = preserve -danglingParentheses.preset = true -continuationIndent.defnSite = 2 -maxColumn = 120 - -rewrite.rules = [AvoidInfix, PreferCurlyFors, RedundantBraces, RedundantParens, SortImports, SortModifiers] - -rewrite.neverInfix.excludeFilters = [ - be - by - eq - every - have - in - ne - size - synchronized - taggedAs - theSameElementsAs - noElementsOf - thrownBy - to - until - when - "contain.*" - "must.*" - "should.*" -] diff --git a/CODE-OF-CONDUCT.md b/CODE-OF-CONDUCT.md deleted file mode 100644 index 636e69cc..00000000 --- a/CODE-OF-CONDUCT.md +++ /dev/null @@ -1,44 +0,0 @@ -## CONTRIBUTOR COVENANT CODE OF CONDUCT - -### Our Pledge - -In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. - -### Our Standards -Examples of behavior that contributes to creating a positive environment include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others’ private information, such as a physical or electronic address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a professional setting - -### Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. - -### Scope - -This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. - -### Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at engineering@permutive.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project’s leadership. - -### Attribution - -This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html - -For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq \ No newline at end of file diff --git a/LICENSE b/LICENSE deleted file mode 100644 index ad410e11..00000000 --- a/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 100644 index 13f86f19..00000000 --- a/README.md +++ /dev/null @@ -1,382 +0,0 @@ -# fs2-google-pubsub -[![Build Status](https://img.shields.io/github/workflow/status/permutive-engineering/fs2-google-pubsub/Continuous%20Integration)](https://github.com/permutive-engineering/fs2-google-pubsub/actions/workflows/ci.yml) -[![Maven Central](https://img.shields.io/maven-central/v/com.permutive/fs2-google-pubsub_2.12.svg)](http://search.maven.org/#search%7Cga%7C1%7Cfs2-google-pubsub) - -[Google Cloud Pub/Sub][0] stream-based client built on top of [cats-effect][1], [fs2][2] and [http4s][6]. - -`fs2-google-pubsub` provides a mix of APIs, depending on the exact module. Consumers are provided as `fs2` streams, -while the producers are effect-based, utilising `cats-effect`. - -## Table of Contents -- [Module overview](#module-overview) - - [Public modules](#public-modules) - - [Internal modules](#internal-modules) -- [Dependencies](#dependencies) -- [Examples](#examples) - - [Consumer (Google)](#consumer-google) - - [Consumer (HTTP)](#consumer-http) - - [Producer (Google)](#producer-google) - - [Producer (HTTP)](#producer-http) - - [Producer (HTTP with automatic batching)](#producer-http-automatic-batching) -- [HTTP vs Google](#http-vs-google) - - [Google pros and cons](#google-pros-and-cons) - - [HTTP pros and cons](#http-pros-and-cons) - -## Module overview -### Public modules -- `fs2-google-pubsub-grpc` - an implementation that utilises Google's own [Java library][3] -- `fs2-google-pubsub-http` - an implementation that uses `http4s` and communicates via the [REST API][4] - -### Internal modules -- `fs2-google-pubsub` - shared classes for all implementations - -## Dependencies -Add one (or more) of the following to your `build.sbt`, see [Releases][5] for latest version: - -``` -libraryDependencies += "com.permutive" %% "fs2-google-pubsub-grpc" % Version -``` -OR -``` -libraryDependencies += "com.permutive" %% "fs2-google-pubsub-http" % Version -``` - -Also note you need to add an explicit HTTP client implementation. `http4s` provides different implementations -for the clients, including `blaze`, `async-http-client`, `jetty`, `okhttp` and others. - -If `async-http-client` is desired, add the following to `build.sbt`: -``` -libraryDependencies += "org.http4s" %% "http4s-async-http-client" % Version -``` - -## Examples - -### Consumer (Google) -See [PubsubGoogleConsumerConfig][7] for more configuration options. -```scala -package com.permutive.pubsub.consumer.google - -import cats.effect.{ExitCode, IO, IOApp} -import cats.syntax.all._ -import com.permutive.pubsub.consumer.Model -import com.permutive.pubsub.consumer.decoder.MessageDecoder - -object SimpleDriver extends IOApp { - case class ValueHolder(value: String) extends AnyVal - - implicit val decoder: MessageDecoder[ValueHolder] = (bytes: Array[Byte]) => { - Right(ValueHolder(new String(bytes))) - } - - override def run(args: List[String]): IO[ExitCode] = { - val stream = PubsubGoogleConsumer.subscribe[IO, ValueHolder]( - Model.ProjectId("test-project"), - Model.Subscription("example-sub"), - (msg, err, ack, _) => IO(println(s"Msg $msg got error $err")) >> ack, - config = PubsubGoogleConsumerConfig( - onFailedTerminate = _ => IO.unit - ) - ) - - stream - .evalTap(t => t.ack >> IO(println(s"Got: ${t.value}"))) - .compile - .drain - .as(ExitCode.Success) - } -} -``` - -### Consumer (HTTP) -See [PubsubHttpConsumerConfig][8] for more configuration options. -```scala -package com.permutive.pubsub.consumer.http - -import cats.effect._ -import cats.syntax.all._ -import com.permutive.pubsub.consumer.Model -import com.permutive.pubsub.consumer.decoder.MessageDecoder -import org.http4s.client.asynchttpclient.AsyncHttpClient -import fs2.Stream - -import scala.util.Try - -object Example extends IOApp { - case class ValueHolder(value: String) extends AnyVal - - implicit val decoder: MessageDecoder[ValueHolder] = (bytes: Array[Byte]) => { - Try(ValueHolder(new String(bytes))).toEither - } - - override def run(args: List[String]): IO[ExitCode] = { - val client = AsyncHttpClient.resource[IO]() - - val mkConsumer = PubsubHttpConsumer.subscribe[IO, ValueHolder]( - Model.ProjectId("test-project"), - Model.Subscription("example-sub"), - Some("/path/to/service/account"), - PubsubHttpConsumerConfig( - host = "localhost", - port = 8085, - isEmulator = true, - ), - _, - (msg, err, ack, _) => IO(println(s"Msg $msg got error $err")) >> ack, - ) - - Stream.resource(client) - .flatMap(mkConsumer) - .evalTap(t => t.ack >> IO(println(s"Got: ${t.value}"))) - .as(ExitCode.Success) - .compile - .lastOrError - } -} - -``` - -### Producer (Google) -See [PubsubProducerConfig][9] for more configuration. -```scala -package com.permutive.pubsub.producer.google - -import cats.effect.{ExitCode, IO, IOApp} -import cats.syntax.all._ -import com.permutive.pubsub.producer.Model -import com.permutive.pubsub.producer.encoder.MessageEncoder - -import scala.concurrent.duration._ - -object PubsubProducerExample extends IOApp { - - case class Value(v: Int) extends AnyVal - - implicit val encoder: MessageEncoder[Value] = new MessageEncoder[Value] { - override def encode(a: Value): Either[Throwable, Array[Byte]] = - Right(BigInt(a.v).toByteArray) - } - - override def run(args: List[String]): IO[ExitCode] = { - GooglePubsubProducer.of[IO, Value]( - Model.ProjectId("test-project"), - Model.Topic("values"), - config = PubsubProducerConfig[IO]( - batchSize = 100, - delayThreshold = 100.millis, - onFailedTerminate = e => IO(println(s"Got error $e")) >> IO.unit - ) - ).use { producer => - producer.produce( - Value(10), - ) - }.map(_ => ExitCode.Success) - } -} - -``` - -### Producer (HTTP) -See [PubsubHttpProducerConfig][10] for more configuration options. -```scala -package com.permutive.pubsub.producer.http - -import cats.effect.{ExitCode, IO, IOApp} -import cats.syntax.all._ -import com.github.plokhotnyuk.jsoniter_scala.core._ -import com.github.plokhotnyuk.jsoniter_scala.macros._ -import com.permutive.pubsub.producer.Model -import com.permutive.pubsub.producer.encoder.MessageEncoder -import org.http4s.client.asynchttpclient.AsyncHttpClient - -import scala.concurrent.duration._ -import scala.util.Try - -object ExampleGoogle extends IOApp { - - final implicit val Codec: JsonValueCodec[ExampleObject] = - JsonCodecMaker.make[ExampleObject](CodecMakerConfig) - - implicit val encoder: MessageEncoder[ExampleObject] = (a: ExampleObject) => { - Try(writeToArray(a)).toEither - } - - case class ExampleObject( - projectId: String, - url: String, - ) - - override def run(args: List[String]): IO[ExitCode] = { - val mkProducer = HttpPubsubProducer.resource[IO, ExampleObject]( - projectId = Model.ProjectId("test-project"), - topic = Model.Topic("example-topic"), - googleServiceAccountPath = Some("/path/to/service/account"), - config = PubsubHttpProducerConfig( - host = "pubsub.googleapis.com", - port = 443, - oauthTokenRefreshInterval = 30.minutes, - ), - _ - ) - - val http = AsyncHttpClient.resource[IO]() - http.flatMap(mkProducer).use { producer => - producer.produce( - data = ExampleObject("70251cf8-5ffb-4c3f-8f2f-40b9bfe4147c", "example.com") - ) - }.flatTap(output => IO(println(output))) >> IO.pure(ExitCode.Success) - } -} -``` - -### Producer (HTTP) automatic-batching -See [PubsubHttpProducerConfig][10] and [BatchingHttpPublisherConfig][11] for more configuration options. -```scala -package com.permutive.pubsub.producer.http - -import cats.effect.{ExitCode, IO, IOApp} -import cats.syntax.all._ -import com.github.plokhotnyuk.jsoniter_scala.core._ -import com.github.plokhotnyuk.jsoniter_scala.macros._ -import com.permutive.pubsub.producer.Model -import com.permutive.pubsub.producer.encoder.MessageEncoder -import org.typelevel.log4cats.Logger -import org.typelevel.log4cats.slf4j.Slf4jLogger -import org.http4s.client.asynchttpclient.AsyncHttpClient - -import scala.concurrent.duration._ -import scala.util.Try - -object ExampleBatching extends IOApp { - - private[this] final implicit val unsafeLogger: Logger[IO] = Slf4jLogger.unsafeCreate[IO] - - final implicit val Codec: JsonValueCodec[ExampleObject] = - JsonCodecMaker.make[ExampleObject](CodecMakerConfig) - - implicit val encoder: MessageEncoder[ExampleObject] = (a: ExampleObject) => { - Try(writeToArray(a)).toEither - } - - case class ExampleObject( - projectId: String, - url: String, - ) - - override def run(args: List[String]): IO[ExitCode] = { - val mkProducer = BatchingHttpPubsubProducer.resource[IO, ExampleObject]( - projectId = Model.ProjectId("test-project"), - topic = Model.Topic("example-topic"), - googleServiceAccountPath = Some("/path/to/service/account"), - config = PubsubHttpProducerConfig( - host = "localhost", - port = 8085, - oauthTokenRefreshInterval = 30.minutes, - isEmulator = true, - ), - - batchingConfig = BatchingHttpProducerConfig( - batchSize = 10, - maxLatency = 100.millis, - - retryTimes = 0, - retryInitialDelay = 0.millis, - retryNextDelay = _ + 250.millis, - ), - _ - ) - - val messageCallback: Either[Throwable, Unit] => IO[Unit] = { - case Right(_) => Logger[IO].info("Async message was sent successfully!") - case Left(e) => Logger[IO].warn(e)("Async message was sent unsuccessfully!") - } - - client - .flatMap(mkProducer) - .use { producer => - val produceOne = producer.produce( - data = ExampleObject("1f9774be-9d7c-4dd9-8d97-855b681938a9", "example.com"), - ) - - val produceOneAsync = producer.produceAsync( - data = ExampleObject("a84a3318-adbd-4eac-af78-eacf33be91ef", "example.com"), - callback = messageCallback - ) - - for { - result1 <- produceOne - result2 <- produceOne - result3 <- produceOne - _ <- result1 - _ <- Logger[IO].info("First message was sent!") - _ <- result2 - _ <- Logger[IO].info("Second message was sent!") - _ <- result3 - _ <- Logger[IO].info("Third message was sent!") - _ <- produceOneAsync - _ <- IO.never - } yield () - } - .as(ExitCode.Success) - } -} -``` - -## HTTP vs Google -### Google pros and cons -Pros of using the Google library -- Underlying library well supported (theoretically) -- Uses gRPC and HTTP/2 (should be faster) -- Automatically handles authentication - -Cons of using Google Library -- Uses gRPC (if you uses multiple Google libraries with different gRPC versions, something *will* break) -- Bloated -- More dependencies -- Less functional -- Doesn't work with the official [PubSub emulator][12] (is in [feature backlog][13]) -- Google API can change at any point (shouldn't be exposed to users of `fs2-google-pubsub`, but slows development/updating) - -### HTTP pros and cons -Pros of using HTTP variant -- Less dependencies -- Works with the [PubSub emulator][12] -- Fully functional -- Stable API -- Theoretically less memory usage, especially for producer - -Cons of using HTTP variant -- Authentication is handled manually, hence *potentially* less secure/reliable -- By default uses old HTTP 1.1 (potentially slower), but can be configured to use HTTP/2 if supported HTTP client backend is chosen - - -## Licence -``` - Copyright 2018-2019 Permutive, Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -``` - -[0]: https://cloud.google.com/pubsub -[1]: https://github.com/typelevel/cats-effect -[2]: https://github.com/functional-streams-for-scala/fs2 -[3]: https://cloud.google.com/pubsub/docs/reference/libraries -[4]: https://cloud.google.com/pubsub/docs/reference/rest/ -[5]: https://github.com/permutive/fs2-google-pubsub/releases -[6]: https://github.com/http4s/http4s -[7]: https://github.com/permutive/fs2-google-pubsub/blob/master/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/PubsubGoogleConsumerConfig.scala -[8]: https://github.com/permutive/fs2-google-pubsub/blob/master/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubHttpConsumerConfig.scala -[9]: https://github.com/permutive/fs2-google-pubsub/blob/master/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/PubsubProducerConfig.scala -[10]: https://github.com/permutive/fs2-google-pubsub/blob/master/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/PubsubHttpProducerConfig.scala -[11]: https://github.com/permutive/fs2-google-pubsub/blob/master/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/BatchingHttpProducerConfig.scala -[12]: https://cloud.google.com/pubsub/docs/emulator -[13]: https://github.com/googleapis/google-cloud-java/wiki/Feature-backlog diff --git a/RELEASE.md b/RELEASE.md deleted file mode 100644 index 512735ef..00000000 --- a/RELEASE.md +++ /dev/null @@ -1,26 +0,0 @@ -# Release process - -- Update build.sc version (follow SEMVER) -- Make a commit ("Set version to 0.6.6") -- Make a tag ("v0.6.6") -- Run the below (may require multiple attempts) -``` -mill mill.scalalib.PublishModule/publishAll \ - --sonatypeCreds $SONATYPE_USERNAME:$SONATYPE_PASSWORD \ - --gpgPassphrase $GPG_PASSPHRASE \ - --publishArtifacts \ - __.publishArtifacts -``` -- Login to Sonatype (https://oss.sonatype.org/) -- Go to Staging Repositories -- Find the repository (usually something like compermutive-####) -- Check contents of repository (see all jars are there) -- Close repository -- Release repository (select auto-drop) -- Make a commit ("Set version to 0.6.7-SNAPSHOT") -- Push both commits -- Push new tag -- Check https://mvnrepository.com/artifact/com.permutive/fs2-google-pubsub in a while -- You're done! - - diff --git a/build.sbt b/build.sbt deleted file mode 100644 index 623082c7..00000000 --- a/build.sbt +++ /dev/null @@ -1,50 +0,0 @@ -ThisBuild / tlBaseVersion := "0.22" // your current series x.y - -ThisBuild / organization := "com.permutive" -ThisBuild / organizationName := "Permutive" -ThisBuild / organizationHomepage := Some(url("https://github.com/permutive")) -ThisBuild / licenses := Seq(License.Apache2) -ThisBuild / developers := List( - tlGitHubDev("cremboc", "Paulius Imbrasas"), - tlGitHubDev("TimWSpence", "Tim Spence"), - tlGitHubDev("bastewart", "Ben Stewart"), - tlGitHubDev("travisbrown", "Travis Brown") -) -ThisBuild / startYear := Some(2018) - -ThisBuild / tlSonatypeUseLegacyHost := true - -val Scala213 = "2.13.10" -ThisBuild / crossScalaVersions := Seq(Scala213, "2.12.17", "3.2.0") -ThisBuild / scalaVersion := Scala213 // the default Scala -ThisBuild / tlJdkRelease := Some(11) -ThisBuild / githubWorkflowJavaVersions := Seq(JavaSpec.temurin("11")) - -lazy val root = tlCrossRootProject - .aggregate(common, http, grpc) - -lazy val common = project - .in(file("fs2-google-pubsub")) - .settings( - name := "fs2-google-pubsub", - libraryDependencies ++= Dependencies.commonDependencies, - libraryDependencies ++= Dependencies.testsDependencies - ) - -lazy val http = project - .in(file("fs2-google-pubsub-http")) - .dependsOn(common) - .settings( - name := "fs2-google-pubsub-http", - libraryDependencies ++= Dependencies.httpDependencies, - libraryDependencies ++= Dependencies.testsDependencies - ) - -lazy val grpc = project - .in(file("fs2-google-pubsub-grpc")) - .dependsOn(common) - .settings( - name := "fs2-google-pubsub-grpc", - libraryDependencies ++= Dependencies.grpcDependencies, - libraryDependencies ++= Dependencies.testsDependencies - ) diff --git a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/PubsubGoogleConsumer.scala b/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/PubsubGoogleConsumer.scala deleted file mode 100644 index b4fe60e1..00000000 --- a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/PubsubGoogleConsumer.scala +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer.grpc - -import cats.Applicative -import cats.effect.kernel.Sync -import cats.syntax.all._ -import com.google.pubsub.v1.PubsubMessage -import com.permutive.pubsub.consumer.decoder.MessageDecoder -import com.permutive.pubsub.consumer.grpc.internal.PubsubSubscriber -import com.permutive.pubsub.consumer.{ConsumerRecord, Model} -import fs2.Stream - -import scala.jdk.CollectionConverters._ -import scala.util.control.NoStackTrace - -object PubsubGoogleConsumer { - - /** - * Indicates the underlying Java PubSub consumer has failed. - * - * @param cause the cause of the failure - */ - case class InternalPubSubError(cause: Throwable) - extends Throwable("Internal Java PubSub consumer failed", cause) - with NoStackTrace - - /** - * Subscribe with manual acknowledgement - * - * The stream fails with an [[InternalPubSubError]] if the underlying Java consumer fails. - * - * @param projectId google cloud project id - * @param subscription name of the subscription - * @param errorHandler upon failure to decode, an exception is thrown. Allows acknowledging the message. - */ - final def subscribe[F[_]: Sync, A: MessageDecoder]( - projectId: Model.ProjectId, - subscription: Model.Subscription, - errorHandler: (PubsubMessage, Throwable, F[Unit], F[Unit]) => F[Unit], - config: PubsubGoogleConsumerConfig[F] - ): Stream[F, ConsumerRecord[F, A]] = - subscribeDecode[F, A, ConsumerRecord[F, A]]( - projectId, - subscription, - errorHandler, - config, - onDecode = (record, value) => - Applicative[F].pure( - ConsumerRecord( - value, - record.value.getAttributesMap.asScala.toMap, - record.ack, - record.nack, - _ => Applicative[F].unit - ) - ), - ) - - /** - * Subscribe with automatic acknowledgement - * - * The stream fails with an [[InternalPubSubError]] if the underlying Java consumer fails. - * - * @param projectId google cloud project id - * @param subscription name of the subscription - * @param errorHandler upon failure to decode, an exception is thrown. Allows acknowledging the message. - */ - final def subscribeAndAck[F[_]: Sync, A: MessageDecoder]( - projectId: Model.ProjectId, - subscription: Model.Subscription, - errorHandler: (PubsubMessage, Throwable, F[Unit], F[Unit]) => F[Unit], - config: PubsubGoogleConsumerConfig[F] - ): Stream[F, A] = - subscribeDecode[F, A, A]( - projectId, - subscription, - errorHandler, - config, - onDecode = (record, value) => record.ack.as(value), - ) - - /** - * Subscribe to the raw stream, receiving the the message as retrieved from PubSub - * - * The stream fails with an [[InternalPubSubError]] if the underlying Java consumer fails. - */ - final def subscribeRaw[F[_]: Sync]( - projectId: Model.ProjectId, - subscription: Model.Subscription, - config: PubsubGoogleConsumerConfig[F] - ): Stream[F, ConsumerRecord[F, PubsubMessage]] = - PubsubSubscriber - .subscribe(projectId, subscription, config) - .map(msg => - ConsumerRecord(msg.value, msg.value.getAttributesMap.asScala.toMap, msg.ack, msg.nack, _ => Applicative[F].unit) - ) - - private def subscribeDecode[F[_]: Sync, A: MessageDecoder, B]( - projectId: Model.ProjectId, - subscription: Model.Subscription, - errorHandler: (PubsubMessage, Throwable, F[Unit], F[Unit]) => F[Unit], - config: PubsubGoogleConsumerConfig[F], - onDecode: (internal.Model.Record[F], A) => F[B], - ): Stream[F, B] = - PubsubSubscriber - .subscribe(projectId, subscription, config) - .evalMapChunk[F, Option[B]](record => - MessageDecoder[A].decode(record.value.getData.toByteArray) match { - case Left(e) => errorHandler(record.value, e, record.ack, record.ack).as(None) - case Right(v) => onDecode(record, v).map(Some(_)) - } - ) - .unNone -} diff --git a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/PubsubGoogleConsumerConfig.scala b/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/PubsubGoogleConsumerConfig.scala deleted file mode 100644 index 15dae532..00000000 --- a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/PubsubGoogleConsumerConfig.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer.grpc - -import com.google.cloud.pubsub.v1.Subscriber - -import scala.concurrent.duration._ - -/** - * Pubsub subscriber config. - * - * @param maxQueueSize configures two options: the max size of the backing queue, and, the "max outstanding element count" option of Pubsub - * @param parallelPullCount number of parallel pullers, see [[https://javadoc.io/static/com.google.cloud/google-cloud-pubsub/1.100.0/com/google/cloud/pubsub/v1/Subscriber.Builder.html#setParallelPullCount-int-]] - * @param maxAckExtensionPeriod see [[https://javadoc.io/static/com.google.cloud/google-cloud-pubsub/1.100.0/com/google/cloud/pubsub/v1/Subscriber.Builder.html#setMaxAckExtensionPeriod-org.threeten.bp.Duration-]] - * @param awaitTerminatePeriod if the underlying PubSub subcriber fails to terminate cleanly, how long do we wait until it's forcibly timed out. - * @param onFailedTerminate upon failure to terminate, call this function - * @param customizeSubscriber optionally, provide a function that allows full customisation of the underlying Java Subscriber object. - */ -case class PubsubGoogleConsumerConfig[F[_]]( - maxQueueSize: Int = 1000, - parallelPullCount: Int = 3, - maxAckExtensionPeriod: FiniteDuration = 10.seconds, - awaitTerminatePeriod: FiniteDuration = 30.seconds, - onFailedTerminate: Throwable => F[Unit], - customizeSubscriber: Option[Subscriber.Builder => Subscriber.Builder] = None -) diff --git a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/internal/Model.scala b/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/internal/Model.scala deleted file mode 100644 index 75bae3c7..00000000 --- a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/internal/Model.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer.grpc.internal - -import com.google.pubsub.v1.PubsubMessage - -private[consumer] object Model { - case class Record[F[_]](value: PubsubMessage, ack: F[Unit], nack: F[Unit]) -} diff --git a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/internal/PubsubSubscriber.scala b/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/internal/PubsubSubscriber.scala deleted file mode 100644 index d754e1b9..00000000 --- a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/internal/PubsubSubscriber.scala +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer.grpc.internal - -import cats.Applicative -import cats.effect.kernel.{Resource, Sync} -import cats.syntax.all._ -import com.google.api.core.ApiService -import com.google.api.gax.batching.FlowControlSettings -import com.google.cloud.pubsub.v1.{AckReplyConsumer, MessageReceiver, Subscriber} -import com.google.common.util.concurrent.MoreExecutors -import com.google.pubsub.v1.{ProjectSubscriptionName, PubsubMessage} -import com.permutive.pubsub.consumer.grpc.PubsubGoogleConsumer.InternalPubSubError -import com.permutive.pubsub.consumer.grpc.PubsubGoogleConsumerConfig -import com.permutive.pubsub.consumer.{Model => PublicModel} -import fs2.{Chunk, Stream} -import org.threeten.bp.Duration - -import java.util -import java.util.concurrent.{BlockingQueue, LinkedBlockingQueue, TimeUnit} -import scala.collection.mutable.Builder - -private[consumer] object PubsubSubscriber { - - def createSubscriber[F[_]: Sync]( - projectId: PublicModel.ProjectId, - subscription: PublicModel.Subscription, - config: PubsubGoogleConsumerConfig[F], - queue: BlockingQueue[Either[InternalPubSubError, Model.Record[F]]], - ): Resource[F, ApiService] = - Resource.make( - Sync[F].delay { - val receiver = new PubsubMessageReceiver(queue) - val subscriptionName = ProjectSubscriptionName.of(projectId.value, subscription.value) - - // build subscriber with "normal" settings - val builder = - Subscriber - .newBuilder(subscriptionName, receiver) - .setFlowControlSettings( - FlowControlSettings - .newBuilder() - .setMaxOutstandingElementCount(config.maxQueueSize.toLong) - .build() - ) - .setParallelPullCount(config.parallelPullCount) - .setMaxAckExtensionPeriod(Duration.ofMillis(config.maxAckExtensionPeriod.toMillis)) - - // if provided, use subscriber transformer to modify subscriber - val sub = - config.customizeSubscriber - .map(f => f(builder)) - .getOrElse(builder) - .build() - - sub.addListener(new PubsubErrorListener(queue), MoreExecutors.directExecutor) - - sub.startAsync() - } - )(service => - Sync[F] - .blocking(service.stopAsync().awaitTerminated(config.awaitTerminatePeriod.toSeconds, TimeUnit.SECONDS)) - .handleErrorWith(config.onFailedTerminate) - ) - - class PubsubMessageReceiver[F[_]: Sync, L](queue: BlockingQueue[Either[L, Model.Record[F]]]) extends MessageReceiver { - override def receiveMessage(message: PubsubMessage, consumer: AckReplyConsumer): Unit = - queue.put(Right(Model.Record(message, Sync[F].delay(consumer.ack()), Sync[F].delay(consumer.nack())))) - } - - class PubsubErrorListener[R](queue: BlockingQueue[Either[InternalPubSubError, R]]) extends ApiService.Listener { - override def failed(from: ApiService.State, failure: Throwable): Unit = - queue.put(Left(InternalPubSubError(failure))) - } - - def takeNextElements[F[_]: Sync, A](messages: BlockingQueue[A]): F[Chunk[A]] = - for { - nextOpt <- Sync[F].delay(messages.poll()) // `poll` is non-blocking, returning `null` if queue is empty - // `take` can wait for an element - next <- - if (nextOpt == null) Sync[F].interruptibleMany(messages.take()) - else Applicative[F].pure(nextOpt) - chunk <- Sync[F].delay { - val c = Wrapper.empty[A] - c.add(next) - messages.drainTo(c) - - c.toChunk - } - } yield chunk - - def subscribe[F[_]: Sync]( - projectId: PublicModel.ProjectId, - subscription: PublicModel.Subscription, - config: PubsubGoogleConsumerConfig[F], - ): Stream[F, Model.Record[F]] = - for { - queue <- Stream.eval( - Sync[F].delay(new LinkedBlockingQueue[Either[InternalPubSubError, Model.Record[F]]](config.maxQueueSize)) - ) - _ <- Stream.resource(PubsubSubscriber.createSubscriber(projectId, subscription, config, queue)) - taken <- Stream.repeatEval(takeNextElements(queue)) - // Only retains the first error (if there are multiple), but that is OK, the stream is failing anyway... - msg <- Stream.fromEither[F](taken.sequence).unchunks - } yield msg -} - -/** A wrapper that implements `java.util.Collection[A]` for `Builder[A, Vector[A]]` - * so that we can pass the builder directly to the underlying library and - * avoid copying. - * - * Only the minimal set of methods required are actually implemented. - */ -private[consumer] class Wrapper[A](val underlying: Builder[A, Vector[A]]) extends util.Collection[A] { - - override def size(): Int = ??? - - override def isEmpty(): Boolean = ??? - - override def contains(x$1: Object): Boolean = ??? - - override def iterator(): util.Iterator[A] = ??? - - override def toArray(): Array[Object] = ??? - - override def toArray[T](x$1: Array[T with Object]): Array[T with Object] = ??? - - override def add(x: A): Boolean = { - underlying += x - true - } - - override def remove(x$1: Object): Boolean = ??? - - override def containsAll(x$1: util.Collection[_]): Boolean = ??? - - override def addAll(xs: util.Collection[_ <: A]): Boolean = { - xs.forEach(x => underlying += x) - true - } - - override def removeAll(x$1: util.Collection[_]): Boolean = ??? - - override def retainAll(x$1: util.Collection[_]): Boolean = ??? - - override def clear(): Unit = ??? - - def toChunk: Chunk[A] = Chunk.vector(underlying.result()) - -} - -object Wrapper { - def empty[A]: Wrapper[A] = new Wrapper(Vector.newBuilder[A]) -} diff --git a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/GooglePubsubProducer.scala b/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/GooglePubsubProducer.scala deleted file mode 100644 index e5d06f82..00000000 --- a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/GooglePubsubProducer.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.grpc - -import cats.effect.kernel.{Async, Resource} -import com.permutive.pubsub.producer.Model.{ProjectId, Topic} -import com.permutive.pubsub.producer.PubsubProducer -import com.permutive.pubsub.producer.encoder.MessageEncoder -import com.permutive.pubsub.producer.grpc.internal.{DefaultPublisher, PubsubPublisher} - -object GooglePubsubProducer { - def of[F[_]: Async, A: MessageEncoder]( - projectId: ProjectId, - topic: Topic, - config: PubsubProducerConfig[F] - ): Resource[F, PubsubProducer[F, A]] = - for { - publisher <- PubsubPublisher.createJavaPublisher(projectId, topic, config) - } yield new DefaultPublisher(publisher) -} diff --git a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/PubsubProducerConfig.scala b/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/PubsubProducerConfig.scala deleted file mode 100644 index 83aa325a..00000000 --- a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/PubsubProducerConfig.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.grpc - -import com.google.cloud.pubsub.v1.Publisher - -import scala.concurrent.duration._ - -case class PubsubProducerConfig[F[_]]( - batchSize: Long, - delayThreshold: FiniteDuration, - requestByteThreshold: Option[Long] = None, - averageMessageSize: Long = 1024, // 1kB - // modify publisher - customizePublisher: Option[Publisher.Builder => Publisher.Builder] = None, - // termination - awaitTerminatePeriod: FiniteDuration = 30.seconds, - onFailedTerminate: Throwable => F[Unit] -) diff --git a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/internal/DefaultPublisher.scala b/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/internal/DefaultPublisher.scala deleted file mode 100644 index 23e147bb..00000000 --- a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/internal/DefaultPublisher.scala +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.grpc.internal - -import cats.Traverse -import cats.effect.kernel.{Async, Sync} -import cats.syntax.all._ -import com.google.cloud.pubsub.v1.Publisher -import com.google.protobuf.ByteString -import com.google.pubsub.v1.PubsubMessage -import com.permutive.pubsub.producer.Model.MessageId -import com.permutive.pubsub.producer.encoder.MessageEncoder -import com.permutive.pubsub.producer.{Model, PubsubProducer} - -import java.util.UUID -import scala.jdk.CollectionConverters._ - -private[pubsub] class DefaultPublisher[F[_]: Async, A: MessageEncoder]( - publisher: Publisher, -) extends PubsubProducer[F, A] { - final override def produce( - data: A, - attributes: Map[String, String] = Map.empty, - uniqueId: String = UUID.randomUUID.toString - ): F[MessageId] = - MessageEncoder[A].encode(data).liftTo[F].flatMap { v => - val message = - PubsubMessage.newBuilder - .setData(ByteString.copyFrom(v)) - .setMessageId(uniqueId) - .putAllAttributes(attributes.asJava) - .build() - - FutureInterop.fFromFuture(Sync[F].delay(publisher.publish(message))).map(MessageId(_)) - } - - override def produceMany[G[_]: Traverse](records: G[Model.Record[A]]): F[List[MessageId]] = - records.traverse(r => produce(r.data, r.attributes, r.uniqueId)).map(_.toList) -} diff --git a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/internal/FutureInterop.scala b/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/internal/FutureInterop.scala deleted file mode 100644 index e941f9b3..00000000 --- a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/internal/FutureInterop.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.grpc.internal - -import cats.syntax.all._ -import cats.effect.kernel.{Async, Sync} -import com.google.api.core.{ApiFuture, ApiFutureCallback, ApiFutures} -import com.google.common.util.concurrent.MoreExecutors - -private[internal] object FutureInterop { - def fFromFuture[F[_]: Async, A](future: F[ApiFuture[A]]): F[A] = - Async[F] - .async[A] { cb => - future.flatMap { futA => - Sync[F].delay { - val futureApi = futA - addCallback(futureApi)(cb) - Some( - Sync[F] - .delay( - // This boolean setting is `mayInterruptIfRunning`: - // `if the thread executing this task should be interrupted; otherwise, in-progress tasks are allowed - // to complete`. - // - // We set this `false` as testing showed that it was not required for calling `cancel` on the `F[A]` effect - // to return immediately. It also slowed down the execution of this code block. We do not mind if the future - // eventually completes as long as cancelling the effect does not block until completion. - // - // See https://permutive.atlassian.net/browse/PLAT-255 for details (see links in ticket description). - futureApi.cancel(false), - ) - .void - ) - } - } - } - - @inline - private def addCallback[A](futA: ApiFuture[A])(cb: Either[Throwable, A] => Unit): Unit = - ApiFutures.addCallback( - futA, - new ApiFutureCallback[A] { - override def onFailure(t: Throwable): Unit = - cb(Left(t)) - - override def onSuccess(result: A): Unit = - cb(Right(result)) - }, - // We use the `directExecutor` as this is the location to run the callback _after_ completion*. In our case that - // means where to run `onFailure` and `onSuccess` which are both lightweight. The underlying library uses this - // executor for similarly simple functions. - // - // * From a docstring used by `addCallback` (`Futures.addCallback`): - // `The executor to run {@code callback} when the future completes.` - // - // See docstring of `ListenableFuture.addListener` for more details on this being safe. - // As above see https://permutive.atlassian.net/browse/PLAT-255 for further details on deciding this. - MoreExecutors.directExecutor(), - ) -} diff --git a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/internal/PubsubPublisher.scala b/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/internal/PubsubPublisher.scala deleted file mode 100644 index 33a10fcb..00000000 --- a/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/internal/PubsubPublisher.scala +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.grpc.internal - -import cats.effect.kernel.{Resource, Sync} -import cats.syntax.all._ -import com.google.api.gax.batching.BatchingSettings -import com.google.cloud.pubsub.v1.Publisher -import com.google.pubsub.v1.ProjectTopicName -import com.permutive.pubsub.producer.Model.{ProjectId, Topic} -import com.permutive.pubsub.producer.grpc.PubsubProducerConfig -import org.threeten.bp.Duration - -import java.util.concurrent.TimeUnit - -private[producer] object PubsubPublisher { - def createJavaPublisher[F[_]: Sync]( - projectId: ProjectId, - topic: Topic, - config: PubsubProducerConfig[F] - ): Resource[F, Publisher] = - Resource[F, Publisher] { - Sync[F].delay { - val topicName = ProjectTopicName.of(projectId.value, topic.value) - - val publisherBuilder = - Publisher - .newBuilder(topicName) - .setBatchingSettings( - BatchingSettings - .newBuilder() - .setElementCountThreshold(config.batchSize) - .setRequestByteThreshold( - config.requestByteThreshold.getOrElse[Long](config.batchSize * config.averageMessageSize * 2L) - ) - .setDelayThreshold(Duration.ofMillis(config.delayThreshold.toMillis)) - .build() - ) - - val publisher = - config.customizePublisher - .map(f => f(publisherBuilder)) - .getOrElse(publisherBuilder) - .build() - - val shutdown = - for { - _ <- Sync[F].blocking(publisher.shutdown()) - _ <- Sync[F].blocking( - publisher.awaitTermination(config.awaitTerminatePeriod.toMillis, TimeUnit.MILLISECONDS) - ) - } yield () - - (publisher, shutdown) - } - } -} diff --git a/fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/GrpcPingPongSpec.scala b/fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/GrpcPingPongSpec.scala deleted file mode 100644 index f219582f..00000000 --- a/fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/GrpcPingPongSpec.scala +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub - -import cats.effect._ -import cats.syntax.all._ -import com.google.cloud.pubsub.v1.{SubscriptionAdminClient, TopicAdminClient} -import com.google.pubsub.v1.{SubscriptionName, TopicName} -import com.permutive.pubsub.consumer.ConsumerRecord -import com.permutive.pubsub.producer.PubsubProducer -import fs2.Stream -import org.scalatest.BeforeAndAfterEach -import org.typelevel.log4cats.Logger -import org.typelevel.log4cats.slf4j.Slf4jLogger - -import scala.concurrent.duration._ - -class GrpcPingPongSpec extends PubSubSpec with BeforeAndAfterEach { - - implicit val logger: Logger[IO] = Slf4jLogger.getLogger - - // Delete topic and subscriptions after each test to ensure state is clean - override def afterEach(): Unit = - clearTopicSubscription - .handleErrorWith(_ => logger.warn("Errors were thrown after tests on clean-up")) - .unsafeRunSync() - - private[this] val topicAndSubscriptionClient: Resource[IO, (TopicAdminClient, SubscriptionAdminClient)] = - for { - transCreds <- providers - topicClient <- topicAdminClient(transCreds._1, transCreds._2) - subscriptionClient <- subscriptionAdminClient(transCreds._1, transCreds._2) - } yield (topicClient, subscriptionClient) - - private[this] val clearTopicSubscription: IO[Unit] = - topicAndSubscriptionClient.use { case (topicClient, subscriptionClient) => - for { - _ <- deleteSubscription(subscriptionClient, SubscriptionName.of(project, subscription)) - _ <- deleteTopic(topicClient, TopicName.of(project, topic)) - } yield () - } - - private def setup(ackDeadlineSeconds: Int): Resource[IO, (PubsubProducer[IO, ValueHolder])] = - for { - _ <- Resource.eval(createTopic(project, topic)) - _ <- Resource.eval(createSubscription(project, topic, subscription, ackDeadlineSeconds)) - p <- producer() - } yield p - - private def consumeExpectingLimitedMessages( - messagesExpected: Int, - ): Stream[IO, ConsumerRecord[IO, ValueHolder]] = - consumer() - // Check we only receive a single element - .zipWithIndex - .flatMap { case (record, ix) => - Stream.fromEither[IO]( - // Index is 0-based, so we expected index to reach 1 less than `messagesExpected` - if (ix < messagesExpected.toLong) Right(record) - else Left(new RuntimeException(s"Received more than $messagesExpected from PubSub")) - ) - } - // Check body is as we expect - .flatMap(record => - Stream.fromEither[IO]( - if (record.value == ValueHolder("ping")) Right(record) - else Left(new RuntimeException(s"Consumed element did not have correct value: ${record.value}")) - ) - ) - - private def consumeAndAck( - elementsReceived: Ref[IO, Int], - ): Stream[IO, ConsumerRecord[IO, ValueHolder]] = - consumeExpectingLimitedMessages(messagesExpected = 1) - // Indicate we have received the element as expected - .evalTap(_ => elementsReceived.update(_ + 1)) - .evalTap(_.ack) - - it should "send and receive a message, acknowledging as expected" in { - (for { - // We will sleep for 10 seconds, which means if the message is not acked it will be redelivered before end of test - producer <- Stream.resource(setup(ackDeadlineSeconds = 5)) - _ <- Stream.eval(producer.produce(ValueHolder("ping"))) - ref <- Stream.eval(Ref.of[IO, Int](0)) - // Wait 10 seconds whilst we run the consumer to check we have a single element and it has the right data - _ <- Stream.sleep[IO](10.seconds).concurrently(consumeAndAck(ref)) - elementsReceived <- Stream.eval(ref.get) - } yield elementsReceived should ===(1)).compile.drain - .timeout(30.seconds) // avoiding running forever in case of an issue - .unsafeRunSync() - } - - private def consumeExtendSleepAck( - elementsReceived: Ref[IO, Int], - extendDuration: FiniteDuration, - sleepDuration: FiniteDuration, - ): Stream[IO, ConsumerRecord[IO, ValueHolder]] = - consumeExpectingLimitedMessages(messagesExpected = 1) - .evalTap(_.extendDeadline(extendDuration)) - .evalTap(_ => IO.sleep(sleepDuration)) - // Indicate we have received the element as expected - .evalTap(_ => elementsReceived.update(_ + 1)) - .evalTap(_.ack) - - it should "extend the deadline for a message" in { - // These setting mean that if extension does not work the message will be redelivered before the end of the test - val ackDeadlineSeconds = 2 - val sleepDuration = 3.seconds - val extendDuration = 10.seconds - - (for { - producer <- Stream.resource(setup(ackDeadlineSeconds = ackDeadlineSeconds)) - _ <- Stream.eval(producer.produce(ValueHolder("ping"))) - ref <- Stream.eval(Ref.of[IO, Int](0)) - // Wait 10 seconds whilst we run the consumer to check we have a single element and it has the right data - _ <- Stream - .sleep[IO](10.seconds) - .concurrently( - consumeExtendSleepAck(ref, extendDuration = extendDuration, sleepDuration = sleepDuration) - ) - elementsReceived <- Stream.eval(ref.get) - } yield elementsReceived should ===(1)) - .as(ExitCode.Success) - .compile - .drain - .timeout(30.seconds) // avoiding running forever in case of an issue - .unsafeRunSync() - } - - private def consumeNackThenAck( - elementsReceived: Ref[IO, Int], - messagesExpected: Int, - ): Stream[IO, Unit] = - // We expect the message to be redelivered once, so expect 2 messages total - consumeExpectingLimitedMessages(messagesExpected) - // Indicate we have received the element as expected - .evalTap(_ => elementsReceived.update(_ + 1)) - // Nack the first message, then ack subsequent ones - .evalScan(false) { case (nackedAlready, record) => - if (nackedAlready) record.ack.as(true) else record.nack.as(true) - } - .void - - it should "nack a message properly" in { - // These setting mean that a message will only be redelivered if it is nacked - val ackDeadlineSeconds = 100 - val messagesExpected = 2 - - (for { - producer <- Stream.resource(setup(ackDeadlineSeconds = ackDeadlineSeconds)) - _ <- Stream.eval(producer.produce(ValueHolder("ping"))) - ref <- Stream.eval(Ref.of[IO, Int](0)) - // Wait 10 seconds whilst we run the consumer which nacks the message, then acks - _ <- Stream - .sleep[IO](10.seconds) - .concurrently(consumeNackThenAck(ref, messagesExpected)) - elementsReceived <- Stream.eval(ref.get) - } yield elementsReceived should ===(messagesExpected)) - .as(ExitCode.Success) - .compile - .drain - .timeout(30.seconds) // avoiding running forever in case of an issue - .unsafeRunSync() - } - -} diff --git a/fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/PubSubSpec.scala b/fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/PubSubSpec.scala deleted file mode 100644 index 1bf43679..00000000 --- a/fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/PubSubSpec.scala +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub - -import cats.effect.unsafe.IORuntime -import cats.effect.{IO, Resource} -import com.dimafeng.testcontainers.{ForAllTestContainer, GenericContainer} -import com.google.api.gax.core.{CredentialsProvider, NoCredentialsProvider} -import com.google.api.gax.grpc.GrpcTransportChannel -import com.google.api.gax.rpc.{FixedTransportChannelProvider, TransportChannelProvider} -import com.google.cloud.pubsub.v1.{ - SubscriptionAdminClient, - SubscriptionAdminSettings, - TopicAdminClient, - TopicAdminSettings -} -import com.google.pubsub.v1._ -import com.permutive.pubsub.consumer.grpc.{PubsubGoogleConsumer, PubsubGoogleConsumerConfig} -import com.permutive.pubsub.consumer.{ConsumerRecord, Model => ConsumerModel} -import com.permutive.pubsub.producer.grpc.{GooglePubsubProducer, PubsubProducerConfig} -import com.permutive.pubsub.producer.{PubsubProducer, Model => ProducerModel} -import fs2.Stream -import io.grpc.{ManagedChannel, ManagedChannelBuilder} -import org.http4s.client.Client -import org.http4s.okhttp.client.OkHttpBuilder -import org.scalactic.TripleEquals -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -import org.testcontainers.containers.wait.strategy.Wait -import org.typelevel.log4cats.Logger - -import scala.concurrent.duration._ - -trait PubSubSpec extends AnyFlatSpec with ForAllTestContainer with Matchers with TripleEquals { - - implicit val logger: Logger[IO] - implicit val ioRuntime: IORuntime = IORuntime.global - - val project = "test-project" - val topic = "example-topic" - val subscription = "example-subscription" - - override val container: GenericContainer = - GenericContainer( - "google/cloud-sdk:311.0.0", // newer version don't work for some reason - exposedPorts = Seq(8085), - waitStrategy = Wait.forLogMessage("(?s).*started.*$", 1), - command = s"gcloud beta emulators pubsub start --project=$project --host-port 0.0.0.0:8085" - .split(" ") - .toSeq - ) - - override def afterStart(): Unit = - updateEnv("PUBSUB_EMULATOR_HOST", s"localhost:${container.mappedPort(8085)}") - - def providers: Resource[IO, (TransportChannelProvider, CredentialsProvider)] = - Resource - .make( - IO { - ManagedChannelBuilder - .forAddress("0.0.0.0", container.mappedPort(8085)) - .usePlaintext() - .build(): ManagedChannel - } - )(ch => IO.blocking(ch.shutdown()).void) - .map { channel => - val channelProvider: FixedTransportChannelProvider = - FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)) - val credentialsProvider: NoCredentialsProvider = NoCredentialsProvider.create - - (channelProvider: TransportChannelProvider, credentialsProvider: CredentialsProvider) - } - - def topicAdminClient( - transportChannelProvider: TransportChannelProvider, - credentialsProvider: CredentialsProvider, - ): Resource[IO, TopicAdminClient] = - Resource.fromAutoCloseable( - IO( - TopicAdminClient.create( - TopicAdminSettings - .newBuilder() - .setTransportChannelProvider(transportChannelProvider) - .setCredentialsProvider(credentialsProvider) - .build() - ) - ) - ) - - def createTopic(projectId: String, topicId: String): IO[Topic] = - providers - .flatMap { case (transport, creds) => topicAdminClient(transport, creds) } - .use(client => IO.blocking(client.createTopic(TopicName.of(projectId, topicId)))) - .flatTap(topic => IO.println(s"Topic: $topic")) - - def deleteTopic(client: TopicAdminClient, topic: TopicName): IO[Unit] = - IO.blocking(client.deleteTopic(topic)) - - def subscriptionAdminClient( - transportChannelProvider: TransportChannelProvider, - credentialsProvider: CredentialsProvider, - ): Resource[IO, SubscriptionAdminClient] = - Resource.fromAutoCloseable( - IO( - SubscriptionAdminClient.create( - SubscriptionAdminSettings - .newBuilder() - .setTransportChannelProvider(transportChannelProvider) - .setCredentialsProvider(credentialsProvider) - .build() - ) - ) - ) - - def deleteSubscription(client: SubscriptionAdminClient, sub: SubscriptionName): IO[Unit] = - IO.blocking(client.deleteSubscription(sub)) - - def createSubscription( - projectId: String, - topicId: String, - subscription: String, - ackDeadlineSeconds: Int, - ): IO[Subscription] = - providers - .flatMap { case (transport, creds) => subscriptionAdminClient(transport, creds) } - .use(client => - IO.blocking( - client.createSubscription( - ProjectSubscriptionName.format(projectId, subscription), - TopicName.format(projectId, topicId), - PushConfig.getDefaultInstance, - ackDeadlineSeconds - ) - ) - ) - .flatTap(sub => IO.println(s"Sub: $sub")) - - def client: Resource[IO, Client[IO]] = - OkHttpBuilder - .withDefaultClient[IO] - .flatMap(_.resource) - - def producer( - project: String = project, - topic: String = topic - ): Resource[IO, PubsubProducer[IO, ValueHolder]] = - providers - .flatMap { case (transportChannelProvider, credentialsProvider) => - GooglePubsubProducer.of[IO, ValueHolder]( - ProducerModel.ProjectId(project), - ProducerModel.Topic(topic), - PubsubProducerConfig[IO]( - batchSize = 100, - delayThreshold = 100.millis, - awaitTerminatePeriod = 5.seconds, - onFailedTerminate = e => IO.println(s"Failed to terminate: got error $e"), - customizePublisher = Some { - _.setChannelProvider(transportChannelProvider).setCredentialsProvider(credentialsProvider) - } - ) - ) - } - - def consumer( - project: String = project, - subscription: String = subscription, - ): Stream[IO, ConsumerRecord[IO, ValueHolder]] = - for { - (transportChannelProvider, credentialsProvider) <- Stream.resource(providers) - records <- PubsubGoogleConsumer.subscribe[IO, ValueHolder]( - ConsumerModel.ProjectId(project), - ConsumerModel.Subscription(subscription), - (_, _, _, _) => IO.unit, - PubsubGoogleConsumerConfig[IO]( - onFailedTerminate = _ => IO.unit, - customizeSubscriber = Some { - _.setChannelProvider(transportChannelProvider).setCredentialsProvider(credentialsProvider) - } - ) - ) - } yield records - - def updateEnv(name: String, value: String): Unit = { - val env = System.getenv - val field = env.getClass.getDeclaredField("m") - field.setAccessible(true) - field.get(env).asInstanceOf[java.util.Map[String, String]].put(name, value) - () - } -} diff --git a/fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/ValueHolder.scala b/fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/ValueHolder.scala deleted file mode 100644 index 99e449be..00000000 --- a/fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/ValueHolder.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub - -import com.permutive.pubsub.consumer.decoder.MessageDecoder -import com.permutive.pubsub.producer.encoder.MessageEncoder -import scala.util.Try - -case class ValueHolder(value: String) extends AnyVal - -object ValueHolder { - implicit val decoder: MessageDecoder[ValueHolder] = (bytes: Array[Byte]) => { - Try(ValueHolder(new String(bytes))).toEither - } - - implicit val encoder: MessageEncoder[ValueHolder] = - (a: ValueHolder) => Right(a.value.getBytes()) -} diff --git a/fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/consumer/grpc/SimpleDriver.scala b/fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/consumer/grpc/SimpleDriver.scala deleted file mode 100644 index 9a67d5ad..00000000 --- a/fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/consumer/grpc/SimpleDriver.scala +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer.grpc - -import cats.effect.{ExitCode, IO, IOApp} -import com.permutive.pubsub.consumer.Model -import com.permutive.pubsub.consumer.decoder.MessageDecoder - -object SimpleDriver extends IOApp { - case class ValueHolder(value: String) extends AnyVal - - implicit val decoder: MessageDecoder[ValueHolder] = (bytes: Array[Byte]) => { - Right(ValueHolder(new String(bytes))) - } - - override def run(args: List[String]): IO[ExitCode] = { - val stream = for { - _ <- - PubsubGoogleConsumer - .subscribe[IO, ValueHolder]( - Model.ProjectId("test-project"), - Model.Subscription("example-sub"), - (msg, err, ack, _) => IO(println(s"Msg $msg got error $err")) >> ack, - config = PubsubGoogleConsumerConfig( - onFailedTerminate = _ => IO.unit - ) - ) - .evalTap(t => t.ack >> IO(println(s"Got: ${t.value}"))) - } yield ExitCode.Success - - stream.compile.lastOrError - } -} diff --git a/fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/producer/grpc/PubsubProducerExample.scala b/fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/producer/grpc/PubsubProducerExample.scala deleted file mode 100644 index ea318c56..00000000 --- a/fs2-google-pubsub-grpc/src/test/scala/com/permutive/pubsub/producer/grpc/PubsubProducerExample.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.grpc - -import cats.effect.{ExitCode, IO, IOApp} -import com.permutive.pubsub.producer.Model -import com.permutive.pubsub.producer.encoder.MessageEncoder - -import scala.concurrent.duration._ - -object PubsubProducerExample extends IOApp { - - case class Value(v: Int) extends AnyVal - - implicit val encoder: MessageEncoder[Value] = new MessageEncoder[Value] { - override def encode(a: Value): Either[Throwable, Array[Byte]] = - Right(BigInt(a.v).toByteArray) - } - - override def run(args: List[String]): IO[ExitCode] = - GooglePubsubProducer - .of[IO, Value]( - Model.ProjectId("test-project"), - Model.Topic("values"), - config = PubsubProducerConfig[IO]( - batchSize = 100, - delayThreshold = 100.millis, - onFailedTerminate = e => IO(println(s"Got error $e")) >> IO.unit - ) - ) - .use { producer => - producer.produce( - Value(10) - ) - } - .map(_ => ExitCode.Success) -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubHttpConsumer.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubHttpConsumer.scala deleted file mode 100644 index 8a7ddf47..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubHttpConsumer.scala +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer.http - -import cats.Applicative -import cats.effect.kernel.Async -import cats.syntax.all._ -import com.permutive.pubsub.consumer.ConsumerRecord -import com.permutive.pubsub.consumer.Model.{ProjectId, Subscription} -import com.permutive.pubsub.consumer.decoder.MessageDecoder -import com.permutive.pubsub.consumer.http.internal.Model.InternalRecord -import com.permutive.pubsub.consumer.http.internal.PubsubSubscriber -import fs2.Stream -import org.typelevel.log4cats.Logger -import org.http4s.client.Client -import org.http4s.client.middleware.RetryPolicy -import org.http4s.client.middleware.RetryPolicy.{exponentialBackoff, recklesslyRetriable} - -import java.util.Base64 -import scala.concurrent.duration._ - -object PubsubHttpConsumer { - - /** - * Subscribe with manual acknowledgement - * - * @param projectId google cloud project id - * @param subscription name of the subscription - * @param serviceAccountPath path to the Google Service account file (json), if not specified then the GCP metadata - * endpoint is used to retrieve the `default` service account access token - * @param errorHandler upon failure to decode, an exception is thrown. Allows acknowledging the message. - * - * See the following for documentation on GCP metadata endpoint and service accounts: - * - https://cloud.google.com/compute/docs/storing-retrieving-metadata - * - https://cloud.google.com/compute/docs/metadata/default-metadata-values - * - https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances - */ - final def subscribe[F[_]: Async: Logger, A: MessageDecoder]( - projectId: ProjectId, - subscription: Subscription, - serviceAccountPath: Option[String], - config: PubsubHttpConsumerConfig[F], - httpClient: Client[F], - errorHandler: (PubsubMessage, Throwable, F[Unit], F[Unit]) => F[Unit], - httpClientRetryPolicy: RetryPolicy[F] = recklesslyRetryPolicy[F] - ): Stream[F, ConsumerRecord[F, A]] = - subscribeDecode[F, A, ConsumerRecord[F, A]]( - projectId, - subscription, - serviceAccountPath, - config, - httpClient, - errorHandler, - httpClientRetryPolicy, - onDecode = (record, value) => Applicative[F].pure(record.toConsumerRecord(value)), - ) - - /** - * Subscribe with automatic acknowledgement - * - * @param projectId google cloud project id - * @param subscription name of the subscription - * @param serviceAccountPath path to the Google Service account file (json), if not specified then the GCP metadata - * endpoint is used to retrieve the `default` service account access token - * @param errorHandler upon failure to decode, an exception is thrown. Allows acknowledging the message. - * - * See the following for documentation on GCP metadata endpoint and service accounts: - * - https://cloud.google.com/compute/docs/storing-retrieving-metadata - * - https://cloud.google.com/compute/docs/metadata/default-metadata-values - * - https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances - */ - final def subscribeAndAck[F[_]: Async: Logger, A: MessageDecoder]( - projectId: ProjectId, - subscription: Subscription, - serviceAccountPath: Option[String], - config: PubsubHttpConsumerConfig[F], - httpClient: Client[F], - errorHandler: (PubsubMessage, Throwable, F[Unit], F[Unit]) => F[Unit], - httpClientRetryPolicy: RetryPolicy[F] = recklesslyRetryPolicy[F] - ): Stream[F, A] = - subscribeDecode[F, A, A]( - projectId, - subscription, - serviceAccountPath, - config, - httpClient, - errorHandler, - httpClientRetryPolicy, - onDecode = (record, value) => record.ack.as(value), - ) - - /** - * Subscribe to the raw stream, receiving the the message as retrieved from PubSub - * - * @param projectId google cloud project id - * @param subscription name of the subscription - * @param serviceAccountPath path to the Google Service account file (json), if not specified then the GCP metadata - * endpoint is used to retrieve the `default` service account access token - * - * See the following for documentation on GCP metadata endpoint and service accounts: - * - https://cloud.google.com/compute/docs/storing-retrieving-metadata - * - https://cloud.google.com/compute/docs/metadata/default-metadata-values - * - https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances - */ - final def subscribeRaw[F[_]: Async: Logger]( - projectId: ProjectId, - subscription: Subscription, - serviceAccountPath: Option[String], - config: PubsubHttpConsumerConfig[F], - httpClient: Client[F], - httpClientRetryPolicy: RetryPolicy[F] = recklesslyRetryPolicy[F], - ): Stream[F, ConsumerRecord[F, PubsubMessage]] = - PubsubSubscriber - .subscribe(projectId, subscription, serviceAccountPath, config, httpClient, httpClientRetryPolicy) - .map(msg => msg.toConsumerRecord(msg.value)) - - /* - Pub/Sub requests are `POST` and thus are not considered idempotent by http4s, therefore we must - use a different retry behaviour than the default. - */ - def recklesslyRetryPolicy[F[_]]: RetryPolicy[F] = - RetryPolicy(exponentialBackoff(maxWait = 5.seconds, maxRetry = 3), (_, result) => recklesslyRetriable(result)) - - private def subscribeDecode[F[_]: Async: Logger, A: MessageDecoder, B]( - projectId: ProjectId, - subscription: Subscription, - serviceAccountPath: Option[String], - config: PubsubHttpConsumerConfig[F], - httpClient: Client[F], - errorHandler: (PubsubMessage, Throwable, F[Unit], F[Unit]) => F[Unit], - httpClientRetryPolicy: RetryPolicy[F], - onDecode: (InternalRecord[F], A) => F[B], - ): Stream[F, B] = - PubsubSubscriber - .subscribe(projectId, subscription, serviceAccountPath, config, httpClient, httpClientRetryPolicy) - .evalMapChunk[F, Option[B]](record => - MessageDecoder[A].decode(Base64.getDecoder.decode(record.value.data.getBytes)) match { - case Left(e) => errorHandler(record.value, e, record.ack, record.nack).as(None) - case Right(v) => onDecode(record, v).map(Some(_)) - } - ) - .unNone -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubHttpConsumerConfig.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubHttpConsumerConfig.scala deleted file mode 100644 index 7bb989d5..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubHttpConsumerConfig.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer.http - -import scala.concurrent.duration._ - -/** - * Configuration for the PubSub HTTP consumer. - * - * Note: It is up to the user to raise exceptions and terminate the lifecyle of services if this is desired when - * retries are exhausted. Token refreshing runs on a background fiber so raising exceptions in - * `onTokenRetriesExhausted` will have _no_ effect to the main fibers, in fact errors are swallowed entirely. - * - * @param host host of PubSub - * @param port port of PubSub - * @param isEmulator whether the target PubSub is an emulator or not - * @param oauthTokenRefreshInterval how often to refresh the Google OAuth token - * @param onTokenRefreshSuccess optional callback to execute when refreshing the token succeeds, errors are ignored - * @param onTokenRefreshError callback to execute if refreshing the token fails during retries, errors rethrown and retried - * @param oauthTokenFailureRetryDelay initial delay for retrying OAuth token retrieval - * @param oauthTokenFailureRetryNextDelay next delay for retrying OAuth token retrieval - * @param oauthTokenFailureRetryMaxAttempts how many times to attempt; will raise the last error once reached - * @param onTokenRetriesExhausted callback to execute if refreshing the token exhausts retries. - * See note above on exceptions, up to the *user* to raise exceptions in their - * service using this callback if required. - * @param acknowledgeBatchSize maximum number of acknowledge messages to send in a single batch - * @param acknowledgeBatchLatency maximum time to wait before sending an acknowledge batch - * @param readReturnImmediately when polling for messages whether to return immediately if there are none or wait (can cause higher CPU use if `true`) - * @param readMaxMessages how many messages to retrieve at once simultaneously - * @param readConcurrency how much parallelism to use when fetching messages from PubSub - */ -case class PubsubHttpConsumerConfig[F[_]]( - host: String = "pubsub.googleapis.com", - port: Int = 443, - isEmulator: Boolean = false, - onTokenRefreshSuccess: Option[F[Unit]] = None, - oauthTokenRefreshInterval: FiniteDuration = 30.minutes, - onTokenRefreshError: PartialFunction[Throwable, F[Unit]] = PartialFunction.empty, - oauthTokenFailureRetryDelay: FiniteDuration = 0.millis, - oauthTokenFailureRetryNextDelay: FiniteDuration => FiniteDuration = _ => 5.minutes, - oauthTokenFailureRetryMaxAttempts: Int = Int.MaxValue, - onTokenRetriesExhausted: PartialFunction[Throwable, F[Unit]] = PartialFunction.empty, - acknowledgeBatchSize: Int = 100, - acknowledgeBatchLatency: FiniteDuration = 1.second, - readReturnImmediately: Boolean = false, - readMaxMessages: Int = 1000, - readConcurrency: Int = 1 -) diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubMessage.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubMessage.scala deleted file mode 100644 index 8f69774e..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubMessage.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer.http - -import com.github.plokhotnyuk.jsoniter_scala.core.JsonValueCodec -import com.github.plokhotnyuk.jsoniter_scala.macros.{CodecMakerConfig, JsonCodecMaker} - -case class PubsubMessage( - data: String, - attributes: Map[String, String], - messageId: String, - publishTime: String -) - -object PubsubMessage { - implicit final val Codec: JsonValueCodec[PubsubMessage] = - JsonCodecMaker.make[PubsubMessage](CodecMakerConfig) -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/HttpPubsubReader.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/HttpPubsubReader.scala deleted file mode 100644 index c5d82f39..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/HttpPubsubReader.scala +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer.http.internal - -import cats.Applicative -import cats.effect.kernel._ -import cats.syntax.all._ -import com.github.plokhotnyuk.jsoniter_scala.core.{readFromArray, writeToArray, JsonValueCodec} -import com.github.plokhotnyuk.jsoniter_scala.macros.{CodecMakerConfig, JsonCodecMaker} -import com.permutive.pubsub.consumer.Model.{ProjectId, Subscription} -import com.permutive.pubsub.consumer.http.PubsubHttpConsumerConfig -import com.permutive.pubsub.consumer.http.internal.Model._ -import com.permutive.pubsub.http.oauth._ -import com.permutive.pubsub.http.util.RefreshableEffect -import org.http4s.Method._ -import org.http4s.Uri._ -import org.http4s._ -import org.http4s.client._ -import org.http4s.client.dsl.Http4sClientDsl -import org.http4s.headers._ -import org.typelevel.log4cats.Logger - -import scala.concurrent.duration._ -import scala.util.Try -import scala.util.control.NoStackTrace - -private[internal] class HttpPubsubReader[F[_]: Async: Logger] private ( - baseApiUrl: Uri, - client: Client[F], - requestAuthorizer: RequestAuthorizer[F], - returnImmediately: Boolean, - maxMessages: Int -) extends PubsubReader[F] { - object dsl extends Http4sClientDsl[F] - - import HttpPubsubReader._ - import dsl._ - - private def appendToUrl(s: String): Uri = - Uri.unsafeFromString(s"${baseApiUrl.renderString}:$s") - - final private[this] val pullEndpoint = appendToUrl("pull") - final private[this] val acknowledgeEndpoint = appendToUrl("acknowledge") - final private[this] val modifyDeadlineEndpoint = appendToUrl("modifyAckDeadline") - - final override val read: F[PullResponse] = { - for { - json <- Sync[F].delay( - writeToArray( - PullRequest( - returnImmediately = returnImmediately, - maxMessages = maxMessages - ) - ) - ) - req = POST(json, pullEndpoint, `Content-Type`(MediaType.application.json)) - authedReq <- requestAuthorizer.authorize(req) - resp <- client.expectOr[Array[Byte]](authedReq)(onError) - decoded <- Sync[F].delay(readFromArray[PullResponse](resp)).onError { case _ => - Logger[F].error(s"Pull response from PubSub was invalid. Body: ${new String(resp)}") - } - } yield decoded - } - - final override def ack(ackIds: List[AckId]): F[Unit] = - for { - json <- Sync[F].delay( - writeToArray( - AckRequest( - ackIds = ackIds - ) - ) - ) - req = POST(json, acknowledgeEndpoint, `Content-Type`(MediaType.application.json)) - authedReq <- requestAuthorizer.authorize(req) - _ <- client.expectOr[Array[Byte]](authedReq)(onError) - } yield () - - final override def nack(ackIds: List[AckId]): F[Unit] = - modifyDeadline(ackIds, 0.seconds) - - final override def modifyDeadline(ackId: List[AckId], by: FiniteDuration): F[Unit] = - for { - json <- Sync[F].delay( - writeToArray( - ModifyAckDeadlineRequest( - ackIds = ackId, - ackDeadlineSeconds = by.toSeconds, - ) - ) - ) - req = POST(json, modifyDeadlineEndpoint, `Content-Type`(MediaType.application.json)) - authedReq <- requestAuthorizer.authorize(req) - _ <- client.expectOr[Array[Byte]](authedReq)(onError) - } yield () - - @inline - final private def onError(resp: Response[F]): F[Throwable] = - resp - .as[Array[Byte]] - .map(arr => - Try(readFromArray[PubSubErrorResponse](arr)) - .fold(_ => PubSubError.UnparseableBody(new String(arr)), PubSubError.fromResponse) - ) -} - -private[internal] object HttpPubsubReader { - def resource[F[_]: Async: Logger]( - projectId: ProjectId, - subscription: Subscription, - serviceAccountPath: Option[String], - config: PubsubHttpConsumerConfig[F], - httpClient: Client[F], - ): Resource[F, PubsubReader[F]] = - for { - tokenProvider <- - if (config.isEmulator) Resource.pure[F, TokenProvider[F]](DefaultTokenProvider.noAuth) - else - serviceAccountPath.fold( - CachedTokenProvider - .resource( - DefaultTokenProvider.instanceMetadata(httpClient), - // GCP metadata endpoint caches tokens until 5 minutes before expiry. - // Wait until 4 minutes before expiry to refresh the token in this library. That should ensure a new - // token will be provided and have no risk of any requests using an expired token. - // See: https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#applications - // "The metadata server caches access tokens until they have 5 minutes of remaining time before they expire." - safetyPeriod = 4.minutes, - backgroundFailureHook = config.onTokenRetriesExhausted, - onNewToken = config.onTokenRefreshSuccess.map(onRefreshSuccess => - (_: AccessToken, _: FiniteDuration) => onRefreshSuccess - ), - ) - )(path => - for { - tokenProvider <- Resource.eval(DefaultTokenProvider.google(path, httpClient)) - accessTokenRefEffect <- RefreshableEffect.createRetryResource( - refresh = tokenProvider.accessToken, - refreshInterval = config.oauthTokenRefreshInterval, - onRefreshSuccess = config.onTokenRefreshSuccess.getOrElse(Applicative[F].unit), - onRefreshError = config.onTokenRefreshError, - retryDelay = config.oauthTokenFailureRetryDelay, - retryNextDelay = config.oauthTokenFailureRetryNextDelay, - retryMaxAttempts = config.oauthTokenFailureRetryMaxAttempts, - onRetriesExhausted = config.onTokenRetriesExhausted, - ) - } yield TokenProvider.instance(accessTokenRefEffect.value) - ) - } yield new HttpPubsubReader[F]( - baseApiUrl = createBaseApi(config, ProjectNameSubscription.of(projectId, subscription)), - client = httpClient, - requestAuthorizer = RequestAuthorizer.tokenProvider(tokenProvider), - returnImmediately = config.readReturnImmediately, - maxMessages = config.readMaxMessages - ) - - def createBaseApi[F[_]](config: PubsubHttpConsumerConfig[F], projectNameSubscription: ProjectNameSubscription): Uri = - Uri( - scheme = Option(if (config.port == 443) Uri.Scheme.https else Uri.Scheme.http), - authority = Option(Uri.Authority(host = RegName(config.host), port = Option(config.port))), - path = Uri.Path.unsafeFromString(s"/v1/${projectNameSubscription.value}") - ) - - sealed abstract class PubSubError(msg: String) - extends Throwable(s"Failed request to PubSub. Underlying message: $msg") - with NoStackTrace - - object PubSubError { - case object NoAckIds extends PubSubError("No ack ids specified") - case class Unknown(body: PubSubErrorResponse) extends PubSubError(body.toString) - case class UnparseableBody(body: String) extends PubSubError(s"Body could not be parsed to error response: $body") - - def fromResponse(response: PubSubErrorResponse): PubSubError = - response.error.message match { - case "No ack ids specified." => NoAckIds - case _ => Unknown(response) - } - } - - case class PubSubErrorMessage(message: String, status: String, code: Int) - case class PubSubErrorResponse(error: PubSubErrorMessage) - - object PubSubErrorResponse { - implicit final val Codec: JsonValueCodec[PubSubErrorResponse] = - JsonCodecMaker.make[PubSubErrorResponse](CodecMakerConfig) - } -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/Model.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/Model.scala deleted file mode 100644 index f753cc4c..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/Model.scala +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer.http.internal - -import com.github.plokhotnyuk.jsoniter_scala.core.JsonValueCodec -import com.github.plokhotnyuk.jsoniter_scala.macros.{CodecMakerConfig, JsonCodecMaker} -import com.permutive.pubsub.consumer.http.PubsubMessage -import com.permutive.pubsub.consumer.{ConsumerRecord, Model => PublicModel} - -import scala.concurrent.duration.FiniteDuration - -private[http] object Model { - case class ProjectNameSubscription(value: String) extends AnyVal - object ProjectNameSubscription { - def of(projectId: PublicModel.ProjectId, subscription: PublicModel.Subscription): ProjectNameSubscription = - ProjectNameSubscription(s"projects/${projectId.value}/subscriptions/${subscription.value}") - } - - trait InternalRecord[F[_]] { self => - def value: PubsubMessage - def ack: F[Unit] - def nack: F[Unit] - def extendDeadline(by: FiniteDuration): F[Unit] - - def toConsumerRecord[A](v: A): ConsumerRecord[F, A] = - new ConsumerRecord[F, A] { - override val value: A = v - override val attributes: Map[String, String] = self.value.attributes - override val ack: F[Unit] = self.ack - override val nack: F[Unit] = self.nack - - override def extendDeadline(by: FiniteDuration): F[Unit] = self.extendDeadline(by) - } - } - - case class AckId(value: String) extends AnyVal - - case class PullRequest( - returnImmediately: Boolean, - maxMessages: Int - ) - - object PullRequest { - implicit final val PullRequestCodec: JsonValueCodec[PullRequest] = - JsonCodecMaker.make[PullRequest](CodecMakerConfig) - } - - case class PullResponse( - receivedMessages: List[ReceivedMessage] - ) - - object PullResponse { - implicit final val PullResponseCodec: JsonValueCodec[PullResponse] = - JsonCodecMaker.make[PullResponse](CodecMakerConfig) - } - - case class ReceivedMessage( - ackId: AckId, - message: PubsubMessage - ) - - case class AckRequest( - ackIds: List[AckId] - ) - - object AckRequest { - implicit final val AckRequestCodec: JsonValueCodec[AckRequest] = - JsonCodecMaker.make[AckRequest](CodecMakerConfig) - } - - case class ModifyAckDeadlineRequest( - ackIds: List[AckId], - ackDeadlineSeconds: Long - ) - - object ModifyAckDeadlineRequest { - implicit final val ModifyAckDeadlineRequestCodec: JsonValueCodec[ModifyAckDeadlineRequest] = - JsonCodecMaker.make[ModifyAckDeadlineRequest](CodecMakerConfig) - } - -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/PubsubReader.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/PubsubReader.scala deleted file mode 100644 index f5c50c8b..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/PubsubReader.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer.http.internal - -import com.permutive.pubsub.consumer.http.internal.Model.{AckId, PullResponse} - -import scala.concurrent.duration.FiniteDuration - -trait PubsubReader[F[_]] { - def read: F[PullResponse] - - def ack(ackId: List[AckId]): F[Unit] - - def nack(ackId: List[AckId]): F[Unit] - - /** - * The new ack deadline with respect to the time this request was sent to the Pub/Sub system. - * For example, if the value is 10, the new ack deadline will expire 10 seconds after - * the subscriptions.modifyAckDeadline call was made. Specifying zero might immediately make the message - * available for delivery to another subscriber client. This typically results in an increase in the rate of - * message redeliveries (that is, duplicates). The minimum deadline you can specify is 0 seconds. - * The maximum deadline you can specify is 600 seconds (10 minutes). - * From: https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/modifyAckDeadline - */ - def modifyDeadline(ackId: List[AckId], by: FiniteDuration): F[Unit] -} - -object PubsubReader { - def apply[F[_]: PubsubReader]: PubsubReader[F] = implicitly -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/PubsubSubscriber.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/PubsubSubscriber.scala deleted file mode 100644 index 0ca5ce34..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/internal/PubsubSubscriber.scala +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer.http.internal - -import cats.effect.kernel._ -import cats.syntax.all._ -import cats.effect.std.Queue -import com.permutive.pubsub.consumer.Model.{ProjectId, Subscription} -import com.permutive.pubsub.consumer.http.{PubsubHttpConsumerConfig, PubsubMessage} -import com.permutive.pubsub.consumer.http.internal.HttpPubsubReader.PubSubError -import com.permutive.pubsub.consumer.http.internal.Model.{AckId, InternalRecord} -import fs2.Stream -import org.typelevel.log4cats.Logger -import org.http4s.client.Client -import org.http4s.client.middleware.{Retry, RetryPolicy} - -import scala.concurrent.duration.FiniteDuration - -private[http] object PubsubSubscriber { - - def subscribe[F[_]: Logger: Async]( - projectId: ProjectId, - subscription: Subscription, - serviceAccountPath: Option[String], - config: PubsubHttpConsumerConfig[F], - httpClient: Client[F], - httpClientRetryPolicy: RetryPolicy[F] - ): Stream[F, InternalRecord[F]] = { - val errorHandler: Throwable => F[Unit] = { - case PubSubError.NoAckIds => - Logger[F].warn(s"[PubSub/Ack] a message was sent with no ids in it. This is likely a bug.") - case PubSubError.Unknown(e) => - Logger[F].error(s"[PubSub] Unknown PubSub error occurred. Body is: $e") - case PubSubError.UnparseableBody(body) => - Logger[F].error(s"[PubSub] A response from PubSub could not be parsed. Body is: $body") - case e => - Logger[F].error(e)(s"[PubSub] An unknown error occurred") - } - - for { - ackQ <- Stream.eval(Queue.unbounded[F, AckId]) - nackQ <- Stream.eval(Queue.unbounded[F, AckId]) - reader <- Stream.resource( - HttpPubsubReader.resource( - projectId = projectId, - subscription = subscription, - serviceAccountPath = serviceAccountPath, - config = config, - httpClient = Retry(httpClientRetryPolicy)(httpClient), - ) - ) - source = - if (config.readConcurrency == 1) Stream.repeatEval(reader.read) - else Stream.emit(reader.read).repeat.covary[F].mapAsyncUnordered(config.readConcurrency)(identity) - rec <- - source - .concurrently( - Stream - .fromQueueUnterminated(ackQ) - .groupWithin(config.acknowledgeBatchSize, config.acknowledgeBatchLatency) - .evalMap(ids => reader.ack(ids.toList).handleErrorWith(errorHandler)) - .onFinalize(Logger[F].debug("[PubSub] Ack queue has exited.")) - ) - .concurrently( - Stream - .fromQueueUnterminated(nackQ) - .groupWithin(config.acknowledgeBatchSize, config.acknowledgeBatchLatency) - .evalMap(ids => reader.nack(ids.toList).handleErrorWith(errorHandler)) - .onFinalize(Logger[F].debug("[PubSub] Nack queue has exited.")) - ) - msg <- Stream.emits( - rec.receivedMessages.map { msg => - new InternalRecord[F] { - override val value: PubsubMessage = msg.message - override val ack: F[Unit] = ackQ.offer(msg.ackId) - override val nack: F[Unit] = nackQ.offer(msg.ackId) - override def extendDeadline(by: FiniteDuration): F[Unit] = reader.modifyDeadline(List(msg.ackId), by) - } - } - ) - } yield msg - } - -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/crypto/GoogleAccountParser.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/crypto/GoogleAccountParser.scala deleted file mode 100644 index 0b768dbc..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/crypto/GoogleAccountParser.scala +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.http.crypto - -import java.nio.file.{Files, Path} -import java.security.KeyFactory -import java.security.interfaces.RSAPrivateKey -import java.security.spec.PKCS8EncodedKeySpec -import java.util.Base64 -import java.util.regex.Pattern - -import com.github.plokhotnyuk.jsoniter_scala.core._ -import com.github.plokhotnyuk.jsoniter_scala.macros._ - -import scala.util.Try - -object GoogleAccountParser { - case class JsonGoogleServiceAccount( - `type`: String, - projectId: String, - privateKeyId: String, - privateKey: String, - clientEmail: String, - authUri: String - ) - - object JsonGoogleServiceAccount { - implicit final val codec: JsonValueCodec[JsonGoogleServiceAccount] = - JsonCodecMaker.make[JsonGoogleServiceAccount]( - CodecMakerConfig.withFieldNameMapper(JsonCodecMaker.enforce_snake_case) - ) - } - - final def parse(path: Path): Either[Throwable, GoogleServiceAccount] = - Try { - val serviceAccount = readFromArray[JsonGoogleServiceAccount](Files.readAllBytes(path)) - val spec = new PKCS8EncodedKeySpec(loadPem(serviceAccount.privateKey)) - val kf = KeyFactory.getInstance("RSA") - GoogleServiceAccount( - clientEmail = serviceAccount.clientEmail, - privateKey = kf.generatePrivate(spec).asInstanceOf[RSAPrivateKey] - ) - }.toEither - - final private[this] val privateKeyPattern = Pattern.compile("(?m)(?s)^---*BEGIN.*---*$(.*)^---*END.*---*$.*") - - private def loadPem(pem: String): Array[Byte] = { - val encoded = privateKeyPattern.matcher(pem).replaceFirst("$1") - Base64.getMimeDecoder.decode(encoded) - } -} - -case class GoogleServiceAccount(clientEmail: String, privateKey: RSAPrivateKey) diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/AccessToken.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/AccessToken.scala deleted file mode 100644 index 3062a46e..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/AccessToken.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.http.oauth - -import com.github.plokhotnyuk.jsoniter_scala.core._ -import com.github.plokhotnyuk.jsoniter_scala.macros._ - -final case class AccessToken(accessToken: String, tokenType: String, expiresIn: Int) - -object AccessToken { - implicit final val codec: JsonValueCodec[AccessToken] = - JsonCodecMaker.make[AccessToken]( - CodecMakerConfig.withFieldNameMapper(JsonCodecMaker.enforce_snake_case) - ) -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/CachedTokenProvider.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/CachedTokenProvider.scala deleted file mode 100644 index 90b88760..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/CachedTokenProvider.scala +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.http.oauth - -import cats.effect.kernel.{Resource, Temporal} -import com.permutive.pubsub.http.util.RefCache - -import java.util.concurrent.TimeUnit -import scala.concurrent.duration.FiniteDuration - -object CachedTokenProvider { - - /** - * Generate a cached token provider from an underlying provider. - * - * @param underlying the underlying token provider to use when a new token is required - * @param safetyPeriod how much time less than the indicated expiry to cache a token for; this is to give a - * safety buffer to ensure an expired token is never used in a request - * @param backgroundFailureHook hook called if the background fiber refreshing the token fails - * @param onNewToken a callback invoked whenever a new token is generated, the [[scala.concurrent.duration.FiniteDuration]] - * is the period that will be waited before the next new token - */ - def resource[F[_]: Temporal]( - underlying: TokenProvider[F], - safetyPeriod: FiniteDuration, - backgroundFailureHook: PartialFunction[Throwable, F[Unit]], - onNewToken: Option[(AccessToken, FiniteDuration) => F[Unit]] = None, - ): Resource[F, TokenProvider[F]] = { - val cacheDuration: AccessToken => FiniteDuration = token => - // GCP access token lifetimes are specified in seconds. - // If this is a negative amount then the sleep in `RefCache` will be for no time, it will not error. - FiniteDuration(token.expiresIn.toLong, TimeUnit.SECONDS) - safetyPeriod - - RefCache - .resource(underlying.accessToken, cacheDuration, backgroundFailureHook, onNewValue = onNewToken) - .map(TokenProvider.instance) - } - -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/DefaultTokenProvider.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/DefaultTokenProvider.scala deleted file mode 100644 index 7e0da6ef..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/DefaultTokenProvider.scala +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.http.oauth - -import cats.MonadError -import cats.effect.kernel.{Async, Clock} -import cats.syntax.all._ -import com.permutive.pubsub.http.crypto.GoogleAccountParser -import org.http4s.client.Client -import org.typelevel.log4cats.Logger - -import java.io.File - -class DefaultTokenProvider[F[_]: Clock]( - emailAddress: String, - scope: List[String], - auth: OAuth[F] -)(implicit F: MonadError[F, Throwable]) - extends TokenProvider[F] { - override val accessToken: F[AccessToken] = { - for { - now <- Clock[F].realTimeInstant - token <- auth.authenticate( - emailAddress, - scope.mkString(","), - now.plusMillis(auth.maxDuration.toMillis), - now - ) - tokenOrError <- token.fold(F.raiseError[AccessToken](TokenProvider.FailedToGetToken))(_.pure[F]) - } yield tokenOrError - } -} - -object DefaultTokenProvider { - final private val scope = List("https://www.googleapis.com/auth/pubsub") - - def google[F[_]: Logger: Async]( - serviceAccountPath: String, - httpClient: Client[F] - ): F[TokenProvider[F]] = - for { - serviceAccount <- GoogleAccountParser.parse(new File(serviceAccountPath).toPath).liftTo[F] - } yield new DefaultTokenProvider( - serviceAccount.clientEmail, - scope, - new GoogleOAuth(serviceAccount.privateKey, httpClient) - ) - - def instanceMetadata[F[_]: Async: Logger](httpClient: Client[F]): TokenProvider[F] = - new DefaultTokenProvider[F]("instance-metadata", scope, new InstanceMetadataOAuth[F](httpClient)) - - def noAuth[F[_]: Clock](implicit F: MonadError[F, Throwable]): TokenProvider[F] = - new DefaultTokenProvider("noop", Nil, new NoopOAuth) -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/GoogleOAuth.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/GoogleOAuth.scala deleted file mode 100644 index 544669e1..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/GoogleOAuth.scala +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.http.oauth - -import cats.Applicative - -import java.security.interfaces.{RSAPrivateKey, RSAPublicKey} -import java.time.Instant -import java.util.Date -import cats.effect.kernel.{Async, Sync} -import cats.syntax.all._ -import com.auth0.jwt.JWT -import com.auth0.jwt.algorithms.Algorithm -import com.github.plokhotnyuk.jsoniter_scala.core.readFromArray -import org.typelevel.log4cats.Logger -import org.http4s.Method.POST -import org.http4s.client.Client -import org.http4s.client.dsl.Http4sClientDsl -import org.http4s._ - -import scala.concurrent.duration._ -import scala.util.control.NoStackTrace - -class GoogleOAuth[F[_]: Async: Logger]( - key: RSAPrivateKey, - httpClient: Client[F] -) extends OAuth[F] { - import GoogleOAuth._ - - object clientDsl extends Http4sClientDsl[F] - import clientDsl._ - - final private[this] val algorithm = Algorithm.RSA256(null: RSAPublicKey, key) - final private[this] val googleOAuthDomainStr = "https://www.googleapis.com/oauth2/v4/token" - final private[this] val googleOAuthDomain = Uri.unsafeFromString(googleOAuthDomainStr) - - final override def authenticate( - iss: String, - scope: String, - exp: Instant, - iat: Instant - ): F[Option[AccessToken]] = { - val tokenF = Sync[F].delay( - JWT.create - .withIssuedAt(Date.from(iat)) - .withExpiresAt(Date.from(exp)) - .withAudience(googleOAuthDomainStr) - .withClaim("scope", scope) - .withClaim("iss", iss) - .sign(algorithm) - ) - - val request = - tokenF.map { token => - POST( - UrlForm( - "grant_type" -> "urn:ietf:params:oauth:grant-type:jwt-bearer", - "assertion" -> token - ), - googleOAuthDomain - ) - } - - httpClient - .expectOr[Array[Byte]](request) { resp => - resp.as[String].map(FailedRequest.apply) - } - .flatMap(bytes => Sync[F].delay(readFromArray[AccessToken](bytes)).map(_.some)) - .handleErrorWith { e => - Logger[F].warn(e)("Failed to retrieve JWT Access Token from Google") >> Applicative[F].pure(none[AccessToken]) - } - } - - final override val maxDuration: FiniteDuration = 1.hour -} - -object GoogleOAuth { - case class FailedRequest(body: String) - extends RuntimeException(s"Failed request, got response: $body") - with NoStackTrace -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/InstanceMetadataOAuth.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/InstanceMetadataOAuth.scala deleted file mode 100644 index 3d63115c..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/InstanceMetadataOAuth.scala +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.http.oauth - -import cats.effect.kernel.{Async, Sync} -import cats.syntax.applicativeError._ -import cats.syntax.flatMap._ -import cats.syntax.functor._ -import cats.syntax.option._ -import com.github.plokhotnyuk.jsoniter_scala.core.readFromArray -import com.permutive.pubsub.http.oauth.GoogleOAuth.FailedRequest -import org.http4s.Method.GET -import org.http4s.Uri -import org.http4s.client.Client -import org.http4s.client.dsl.Http4sClientDsl -import org.typelevel.log4cats.Logger - -import java.time.Instant -import scala.concurrent.duration._ -import scala.util.control.NoStackTrace - -// Obtains OAuth token from instance metadata -// https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#applications -class InstanceMetadataOAuth[F[_]: Async: Logger](httpClient: Client[F]) extends OAuth[F] with Http4sClientDsl[F] { - - // https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#applications - final private[this] val googleInstanceMetadataTokenUri = Uri.unsafeFromString( - "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" - ) - - final private[this] val request = - GET(googleInstanceMetadataTokenUri, "Metadata-Flavor" -> "Google") - - private[this] val doAuthenticate: F[Option[AccessToken]] = - httpClient - .expectOr[Array[Byte]](request) { resp => - resp.as[String].map(FailedRequest.apply) - } - .flatMap(bytes => Sync[F].delay(readFromArray[AccessToken](bytes)).map(_.some)) - .handleErrorWith(Logger[F].warn(_)("Failed to retrieve JWT Access Token from Google").as(None)) - - override def authenticate(iss: String, scope: String, exp: Instant, iat: Instant): F[Option[AccessToken]] = - doAuthenticate - - final override val maxDuration: FiniteDuration = 1.hour -} - -object InstanceMetadataOAuth { - case class FailedRequest(body: String) - extends RuntimeException(s"Failed request, got response: $body") - with NoStackTrace -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/NoopOAuth.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/NoopOAuth.scala deleted file mode 100644 index 0ebfc2d4..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/NoopOAuth.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.http.oauth - -import java.time.Instant - -import cats.Applicative - -import scala.concurrent.duration._ - -class NoopOAuth[F[_]](implicit F: Applicative[F]) extends OAuth[F] { - final override def authenticate(iss: String, scope: String, exp: Instant, iat: Instant): F[Option[AccessToken]] = - F.pure(Some(AccessToken("noop", "noop", 3600))) - - final override val maxDuration: FiniteDuration = 1.hour -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/OAuth.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/OAuth.scala deleted file mode 100644 index 17b37855..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/OAuth.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.http.oauth - -import java.time.Instant - -import scala.concurrent.duration.FiniteDuration - -trait OAuth[F[_]] { - - /** - * Based on https://developers.google.com/identity/protocols/OAuth2ServiceAccount - * @param iss The email address of the service account. - * @param scope A space-delimited list of the permissions that the application requests. - * @param exp The expiration time of the assertion, specified as milliseconds since 00:00:00 UTC, January 1, 1970. - * @param iat The time the assertion was issued, specified as milliseconds since 00:00:00 UTC, January 1, 1970. - */ - def authenticate( - iss: String, - scope: String, - exp: Instant = Instant.now().plusMillis(maxDuration.toMillis), - iat: Instant = Instant.now() - ): F[Option[AccessToken]] - - def maxDuration: FiniteDuration -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/RequestAuthorizer.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/RequestAuthorizer.scala deleted file mode 100644 index 9dfcf896..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/RequestAuthorizer.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.http.oauth - -import cats.syntax.all._ -import cats.{Applicative, Functor} -import org.http4s.{AuthScheme, Credentials, Request} -import org.http4s.headers.Authorization - -// This could be implemented as client middleware but we think that could be a footgun; a client using auth middleware -// could easily be used in the wrong place by accident. -trait RequestAuthorizer[F[_]] { - def authorize(request: Request[F]): F[Request[F]] -} - -object RequestAuthorizer { - - def tokenProvider[F[_]: Functor](tokenProvider: TokenProvider[F]): RequestAuthorizer[F] = - new RequestAuthorizer[F] { - override def authorize(request: Request[F]): F[Request[F]] = - tokenProvider.accessToken.map(token => - request.putHeaders(Authorization(Credentials.Token(AuthScheme.Bearer, token.accessToken))) - ) - } - - def noop[F[_]: Applicative]: RequestAuthorizer[F] = new RequestAuthorizer[F] { - override def authorize(request: Request[F]): F[Request[F]] = Applicative[F].pure(request) - } - -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/TokenProvider.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/TokenProvider.scala deleted file mode 100644 index 60311a0c..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth/TokenProvider.scala +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.http.oauth - -import scala.util.control.NoStackTrace - -trait TokenProvider[F[_]] { - def accessToken: F[AccessToken] -} - -object TokenProvider { - case object TokenValidityTooLong - extends RuntimeException("Valid for duration cannot be longer than maximum of the OAuth provider") - with NoStackTrace - - case object FailedToGetToken extends RuntimeException("Failed to get token after many attempts") - - def instance[F[_]](token: F[AccessToken]): TokenProvider[F] = new TokenProvider[F] { - override val accessToken: F[AccessToken] = token - } -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/util/RefCache.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/util/RefCache.scala deleted file mode 100644 index 252896f0..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/util/RefCache.scala +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.http.util - -import cats.Applicative -import cats.effect.kernel.{Ref, Resource, Temporal} -import cats.effect.syntax.all._ -import cats.syntax.all._ - -import scala.concurrent.duration.FiniteDuration - -object RefCache { - - /** - * Caches a single instance of type `A` for a period of time before refreshing it automatically. - * - * The time between refreshes is dynamic and based on the value of each `A` itself. This is similar to - * [[RefreshableEffect]] except that only exposes a fixed refresh frequency. - * - * An old value is only made unavailable _after_ a new value has been acquired. This means that the time each value - * is exposed for is `cacheDuration` plus the time to evaluate `fa`. - * - * @param fa generate a new value of `A` - * @param cacheDuration how long to cache a newly generated value of `A` for, if an effect is needed to - * calculate this it should be part of `fa` - * @param backgroundFailureHook what to do if retrying to refresh the value fails, up to user handle failing their - * service, the refresh fiber will have stopped at this point - * @param onNewValue a callback invoked whenever a new value is generated, the [[scala.concurrent.duration.FiniteDuration]] - * is the period that will be waited before the next new value - */ - def resource[F[_]: Temporal, A]( - fa: F[A], - cacheDuration: A => FiniteDuration, - backgroundFailureHook: PartialFunction[Throwable, F[Unit]], - onNewValue: Option[(A, FiniteDuration) => F[Unit]] = None, - ): Resource[F, F[A]] = { - val newValueHook: (A, FiniteDuration) => F[Unit] = - onNewValue.getOrElse((_, _) => Applicative[F].unit) - - for { - initialA <- Resource.eval(fa) - ref <- Resource.eval(Ref.of(initialA)) - // `.background` means the refresh loop runs in a fiber, but leaving the scope of the `Resource` will cancel - // it for us. Use the provided callback if a failure occurs in the background fiber, there is no other way to - // signal a failure from the background. - _ <- refreshLoop(initialA, fa, ref, cacheDuration, newValueHook).onError(backgroundFailureHook).attempt.background - } yield ref.get - } - - private def refreshLoop[F[_]: Temporal, A]( - initialA: A, - fa: F[A], - ref: Ref[F, A], - cacheDuration: A => FiniteDuration, - onNewValue: (A, FiniteDuration) => F[Unit], - ): F[Unit] = { - def innerLoop(currentA: A): F[Unit] = { - val duration = cacheDuration(currentA) - for { - _ <- onNewValue(currentA, duration) - _ <- Temporal[F].sleep(duration) - // Note the old value is only removed from the `Ref` after we have acquired a new value. - // We could remove the old value instantly if this implementation also used a `Deferred` and consumers block on - // the empty deferred during acquisition of a new value. This would lead to edge cases that would be unpleasant - // though; for example we'd need to handle the case of failing to acquire a new value ensuring consumers do not - // block on an empty deferred forever. - newA <- fa - _ <- ref.set(newA) - _ <- innerLoop(newA) - } yield () - } - - innerLoop(initialA) - } - -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/util/RefreshableEffect.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/util/RefreshableEffect.scala deleted file mode 100644 index 6539dda6..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/util/RefreshableEffect.scala +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.http.util - -import cats.MonadError -import cats.effect.kernel.{Ref, Resource, Temporal} -import cats.effect.syntax.all._ -import cats.syntax.all._ -import fs2.Stream - -import scala.concurrent.duration.FiniteDuration - -/** - * Represents a value of type `A` with effects in `F` which is refreshed. - * - * Refreshing can be cancelled by evaluating `cancelToken`. - * - * Implementation is backed by a `cats-effect` `Ref` so evaluating the value is fast. - */ -final class RefreshableEffect[F[_], A] private (val value: F[A], val cancelToken: F[Unit]) - -object RefreshableEffect { - - /** - * Create a refreshable effect which exposes the result of `refresh`, retries - * if refreshing the value fails. - * - * @param refreshInterval how frequently to refresh the value - * @param onRefreshError what to do if refreshing the value fails, error is always rethrown - * @param onRefreshSuccess what to do when the value is successfully refresh, errors are ignored - * @param retryDelay duration of delay before the first retry - * @param retryNextDelay what value to delay before the next retry - * @param retryMaxAttempts how many attempts to make before failing with last error - * @param onRetriesExhausted what to do if retrying to refresh the value fails, up to user handle failing their service - */ - def createRetryResource[F[_]: Temporal, A]( - refresh: F[A], - refreshInterval: FiniteDuration, - onRefreshSuccess: F[Unit], - onRefreshError: PartialFunction[Throwable, F[Unit]], - retryDelay: FiniteDuration, - retryNextDelay: FiniteDuration => FiniteDuration, - retryMaxAttempts: Int, - onRetriesExhausted: PartialFunction[Throwable, F[Unit]], - ): Resource[F, RefreshableEffect[F, A]] = { - val updateRef: Ref[F, A] => F[Unit] = - ref => - retry( - updateUnhandled(refresh, ref, onRefreshSuccess).onError(onRefreshError), - retryDelay, - retryNextDelay, - retryMaxAttempts, - ).onError(onRetriesExhausted).attempt.void // Swallow errors entirely, retry will loop around again - - Resource.make(createAndSchedule(refresh, refreshInterval, updateRef))(_.cancelToken) - } - - private def createAndSchedule[F[_]: Temporal, A]( - refresh: F[A], - refreshInterval: FiniteDuration, - updateRef: Ref[F, A] => F[Unit], - ): F[RefreshableEffect[F, A]] = - for { - initial <- refresh - ref <- Ref.of(initial) - fiber <- scheduleRefresh(updateRef(ref), refreshInterval).start - } yield new RefreshableEffect[F, A](ref.get, fiber.cancel) - - private def scheduleRefresh[F[_]: Temporal, A]( - refreshEffect: F[Unit], - refreshInterval: FiniteDuration, - ): F[Unit] = - Stream - .fixedRate(refreshInterval) // Same frequency regardless of time to evaluate refresh - .evalMap(_ => refreshEffect) - .compile - .drain - - private def updateUnhandled[F[_], A](refresh: F[A], ref: Ref[F, A], onRefreshSuccess: F[Unit])(implicit - ME: MonadError[F, Throwable] - ): F[Unit] = - for { - refreshed <- refresh - _ <- ref.set(refreshed) - _ <- onRefreshSuccess.attempt // Ignore exceptions in success callback - } yield () - - private def retry[F[_]: Temporal, A]( - refreshEffect: F[Unit], - retryDelay: FiniteDuration, - retryNextDelay: FiniteDuration => FiniteDuration, - retryMaxAttempts: Int, - ): F[Unit] = - if (retryMaxAttempts < 1) - refreshEffect - else - Stream - .retry( - refreshEffect, - retryDelay, - retryNextDelay, - retryMaxAttempts, - ) - .compile - .lastOrError - -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/BatchingHttpProducerConfig.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/BatchingHttpProducerConfig.scala deleted file mode 100644 index a0aa67bb..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/BatchingHttpProducerConfig.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.http - -import scala.concurrent.duration.FiniteDuration - -case class BatchingHttpProducerConfig( - batchSize: Int, - maxLatency: FiniteDuration, - retryTimes: Int, - retryInitialDelay: FiniteDuration, - retryNextDelay: FiniteDuration => FiniteDuration -) diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/BatchingHttpPubsubProducer.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/BatchingHttpPubsubProducer.scala deleted file mode 100644 index f10a3787..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/BatchingHttpPubsubProducer.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.http - -import cats.effect.kernel.{Async, Resource} -import com.permutive.pubsub.producer.encoder.MessageEncoder -import com.permutive.pubsub.producer.http.internal.{BatchingHttpPublisher, DefaultHttpPublisher} -import com.permutive.pubsub.producer.{AsyncPubsubProducer, Model} -import org.typelevel.log4cats.Logger -import org.http4s.client.Client - -object BatchingHttpPubsubProducer { - - /** - * Create an HTTP PubSub producer which produces in batches. - * - * @param projectId google cloud project id - * @param topic the topic to produce to - * @param googleServiceAccountPath path to the Google Service account file (json), if not specified then the GCP - * metadata endpoint is used to retrieve the `default` service account access token - * - * See the following for documentation on GCP metadata endpoint and service accounts: - * - https://cloud.google.com/compute/docs/storing-retrieving-metadata - * - https://cloud.google.com/compute/docs/metadata/default-metadata-values - * - https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances - */ - def resource[F[_]: Async: Logger, A: MessageEncoder]( - projectId: Model.ProjectId, - topic: Model.Topic, - googleServiceAccountPath: Option[String], - config: PubsubHttpProducerConfig[F], - batchingConfig: BatchingHttpProducerConfig, - httpClient: Client[F] - ): Resource[F, AsyncPubsubProducer[F, A]] = - for { - publisher <- DefaultHttpPublisher.resource( - projectId = projectId, - topic = topic, - serviceAccountPath = googleServiceAccountPath, - config = config, - httpClient = httpClient - ) - batching <- BatchingHttpPublisher.resource( - publisher = publisher, - config = batchingConfig - ) - } yield batching -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/HttpPubsubProducer.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/HttpPubsubProducer.scala deleted file mode 100644 index b3cd8f8a..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/HttpPubsubProducer.scala +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.http - -import cats.effect.kernel.{Async, Resource} -import com.permutive.pubsub.producer.encoder.MessageEncoder -import com.permutive.pubsub.producer.http.internal.DefaultHttpPublisher -import com.permutive.pubsub.producer.{Model, PubsubProducer} -import org.typelevel.log4cats.Logger -import org.http4s.client.Client - -object HttpPubsubProducer { - - /** - * Create an HTTP PubSub producer which does not batch. - * - * @param projectId google cloud project id - * @param topic the topic to produce to - * @param googleServiceAccountPath path to the Google Service account file (json), if not specified then the GCP - * metadata endpoint is used to retrieve the `default` service account access token - * - * See the following for documentation on GCP metadata endpoint and service accounts: - * - https://cloud.google.com/compute/docs/storing-retrieving-metadata - * - https://cloud.google.com/compute/docs/metadata/default-metadata-values - * - https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances - */ - def resource[F[_]: Async: Logger, A: MessageEncoder]( - projectId: Model.ProjectId, - topic: Model.Topic, - googleServiceAccountPath: Option[String], - config: PubsubHttpProducerConfig[F], - httpClient: Client[F] - ): Resource[F, PubsubProducer[F, A]] = - DefaultHttpPublisher.resource( - projectId = projectId, - topic = topic, - serviceAccountPath = googleServiceAccountPath, - config = config, - httpClient = httpClient - ) -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/PubsubHttpProducerConfig.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/PubsubHttpProducerConfig.scala deleted file mode 100644 index 58f1debe..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/PubsubHttpProducerConfig.scala +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.http - -import scala.concurrent.duration._ - -/** - * Configuration for the PubSub HTTP producer. - * - * Note: It is up to the user to raise exceptions and terminate the lifecyle of services if this is desired when - * retries are exhausted. Token refreshing runs on a background fiber so raising exceptions in - * `onTokenRetriesExhausted` will have _no_ effect to the main fibers, in fact errors are swallowed entirely. - * - * @param host host of PubSub - * @param port port of PubSub - * @param isEmulator whether the target PubSub is an emulator or not - * @param oauthTokenRefreshInterval how often to refresh the Google OAuth token - * @param onTokenRefreshSuccess optional callback to execute when refreshing the token succeeds, errors are ignored - * @param onTokenRefreshError callback to execute if refreshing the token fails during retries, errors rethrown and retried - * @param oauthTokenFailureRetryDelay initial delay for retrying OAuth token retrieval - * @param oauthTokenFailureRetryNextDelay next delay for retrying OAuth token retrieval - * @param oauthTokenFailureRetryMaxAttempts how many times to attempt; will raise the last error once reached - * @param onTokenRetriesExhausted callback to execute if refreshing the token exhausts retries. - * See note above on exceptions, up to the *user* to raise exceptions in their - * service using this callback if required. - */ -case class PubsubHttpProducerConfig[F[_]]( - host: String = "pubsub.googleapis.com", - port: Int = 443, - isEmulator: Boolean = false, - oauthTokenRefreshInterval: FiniteDuration = 30.minutes, - onTokenRefreshSuccess: Option[F[Unit]] = None, - onTokenRefreshError: PartialFunction[Throwable, F[Unit]] = PartialFunction.empty, - oauthTokenFailureRetryDelay: FiniteDuration = 0.millis, - oauthTokenFailureRetryNextDelay: FiniteDuration => FiniteDuration = _ => 5.minutes, - oauthTokenFailureRetryMaxAttempts: Int = Int.MaxValue, - onTokenRetriesExhausted: PartialFunction[Throwable, F[Unit]] = PartialFunction.empty, -) diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/internal/BatchingHttpPublisher.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/internal/BatchingHttpPublisher.scala deleted file mode 100644 index a72f651d..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/internal/BatchingHttpPublisher.scala +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.http.internal - -import cats.effect.kernel.{Concurrent, Deferred, Resource, Temporal} -import cats.effect.std.{Queue, QueueSink, QueueSource} -import cats.effect.syntax.all._ -import cats.syntax.all._ -import cats.{Foldable, Traverse} -import com.permutive.pubsub.producer.Model.MessageId -import com.permutive.pubsub.producer.http.BatchingHttpProducerConfig -import com.permutive.pubsub.producer.{AsyncPubsubProducer, Model, PubsubProducer} -import fs2.{Chunk, Stream} - -private[http] class BatchingHttpPublisher[F[_]: Concurrent, A] private ( - queue: QueueSink[F, Model.AsyncRecord[F, A]] -) extends AsyncPubsubProducer[F, A] { - - override def produceAsync( - data: A, - callback: Either[Throwable, Unit] => F[Unit], - attributes: Map[String, String], - uniqueId: String - ): F[Unit] = - queue.offer(Model.AsyncRecord(data, callback, attributes, uniqueId)) - - override def produceManyAsync[G[_]: Foldable](records: G[Model.AsyncRecord[F, A]]): F[Unit] = - records.traverse_(queue.offer) - - override def produce( - data: A, - attributes: Map[String, String], - uniqueId: String - ): F[F[Unit]] = - produceAsync(Model.SimpleRecord(data, attributes, uniqueId)) - - override def produceMany[G[_]: Traverse](records: G[Model.SimpleRecord[A]]): F[G[F[Unit]]] = - records.traverse(produceAsync) - - private def produceAsync(record: Model.SimpleRecord[A]): F[F[Unit]] = - for { - d <- Deferred[F, Either[Throwable, Unit]] - _ <- queue.offer(Model.AsyncRecord(record.data, d.complete(_).void, record.attributes, record.uniqueId)) - } yield d.get.rethrow - -} - -private[http] object BatchingHttpPublisher { - def resource[F[_]: Temporal, A]( - publisher: PubsubProducer[F, A], - config: BatchingHttpProducerConfig - ): Resource[F, AsyncPubsubProducer[F, A]] = - for { - queue <- Resource.eval(Queue.unbounded[F, Model.AsyncRecord[F, A]]) - _ <- Resource.make(consume(publisher, config, queue).start)(_.cancel) - } yield new BatchingHttpPublisher(queue) - - private def consume[F[_]: Temporal, A]( - underlying: PubsubProducer[F, A], - config: BatchingHttpProducerConfig, - queue: QueueSource[F, Model.AsyncRecord[F, A]] - ): F[Unit] = { - val handler: Chunk[Model.AsyncRecord[F, A]] => F[List[MessageId]] = - if (config.retryTimes == 0) { records => - underlying.produceMany[Chunk](records) - } else { records => - Stream - .retry( - underlying.produceMany[Chunk](records), - delay = config.retryInitialDelay, - nextDelay = config.retryNextDelay, - maxAttempts = config.retryTimes - ) - .compile - .lastOrError - } - - Stream - .fromQueueUnterminated(queue) - .groupWithin(config.batchSize, config.maxLatency) - .evalMap { asyncRecords => - handler(asyncRecords).void.attempt - .flatMap(etu => asyncRecords.traverse_(_.callback(etu))) - } - .compile - .drain - } -} diff --git a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/internal/DefaultHttpPublisher.scala b/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/internal/DefaultHttpPublisher.scala deleted file mode 100644 index b9eae7ce..00000000 --- a/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/internal/DefaultHttpPublisher.scala +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.http.internal - -import alleycats.syntax.foldable._ -import cats.effect.kernel._ -import cats.syntax.all._ -import cats.{Applicative, Foldable, Traverse} -import com.github.plokhotnyuk.jsoniter_scala.core._ -import com.github.plokhotnyuk.jsoniter_scala.macros._ -import com.permutive.pubsub.http.oauth._ -import com.permutive.pubsub.http.util.RefreshableEffect -import com.permutive.pubsub.producer.Model.MessageId -import com.permutive.pubsub.producer.encoder.MessageEncoder -import com.permutive.pubsub.producer.http.PubsubHttpProducerConfig -import com.permutive.pubsub.producer.{Model, PubsubProducer} -import org.http4s.Method._ -import org.http4s.Uri._ -import org.http4s._ -import org.http4s.client._ -import org.http4s.client.dsl.Http4sClientDsl -import org.http4s.headers._ -import org.typelevel.log4cats.Logger - -import java.util.Base64 -import scala.concurrent.duration._ - -private[http] class DefaultHttpPublisher[F[_]: Async: Logger, A: MessageEncoder] private ( - baseApiUrl: Uri, - client: Client[F], - requestAuthorizer: RequestAuthorizer[F], -) extends PubsubProducer[F, A] - with Http4sClientDsl[F] { - import DefaultHttpPublisher._ - - final private[this] val publishRoute = - Uri.unsafeFromString(s"${baseApiUrl.renderString}:publish") - - final override def produce(data: A, attributes: Map[String, String], uniqueId: String): F[MessageId] = - produceMany[List](List(Model.SimpleRecord(data, attributes, uniqueId))).map(_.head) - - final override def produceMany[G[_]: Traverse](records: G[Model.Record[A]]): F[List[MessageId]] = - for { - msgs <- records.traverse(recordToMessage) - json <- Sync[F].delay(writeToArray(MessageBundle(msgs))) - resp <- sendHttpRequest(json) - } yield resp - - private def sendHttpRequest(json: Array[Byte]): F[List[MessageId]] = { - val req = POST(json, publishRoute, `Content-Type`(MediaType.application.json)) - for { - authedReq <- requestAuthorizer.authorize(req) - resp <- client.expectOr[Array[Byte]](authedReq)(onError) - decoded <- Sync[F].delay(readFromArray[MessageIds](resp)).onError { case _ => - Logger[F].error(s"Publish response from PubSub was invalid. Body: ${new String(resp)}") - } - } yield decoded.messageIds - } - - @inline - private def recordToMessage(record: Model.Record[A]): F[Message] = - MessageEncoder[A] - .encode(record.data) - .map(toMessage(_, record.uniqueId, record.attributes)) - .liftTo[F] - - @inline - private def toMessage(bytes: Array[Byte], uniqueId: String, attributes: Map[String, String]): Message = - Message( - data = Base64.getEncoder.encodeToString(bytes), - messageId = uniqueId, - attributes = attributes - ) - - @inline - private def onError(resp: Response[F]): F[Throwable] = - resp.as[String].map(FailedRequestToPubsub(resp.status, _)) -} - -private[http] object DefaultHttpPublisher { - - def resource[F[_]: Async: Logger, A: MessageEncoder]( - projectId: Model.ProjectId, - topic: Model.Topic, - serviceAccountPath: Option[String], - config: PubsubHttpProducerConfig[F], - httpClient: Client[F] - ): Resource[F, PubsubProducer[F, A]] = - for { - tokenProvider <- - if (config.isEmulator) Resource.pure[F, TokenProvider[F]](DefaultTokenProvider.noAuth) - else - serviceAccountPath.fold( - CachedTokenProvider - .resource( - DefaultTokenProvider.instanceMetadata(httpClient), - // GCP metadata endpoint caches tokens until 5 minutes before expiry. - // Wait until 4 minutes before expiry to refresh the token in this library. That should ensure a new - // token will be provided and have no risk of any requests using an expired token. - // See: https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#applications - // "The metadata server caches access tokens until they have 5 minutes of remaining time before they expire." - safetyPeriod = 4.minutes, - backgroundFailureHook = config.onTokenRetriesExhausted, - onNewToken = config.onTokenRefreshSuccess.map(onRefreshSuccess => - (_: AccessToken, _: FiniteDuration) => onRefreshSuccess - ), - ) - )(path => - for { - tokenProvider <- Resource.eval(DefaultTokenProvider.google(path, httpClient)) - accessTokenRefEffect <- RefreshableEffect.createRetryResource( - refresh = tokenProvider.accessToken, - refreshInterval = config.oauthTokenRefreshInterval, - onRefreshSuccess = config.onTokenRefreshSuccess.getOrElse(Applicative[F].unit), - onRefreshError = config.onTokenRefreshError, - retryDelay = config.oauthTokenFailureRetryDelay, - retryNextDelay = config.oauthTokenFailureRetryNextDelay, - retryMaxAttempts = config.oauthTokenFailureRetryMaxAttempts, - onRetriesExhausted = config.onTokenRetriesExhausted, - ) - } yield TokenProvider.instance(accessTokenRefEffect.value) - ) - } yield new DefaultHttpPublisher[F, A]( - baseApiUrl = createBaseApiUri(projectId, topic, config), - client = httpClient, - requestAuthorizer = RequestAuthorizer.tokenProvider(tokenProvider), - ) - - def createBaseApiUri[F[_]]( - projectId: Model.ProjectId, - topic: Model.Topic, - config: PubsubHttpProducerConfig[F] - ): Uri = - Uri( - scheme = Option(if (config.port == 443) Uri.Scheme.https else Uri.Scheme.http), - authority = Option(Uri.Authority(host = RegName(config.host), port = Option(config.port))), - path = Uri.Path.unsafeFromString(s"/v1/projects/${projectId.value}/topics/${topic.value}") - ) - - case class Message( - data: String, - messageId: String, - attributes: Map[String, String] - ) - - case class MessageBundle[G[_]]( - messages: G[Message] - ) - - case class MessageIds( - messageIds: List[MessageId] - ) - - implicit final def foldableMessagesCodec[G[_]](implicit G: Foldable[G]): JsonValueCodec[G[Message]] = - new JsonValueCodec[G[Message]] { - override def decodeValue(in: JsonReader, default: G[Message]): G[Message] = ??? - - override def encodeValue(x: G[Message], out: JsonWriter): Unit = { - out.writeArrayStart() - x.foreach(MessageCodec.encodeValue(_, out)) - out.writeArrayEnd() - } - - override def nullValue: G[Message] = ??? - } - - implicit final val MessageCodec: JsonValueCodec[Message] = - JsonCodecMaker.make[Message](CodecMakerConfig) - - implicit final def messageBundleCodec[G[_]](implicit - Codec: JsonValueCodec[G[Message]] - ): JsonValueCodec[MessageBundle[G]] = - new JsonValueCodec[MessageBundle[G]] { - override def decodeValue(in: JsonReader, default: MessageBundle[G]): MessageBundle[G] = ??? - - override def encodeValue(x: MessageBundle[G], out: JsonWriter): Unit = { - out.writeObjectStart(); - out.writeNonEscapedAsciiKey("messages"); - Codec.encodeValue(x.messages, out); - out.writeObjectEnd(); - } - - override def nullValue: MessageBundle[G] = ??? - } - - implicit final val MessageIdsCodec: JsonValueCodec[MessageIds] = - JsonCodecMaker.make[MessageIds](CodecMakerConfig) - - case class FailedRequestToPubsub(status: Status, response: String) - extends Throwable(s"Failed request to pubsub. Response was: $response") -} diff --git a/fs2-google-pubsub-http/src/test/resources/logback.xml b/fs2-google-pubsub-http/src/test/resources/logback.xml deleted file mode 100644 index 5d5785df..00000000 --- a/fs2-google-pubsub-http/src/test/resources/logback.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - System.out - true - - %date{MM/dd HH:mm:ss.SSS} %highlight(%-5level) [%thread] %cyan(%logger{15}) - %msg%n - - - - - 4096 - 1000 - - - - - - - diff --git a/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/HttpPingPongSpec.scala b/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/HttpPingPongSpec.scala deleted file mode 100644 index ceb06d30..00000000 --- a/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/HttpPingPongSpec.scala +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub - -import cats.effect._ -import cats.syntax.all._ -import com.google.cloud.pubsub.v1.{SubscriptionAdminClient, TopicAdminClient} -import com.google.pubsub.v1.{SubscriptionName, TopicName} -import com.permutive.pubsub.consumer.ConsumerRecord -import com.permutive.pubsub.consumer.http.Example.ValueHolder -import com.permutive.pubsub.producer.Model.SimpleRecord -import com.permutive.pubsub.producer.{Model, PubsubProducer} -import fs2.Stream -import org.http4s.client.Client -import org.scalatest.BeforeAndAfterEach -import org.typelevel.log4cats.Logger -import org.typelevel.log4cats.slf4j.Slf4jLogger - -import scala.concurrent.duration._ - -class HttpPingPongSpec extends PubSubSpec with BeforeAndAfterEach { - - implicit val logger: Logger[IO] = Slf4jLogger.getLogger - - // Delete topic and subscriptions after each test to ensure state is clean - override def afterEach(): Unit = - clearTopicSubscription.unsafeRunSync() - - private[this] val topicAndSubscriptionClient: Resource[IO, (TopicAdminClient, SubscriptionAdminClient)] = - for { - transCreds <- providers - topicClient <- topicAdminClient(transCreds._1, transCreds._2) - subscriptionClient <- subscriptionAdminClient(transCreds._1, transCreds._2) - } yield (topicClient, subscriptionClient) - - private[this] val clearTopicSubscription: IO[Unit] = - topicAndSubscriptionClient.use { case (topicClient, subscriptionClient) => - for { - _ <- deleteSubscription(subscriptionClient, SubscriptionName.of(project, subscription)) - _ <- deleteTopic(topicClient, TopicName.of(project, topic)) - } yield () - } - - private def setup(ackDeadlineSeconds: Int): Resource[IO, (Client[IO], PubsubProducer[IO, ValueHolder])] = - for { - c <- client - _ <- Resource.eval(createTopic(project, topic)) - _ <- Resource.eval(createSubscription(project, topic, subscription, ackDeadlineSeconds)) - p <- producer(c) - } yield (c, p) - - private def consumeExpectingLimitedMessages( - client: Client[IO], - messagesExpected: Int, - ): Stream[IO, ConsumerRecord[IO, ValueHolder]] = - consumer(client) - // Check we only receive a single element - .zipWithIndex - .flatMap { case (record, ix) => - Stream.fromEither[IO]( - // Index is 0-based, so we expected index to reach 1 less than `messagesExpected` - if (ix < messagesExpected.toLong) Right(record) - else Left(new RuntimeException(s"Received more than $messagesExpected from PubSub")) - ) - } - // Check body is as we expect - .flatMap(record => - Stream.fromEither[IO]( - if (record.value == ValueHolder("ping")) Right(record) - else Left(new RuntimeException(s"Consumed element did not have correct value: ${record.value}")) - ) - ) - - private def consumeAndAck( - client: Client[IO], - elementsReceived: Ref[IO, Int], - ): Stream[IO, ConsumerRecord[IO, ValueHolder]] = - consumeExpectingLimitedMessages(client, messagesExpected = 1) - // Indicate we have received the element as expected - .evalTap(_ => elementsReceived.update(_ + 1)) - .evalTap(_.ack) - - it should "send and receive a message, acknowledging as expected" in { - (for { - // We will sleep for 10 seconds, which means if the message is not acked it will be redelivered before end of test - (client, producer) <- Stream.resource(setup(ackDeadlineSeconds = 5)) - _ <- Stream.eval(producer.produce(ValueHolder("ping"))) - ref <- Stream.eval(Ref.of[IO, Int](0)) - // Wait 10 seconds whilst we run the consumer to check we have a single element and it has the right data - _ <- Stream.sleep[IO](10.seconds).concurrently(consumeAndAck(client, ref)) - elementsReceived <- Stream.eval(ref.get) - } yield elementsReceived should ===(1)) - .as(ExitCode.Success) - .compile - .drain - .unsafeRunSync() - } - - private def consumeExpectingChunksize( - client: Client[IO], - elementsReceived: Ref[IO, Int], - chunkSizeExpected: Int, - ): Stream[IO, ConsumerRecord[IO, ValueHolder]] = - consumer(client).chunks - .evalTap(c => - IO.raiseError(new RuntimeException(s"Chunks were of the wrong size, received ${c.size}")) - .unlessA(c.size == chunkSizeExpected) - ) - .unchunks - .evalTap(_ => elementsReceived.update(_ + 1)) - .evalTap(_.ack) - - it should "preserve chunksize in the underlying stream" in { - val messagesToSend = 5 - - (for { - // We will sleep for 10 seconds, which means if the message is not acked it will be redelivered before end of test - (client, producer) <- Stream.resource(setup(ackDeadlineSeconds = 5)) - _ <- Stream.eval( - // This must be produced using `produceMany` otherwise the returned elements are in individual chunks - producer.produceMany(List.fill[Model.Record[ValueHolder]](messagesToSend)(SimpleRecord(ValueHolder("ping")))) - ) - ref <- Stream.eval(Ref.of[IO, Int](0)) - // Wait 10 seconds whilst we run the consumer to check we have received all elements in a single chunk - _ <- Stream.sleep[IO](10.seconds).concurrently(consumeExpectingChunksize(client, ref, messagesToSend)) - elementsReceived <- Stream.eval(ref.get) - } yield elementsReceived should ===(messagesToSend)) - .as(ExitCode.Success) - .compile - .drain - .unsafeRunSync() - } - - private def consumeExtendSleepAck( - client: Client[IO], - elementsReceived: Ref[IO, Int], - extendDuration: FiniteDuration, - sleepDuration: FiniteDuration, - ): Stream[IO, ConsumerRecord[IO, ValueHolder]] = - consumeExpectingLimitedMessages(client, messagesExpected = 1) - .evalTap(_.extendDeadline(extendDuration)) - .evalTap(_ => IO.sleep(sleepDuration)) - // Indicate we have received the element as expected - .evalTap(_ => elementsReceived.update(_ + 1)) - .evalTap(_.ack) - - it should "extend the deadline for a message" in { - // These setting mean that if extension does not work the message will be redelivered before the end of the test - val ackDeadlineSeconds = 2 - val sleepDuration = 3.seconds - val extendDuration = 10.seconds - - (for { - (client, producer) <- Stream.resource(setup(ackDeadlineSeconds = ackDeadlineSeconds)) - _ <- Stream.eval(producer.produce(ValueHolder("ping"))) - ref <- Stream.eval(Ref.of[IO, Int](0)) - // Wait 10 seconds whilst we run the consumer to check we have a single element and it has the right data - _ <- Stream - .sleep[IO](10.seconds) - .concurrently( - consumeExtendSleepAck(client, ref, extendDuration = extendDuration, sleepDuration = sleepDuration) - ) - elementsReceived <- Stream.eval(ref.get) - } yield elementsReceived should ===(1)) - .as(ExitCode.Success) - .compile - .drain - .unsafeRunSync() - } - - private def consumeNackThenAck( - client: Client[IO], - elementsReceived: Ref[IO, Int], - messagesExpected: Int, - ): Stream[IO, Unit] = - // We expect the message to be redelivered once, so expect 2 messages total - consumeExpectingLimitedMessages(client, messagesExpected = messagesExpected) - // Indicate we have received the element as expected - .evalTap(_ => elementsReceived.update(_ + 1)) - // Nack the first message, then ack subsequent ones - .evalScan(false) { case (nackedAlready, record) => - if (nackedAlready) record.ack.as(true) else record.nack.as(true) - } - .void - - it should "nack a message properly" in { - // These setting mean that a message will only be redelivered if it is nacked - val ackDeadlineSeconds = 100 - val messagesExpected = 2 - - (for { - (client, producer) <- Stream.resource(setup(ackDeadlineSeconds = ackDeadlineSeconds)) - _ <- Stream.eval(producer.produce(ValueHolder("ping"))) - ref <- Stream.eval(Ref.of[IO, Int](0)) - // Wait 10 seconds whilst we run the consumer which nacks the message, then acks - _ <- Stream - .sleep[IO](10.seconds) - .concurrently(consumeNackThenAck(client, ref, messagesExpected)) - elementsReceived <- Stream.eval(ref.get) - } yield elementsReceived should ===(messagesExpected)) - .as(ExitCode.Success) - .compile - .drain - .unsafeRunSync() - } - -} diff --git a/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/PubSubSpec.scala b/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/PubSubSpec.scala deleted file mode 100644 index eef4d899..00000000 --- a/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/PubSubSpec.scala +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub - -import cats.effect.unsafe.IORuntime -import cats.effect.{IO, Resource} -import com.dimafeng.testcontainers.{ForAllTestContainer, GenericContainer} -import com.google.api.gax.core.{CredentialsProvider, NoCredentialsProvider} -import com.google.api.gax.grpc.GrpcTransportChannel -import com.google.api.gax.rpc.{FixedTransportChannelProvider, TransportChannelProvider} -import com.google.cloud.pubsub.v1.{ - SubscriptionAdminClient, - SubscriptionAdminSettings, - TopicAdminClient, - TopicAdminSettings -} -import com.google.pubsub.v1._ -import com.permutive.pubsub.consumer.http.Example.ValueHolder -import com.permutive.pubsub.consumer.http.{PubsubHttpConsumer, PubsubHttpConsumerConfig} -import com.permutive.pubsub.consumer.{ConsumerRecord, Model} -import com.permutive.pubsub.producer.PubsubProducer -import com.permutive.pubsub.producer.http.{HttpPubsubProducer, PubsubHttpProducerConfig} -import fs2.Stream -import io.grpc.{ManagedChannel, ManagedChannelBuilder} -import org.http4s.client.Client -import org.http4s.okhttp.client.OkHttpBuilder -import org.scalactic.TripleEquals -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -import org.testcontainers.containers.wait.strategy.Wait -import org.typelevel.log4cats.Logger - -trait PubSubSpec extends AnyFlatSpec with ForAllTestContainer with Matchers with TripleEquals { - - implicit val logger: Logger[IO] - implicit val ioRuntime: IORuntime = IORuntime.global - - val project = "test-project" - val topic = "example-topic" - val subscription = "example-subscription" - - override val container: GenericContainer = - GenericContainer( - "google/cloud-sdk:311.0.0", // newer version don't work for some reason - exposedPorts = Seq(8085), - waitStrategy = Wait.forLogMessage("(?s).*started.*$", 1), - command = s"gcloud beta emulators pubsub start --project=$project --host-port 0.0.0.0:8085" - .split(" ") - .toSeq - ) - - override def afterStart(): Unit = - updateEnv("PUBSUB_EMULATOR_HOST", s"localhost:${container.mappedPort(8085)}") - - def providers: Resource[IO, (TransportChannelProvider, CredentialsProvider)] = - Resource - .make( - IO { - ManagedChannelBuilder - .forAddress("localhost", container.mappedPort(8085)) - .usePlaintext() - .build(): ManagedChannel - } - )(ch => IO.blocking(ch.shutdown()).void) - .map { channel => - val channelProvider: FixedTransportChannelProvider = - FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)) - val credentialsProvider: NoCredentialsProvider = NoCredentialsProvider.create - - (channelProvider: TransportChannelProvider, credentialsProvider: CredentialsProvider) - } - - def topicAdminClient( - transportChannelProvider: TransportChannelProvider, - credentialsProvider: CredentialsProvider, - ): Resource[IO, TopicAdminClient] = - Resource.fromAutoCloseable( - IO( - TopicAdminClient.create( - TopicAdminSettings - .newBuilder() - .setTransportChannelProvider(transportChannelProvider) - .setCredentialsProvider(credentialsProvider) - .build() - ) - ) - ) - - def createTopic(projectId: String, topicId: String): IO[Topic] = - providers - .flatMap { case (transport, creds) => topicAdminClient(transport, creds) } - .use(client => IO.blocking(client.createTopic(TopicName.of(projectId, topicId)))) - .flatTap(topic => IO.println(s"Topic: $topic")) - - def deleteTopic(client: TopicAdminClient, topic: TopicName): IO[Unit] = - IO.blocking(client.deleteTopic(topic)) - - def subscriptionAdminClient( - transportChannelProvider: TransportChannelProvider, - credentialsProvider: CredentialsProvider, - ): Resource[IO, SubscriptionAdminClient] = - Resource.fromAutoCloseable( - IO( - SubscriptionAdminClient.create( - SubscriptionAdminSettings - .newBuilder() - .setTransportChannelProvider(transportChannelProvider) - .setCredentialsProvider(credentialsProvider) - .build() - ) - ) - ) - - def deleteSubscription(client: SubscriptionAdminClient, sub: SubscriptionName): IO[Unit] = - IO.blocking(client.deleteSubscription(sub)) - - def createSubscription( - projectId: String, - topicId: String, - subscription: String, - ackDeadlineSeconds: Int, - ): IO[Subscription] = - providers - .flatMap { case (transport, creds) => subscriptionAdminClient(transport, creds) } - .use(client => - IO.blocking( - client.createSubscription( - ProjectSubscriptionName.format(projectId, subscription), - TopicName.format(projectId, topicId), - PushConfig.getDefaultInstance, - ackDeadlineSeconds - ) - ) - ) - .flatTap(sub => IO.println(s"Sub: $sub")) - - def client: Resource[IO, Client[IO]] = - OkHttpBuilder - .withDefaultClient[IO] - .flatMap(_.resource) - - def producer( - client: Client[IO], - project: String = project, - topic: String = topic - ): Resource[IO, PubsubProducer[IO, ValueHolder]] = - HttpPubsubProducer.resource[IO, ValueHolder]( - com.permutive.pubsub.producer.Model.ProjectId(project), - com.permutive.pubsub.producer.Model.Topic(topic), - Some("/path/to/service/account"), - config = PubsubHttpProducerConfig( - host = container.host, - port = container.mappedPort(8085), - isEmulator = true, - ), - client - ) - - def consumer( - client: Client[IO], - project: String = project, - subscription: String = subscription, - ): Stream[IO, ConsumerRecord[IO, ValueHolder]] = - PubsubHttpConsumer.subscribe[IO, ValueHolder]( - Model.ProjectId(project), - Model.Subscription(subscription), - Some("/path/to/service/account"), - PubsubHttpConsumerConfig( - host = container.host, - port = container.mappedPort(8085), - isEmulator = true - ), - client, - (msg, err, ack, _) => IO.println(s"Msg $msg got error $err") >> ack - ) - - def updateEnv(name: String, value: String): Unit = { - val env = System.getenv - val field = env.getClass.getDeclaredField("m") - field.setAccessible(true) - field.get(env).asInstanceOf[java.util.Map[String, String]].put(name, value) - () - } -} diff --git a/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/consumer/http/Example.scala b/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/consumer/http/Example.scala deleted file mode 100644 index 8c318234..00000000 --- a/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/consumer/http/Example.scala +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer.http - -import cats.effect._ -import com.permutive.pubsub.consumer.Model -import com.permutive.pubsub.consumer.decoder.MessageDecoder -import com.permutive.pubsub.producer.encoder.MessageEncoder -import fs2.Stream -import org.http4s.client.Client -import org.http4s.okhttp.client.OkHttpBuilder -import org.typelevel.log4cats.Logger -import org.typelevel.log4cats.slf4j.Slf4jLogger - -import scala.util.Try - -object Example extends IOApp { - case class ValueHolder(value: String) extends AnyVal - - implicit val decoder: MessageDecoder[ValueHolder] = (bytes: Array[Byte]) => { - Try(ValueHolder(new String(bytes))).toEither - } - - implicit val encoder: MessageEncoder[ValueHolder] = - (a: ValueHolder) => Right(a.value.getBytes()) - - override def run(args: List[String]): IO[ExitCode] = { - val client = OkHttpBuilder - .withDefaultClient[IO] - .flatMap(_.resource) - - implicit val unsafeLogger: Logger[IO] = Slf4jLogger.getLoggerFromName("fs2-google-pubsub") - - def mkConsumer(client: Client[IO]) = - PubsubHttpConsumer.subscribe[IO, ValueHolder]( - Model.ProjectId("test-project"), - Model.Subscription("example-sub"), - Some("/path/to/service/account"), - PubsubHttpConsumerConfig( - host = "localhost", - port = 8085, - isEmulator = true - ), - client, - (msg, err, ack, _) => IO(println(s"Msg $msg got error $err")) >> ack, - ) - - Stream - .resource(client) - .flatMap(mkConsumer) - .evalTap(t => t.ack >> IO(println(s"Got: ${t.value}"))) - .as(ExitCode.Success) - .compile - .lastOrError - } -} diff --git a/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/http/package.scala b/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/http/package.scala deleted file mode 100644 index c43cd01c..00000000 --- a/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/http/package.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub - -package object http {} diff --git a/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/producer/http/ExampleBatching.scala b/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/producer/http/ExampleBatching.scala deleted file mode 100644 index 6ccb74b4..00000000 --- a/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/producer/http/ExampleBatching.scala +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.http - -import cats.effect._ -import com.github.plokhotnyuk.jsoniter_scala.core._ -import com.github.plokhotnyuk.jsoniter_scala.macros._ -import com.permutive.pubsub.producer.Model -import com.permutive.pubsub.producer.encoder.MessageEncoder -import org.http4s.okhttp.client.OkHttpBuilder -import org.typelevel.log4cats.Logger -import org.typelevel.log4cats.slf4j.Slf4jLogger - -import scala.concurrent.duration._ -import scala.util.Try - -object ExampleBatching extends IOApp { - - implicit final val Codec: JsonValueCodec[ExampleObject] = - JsonCodecMaker.make[ExampleObject](CodecMakerConfig) - - implicit val encoder: MessageEncoder[ExampleObject] = (a: ExampleObject) => { - Try(writeToArray(a)).toEither - } - - case class ExampleObject( - projectId: String, - url: String - ) - - override def run(args: List[String]): IO[ExitCode] = { - val client = OkHttpBuilder - .withDefaultClient[IO] - .flatMap(_.resource) - - implicit val unsafeLogger: Logger[IO] = Slf4jLogger.getLoggerFromName("fs2-google-pubsub") - - val mkProducer = BatchingHttpPubsubProducer.resource[IO, ExampleObject]( - projectId = Model.ProjectId("test-project"), - topic = Model.Topic("example-topic"), - googleServiceAccountPath = Some("/path/to/service/account"), - config = PubsubHttpProducerConfig( - host = "localhost", - port = 8085, - oauthTokenRefreshInterval = 30.minutes, - isEmulator = true - ), - batchingConfig = BatchingHttpProducerConfig( - batchSize = 10, - maxLatency = 100.millis, - retryTimes = 0, - retryInitialDelay = 0.millis, - retryNextDelay = _ + 250.millis - ), - _ - ) - - val messageCallback: Either[Throwable, Unit] => IO[Unit] = { - case Right(_) => Logger[IO].info("Async message was sent successfully!") - case Left(e) => Logger[IO].warn(e)("Async message was sent unsuccessfully!") - } - - client - .flatMap(mkProducer) - .use { producer => - val produceOne = producer.produce( - data = ExampleObject("1f9774be-9d7c-4dd9-8d97-855b681938a9", "example.com") - ) - - val produceOneAsync = producer.produceAsync( - data = ExampleObject("a84a3318-adbd-4eac-af78-eacf33be91ef", "example.com"), - callback = messageCallback - ) - - for { - result1 <- produceOne - result2 <- produceOne - result3 <- produceOne - _ <- result1 - _ <- Logger[IO].info("First message was sent!") - _ <- result2 - _ <- Logger[IO].info("Second message was sent!") - _ <- result3 - _ <- Logger[IO].info("Third message was sent!") - _ <- produceOneAsync - _ <- IO.never[Unit] - } yield () - } - .as(ExitCode.Success) - } -} diff --git a/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/producer/http/ExampleEmulator.scala b/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/producer/http/ExampleEmulator.scala deleted file mode 100644 index b6a14fab..00000000 --- a/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/producer/http/ExampleEmulator.scala +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.http - -import cats.effect._ -import com.github.plokhotnyuk.jsoniter_scala.core._ -import com.github.plokhotnyuk.jsoniter_scala.macros._ -import com.permutive.pubsub.producer.Model -import com.permutive.pubsub.producer.encoder.MessageEncoder -import org.http4s.okhttp.client.OkHttpBuilder -import org.typelevel.log4cats.Logger -import org.typelevel.log4cats.slf4j.Slf4jLogger - -import scala.concurrent.duration._ -import scala.util.Try - -object ExampleEmulator extends IOApp { - - implicit final val Codec: JsonValueCodec[ExampleObject] = - JsonCodecMaker.make[ExampleObject](CodecMakerConfig) - - implicit val encoder: MessageEncoder[ExampleObject] = (a: ExampleObject) => { - Try(writeToArray(a)).toEither - } - - case class ExampleObject( - projectId: String, - url: String - ) - - override def run(args: List[String]): IO[ExitCode] = { - val client = OkHttpBuilder - .withDefaultClient[IO] - .flatMap(_.resource) - - implicit val unsafeLogger: Logger[IO] = Slf4jLogger.getLoggerFromName("fs2-google-pubsub") - - val mkProducer = HttpPubsubProducer.resource[IO, ExampleObject]( - projectId = Model.ProjectId("test-project"), - topic = Model.Topic("example-topic"), - googleServiceAccountPath = Some("/path/to/nothing"), - config = PubsubHttpProducerConfig( - host = "localhost", - port = 8085, - oauthTokenRefreshInterval = 30.minutes, - isEmulator = true - ), - _ - ) - - client - .flatMap(mkProducer) - .use { producer => - producer.produce( - data = ExampleObject("hsaudhiasuhdiu21hi3und", "example.com") - ) - } - .flatTap(output => IO(println(output))) - .map(_ => ExitCode.Success) - } -} diff --git a/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/producer/http/ExampleGoogle.scala b/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/producer/http/ExampleGoogle.scala deleted file mode 100644 index ed4f1edf..00000000 --- a/fs2-google-pubsub-http/src/test/scala/com/permutive/pubsub/producer/http/ExampleGoogle.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.http - -import cats.effect._ -import com.github.plokhotnyuk.jsoniter_scala.core._ -import com.github.plokhotnyuk.jsoniter_scala.macros._ -import com.permutive.pubsub.producer.Model -import com.permutive.pubsub.producer.encoder.MessageEncoder -import org.http4s.okhttp.client.OkHttpBuilder -import org.typelevel.log4cats.Logger -import org.typelevel.log4cats.slf4j.Slf4jLogger - -import scala.concurrent.duration._ -import scala.util.Try - -object ExampleGoogle extends IOApp { - - implicit final val Codec: JsonValueCodec[ExampleObject] = - JsonCodecMaker.make[ExampleObject](CodecMakerConfig) - - implicit val encoder: MessageEncoder[ExampleObject] = (a: ExampleObject) => { - Try(writeToArray(a)).toEither - } - - case class ExampleObject( - projectId: String, - url: String - ) - - override def run(args: List[String]): IO[ExitCode] = { - val client = OkHttpBuilder - .withDefaultClient[IO] - .flatMap(_.resource) - - implicit val unsafeLogger: Logger[IO] = Slf4jLogger.getLoggerFromName("fs2-google-pubsub") - - val mkProducer = HttpPubsubProducer.resource[IO, ExampleObject]( - projectId = Model.ProjectId("test-project"), - topic = Model.Topic("example-topic"), - googleServiceAccountPath = Some("/path/to/service/account"), - config = PubsubHttpProducerConfig( - host = "pubsub.googleapis.com", - port = 443, - oauthTokenRefreshInterval = 30.minutes - ), - _ - ) - - client - .flatMap(mkProducer) - .use { producer => - producer.produce( - data = ExampleObject("70251cf8-5ffb-4c3f-8f2f-40b9bfe4147c", "example.com") - ) - } - .flatTap(output => IO(println(output))) - .as(ExitCode.Success) - } -} diff --git a/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/ConsumerRecord.scala b/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/ConsumerRecord.scala deleted file mode 100644 index 599b945f..00000000 --- a/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/ConsumerRecord.scala +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer - -import cats.Show -import cats.syntax.show._ - -import scala.concurrent.duration.FiniteDuration - -trait ConsumerRecord[F[_], A] { - def value: A - def attributes: Map[String, String] - def ack: F[Unit] - def nack: F[Unit] - def extendDeadline(by: FiniteDuration): F[Unit] -} - -object ConsumerRecord { - implicit def show[F[_], A: Show]: Show[ConsumerRecord[F, A]] = - (record: ConsumerRecord[F, A]) => s"Record(${record.value.show})" - - abstract private[this] case class RecordImpl[F[_], A]( - value: A, - attributes: Map[String, String], - ack: F[Unit], - nack: F[Unit], - ) extends ConsumerRecord[F, A] - - def apply[F[_], A]( - value: A, - attributes: Map[String, String], - ack: F[Unit], - nack: F[Unit], - extend: FiniteDuration => F[Unit], - ): ConsumerRecord[F, A] = - new RecordImpl(value, attributes, ack, nack) { - final override def extendDeadline(by: FiniteDuration): F[Unit] = extend(by) - } -} diff --git a/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/Model.scala b/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/Model.scala deleted file mode 100644 index 752e15fe..00000000 --- a/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/Model.scala +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer - -import cats.Show - -object Model { - case class ProjectId(value: String) extends AnyVal - object ProjectId { - implicit val show: Show[ProjectId] = Show.fromToString - } - - case class Subscription(value: String) extends AnyVal - object Subscription { - implicit val show: Show[Subscription] = Show.fromToString - } -} diff --git a/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/decoder/MessageDecoder.scala b/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/decoder/MessageDecoder.scala deleted file mode 100644 index bafab0bf..00000000 --- a/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/decoder/MessageDecoder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.consumer.decoder - -import cats.Functor - -trait MessageDecoder[A] { - def decode(message: Array[Byte]): Either[Throwable, A] -} - -object MessageDecoder { - def apply[A: MessageDecoder]: MessageDecoder[A] = implicitly - - implicit val functor: Functor[MessageDecoder] = new Functor[MessageDecoder] { - - override def map[A, B](fa: MessageDecoder[A])(f: A => B): MessageDecoder[B] = - (message: Array[Byte]) => fa.decode(message).map(f) - - } -} diff --git a/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/AsyncPubsubProducer.scala b/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/AsyncPubsubProducer.scala deleted file mode 100644 index 40f84b37..00000000 --- a/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/AsyncPubsubProducer.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer - -import java.util.UUID - -import cats.{Foldable, Traverse} - -trait AsyncPubsubProducer[F[_], A] { - def produceAsync( - data: A, - callback: Either[Throwable, Unit] => F[Unit], - attributes: Map[String, String] = Map.empty, - uniqueId: String = UUID.randomUUID().toString - ): F[Unit] - - def produceManyAsync[G[_]: Foldable]( - records: G[Model.AsyncRecord[F, A]] - ): F[Unit] - - def produce( - data: A, - attributes: Map[String, String] = Map.empty, - uniqueId: String = UUID.randomUUID().toString - ): F[F[Unit]] - - def produceMany[G[_]: Traverse]( - records: G[Model.SimpleRecord[A]] - ): F[G[F[Unit]]] -} diff --git a/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/Model.scala b/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/Model.scala deleted file mode 100644 index 8375e2b7..00000000 --- a/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/Model.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer - -import java.util.UUID - -object Model { - case class MessageId(value: String) extends AnyVal - case class ProjectId(value: String) extends AnyVal - case class Topic(value: String) extends AnyVal - - trait Record[A] { - def data: A - def attributes: Map[String, String] - def uniqueId: String - } - - final case class SimpleRecord[A]( - data: A, - attributes: Map[String, String] = Map.empty, - uniqueId: String = UUID.randomUUID().toString - ) extends Record[A] - - final case class AsyncRecord[F[_], A]( - data: A, - callback: Either[Throwable, Unit] => F[Unit], - attributes: Map[String, String] = Map.empty, - uniqueId: String = UUID.randomUUID().toString - ) extends Record[A] -} diff --git a/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/PubsubProducer.scala b/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/PubsubProducer.scala deleted file mode 100644 index 5251ebe7..00000000 --- a/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/PubsubProducer.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer - -import java.util.UUID - -import cats.Traverse -import com.permutive.pubsub.producer.Model.MessageId - -trait PubsubProducer[F[_], A] { - def produce( - data: A, - attributes: Map[String, String] = Map.empty, - uniqueId: String = UUID.randomUUID().toString - ): F[MessageId] - - def produceMany[G[_]: Traverse](records: G[Model.Record[A]]): F[List[MessageId]] -} diff --git a/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/encoder/MessageEncoder.scala b/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/encoder/MessageEncoder.scala deleted file mode 100644 index 0d5991da..00000000 --- a/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/encoder/MessageEncoder.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2018 Permutive - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.permutive.pubsub.producer.encoder - -trait MessageEncoder[A] { - def encode(a: A): Either[Throwable, Array[Byte]] -} - -object MessageEncoder { - def apply[A: MessageEncoder]: MessageEncoder[A] = implicitly -} diff --git a/project/Dependencies.scala b/project/Dependencies.scala deleted file mode 100644 index c66d0a7a..00000000 --- a/project/Dependencies.scala +++ /dev/null @@ -1,79 +0,0 @@ -import sbt._ - -object Dependencies { - object Versions { - val catsCore = "2.8.0" - val effect = "3.3.14" - val fs2 = "3.3.0" - val http4s = "0.23.16" - val http4sOkHttp = "0.23.11" - val log4cats = "2.5.0" - val jwt = "3.18.2" - val jsoniter = "2.17.9" - val gcp = "1.116.0" - val scalatest = "3.2.14" - val scalatestPlus = "3.2.14.0" - val testContainers = "0.39.7" - val collectionCompat = "2.8.1" - } - - object Libraries { - val catsCore = "org.typelevel" %% "cats-core" % Versions.catsCore - val alleyCatsCore = "org.typelevel" %% "alleycats-core" % Versions.catsCore - val effect = "org.typelevel" %% "cats-effect-kernel" % Versions.effect - val fs2 = "co.fs2" %% "fs2-core" % Versions.fs2 - - val http4sDsl = "org.http4s" %% "http4s-dsl" % Versions.http4s - val http4sClient = "org.http4s" %% "http4s-client" % Versions.http4s - val http4sHttp = "org.http4s" %% "http4s-okhttp-client" % Versions.http4sOkHttp % Test - - val log4cats = "org.typelevel" %% "log4cats-core" % Versions.log4cats - val log4catsSlf4j = "org.typelevel" %% "log4cats-slf4j" % Versions.log4cats % Test - val slf4j = "org.slf4j" % "slf4j-simple" % "1.7.32" % Test - - val jwt = "com.auth0" % "java-jwt" % Versions.jwt - val gcp = "com.google.cloud" % "google-cloud-pubsub" % Versions.gcp - - val jsoniterCore = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-core" % Versions.jsoniter % Compile - val jsoniterMacros = - "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-macros" % Versions.jsoniter % Provided - - val testContainers = "com.dimafeng" %% "testcontainers-scala-scalatest" % Versions.testContainers % Test - val scalatest = "org.scalatest" %% "scalatest" % Versions.scalatest % Test - val scalatestPlus = "org.scalatestplus" %% "scalacheck-1-16" % Versions.scalatestPlus % Test - - val collectionCompat = "org.scala-lang.modules" %% "scala-collection-compat" % Versions.collectionCompat - } - - lazy val testsDependencies = Seq( - Libraries.scalatest, - Libraries.scalatestPlus, - Libraries.http4sHttp, - Libraries.log4cats, - Libraries.log4catsSlf4j, - Libraries.slf4j, - Libraries.testContainers, - Libraries.gcp % Test, - ) - - lazy val commonDependencies = Seq( - Libraries.catsCore, - Libraries.effect, - Libraries.fs2, - ) - - lazy val httpDependencies = Seq( - Libraries.alleyCatsCore, - Libraries.http4sDsl, - Libraries.http4sClient, - Libraries.log4cats, - Libraries.jwt, - Libraries.jsoniterCore, - Libraries.jsoniterMacros, - ) - - lazy val grpcDependencies = Seq( - Libraries.gcp, - Libraries.collectionCompat - ) -} diff --git a/project/build.properties b/project/build.properties deleted file mode 100644 index 6a9f0388..00000000 --- a/project/build.properties +++ /dev/null @@ -1 +0,0 @@ -sbt.version=1.7.3 diff --git a/project/plugins.sbt b/project/plugins.sbt deleted file mode 100644 index 3d0115bc..00000000 --- a/project/plugins.sbt +++ /dev/null @@ -1,2 +0,0 @@ -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.8.2") -addSbtPlugin("org.typelevel" % "sbt-typelevel" % "0.4.16") From fa5f9f288caea31ce7662d17d7c96fcd04e24c03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Herna=CC=81ndez?= Date: Mon, 1 Apr 2024 18:02:09 +0200 Subject: [PATCH 2/9] Add build files in line with other repositories in the organization --- .github/CODEOWNERS | 1 + .github/docs/LICENSE.md | 201 ++++++++++++++++++++++++++++++++++ .github/release.yml | 28 +++++ .github/workflows/ci.yml | 74 +++++++++++++ .github/workflows/release.yml | 95 ++++++++++++++++ .gitignore | 89 +++++++++++++++ LICENSE.md | 201 ++++++++++++++++++++++++++++++++++ project/build.properties | 1 + project/plugins.sbt | 16 +++ 9 files changed, 706 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/docs/LICENSE.md create mode 100644 .github/release.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 .gitignore create mode 100644 LICENSE.md create mode 100644 project/build.properties create mode 100644 project/plugins.sbt diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..a87d1b94 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @permutive-engineering/developer-experience \ No newline at end of file diff --git a/.github/docs/LICENSE.md b/.github/docs/LICENSE.md new file mode 100644 index 00000000..76b6d4ef --- /dev/null +++ b/.github/docs/LICENSE.md @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (C) @YEAR_RANGE@ @COPYRIGHT_OWNER@ + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 00000000..afbe900c --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,28 @@ +# Don't edit this file! +# It is automatically updated after every release of https://github.com/alejandrohdezma/sbt-ci +# If you want to suggest a change, please open a PR or issue in that repository + +# This file contains the template for the "auto-generated release notes" + +changelog: + exclude: + labels: + - ":chart_with_upwards_trend: dependency-update" + authors: + - dependabot + categories: + - title: "⚠️ Breaking changes" + labels: + - ":warning: breaking" + - title: "🚀 New features" + labels: + - ":rocket: feature" + - title: "📘 Documentation updates" + labels: + - ":blue_book: documentation" + - title: "🐛 Bug fixes" + labels: + - ":beetle: bug" + - title: Other Changes + labels: + - "*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..c1dcd0d7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,74 @@ +# Don't edit this file! +# It is automatically updated after every release of https://github.com/alejandrohdezma/sbt-ci +# If you want to suggest a change, please open a PR or issue in that repository + +# Runs `sbt ci-test` on the project on differnt JDKs (this task should be added to the project as a command alias +# containing the necessary steps to compile, check formatters, launch tests...). +# +# Examples of this `ci-test` alias can be found [here](https://github.com/search?q=org%3Aalejandrohdezma+%22ci-test%22+path%3Abuild.sbt++NOT+is%3Aarchived&type=code). +# +# It will also do the following: +# +# - It will automatically label PRs based on head branch. +# - It will automatically enable auto-merge on `Scala Steward` PRs. You'll need to add a `STEWARD_BOT` repository or +# organization variable with the name of your scala-steward bot. See https://docs.github.com/en/actions/learn-github-actions/variables. + +name: CI + +on: + pull_request: + types: [opened, reopened, labeled, unlabeled, synchronize] + +jobs: + labeler: + if: github.event.pull_request.state == 'OPEN' && github.actor != 'dependabot[bot]' + name: Labeler + runs-on: ubuntu-latest + steps: + - name: Update PR labels + uses: alejandrohdezma/actions/labeler@v1 + if: github.event.pull_request.head.repo.full_name == github.repository + + - name: Check PR labels + uses: alejandrohdezma/actions/label-check@v1 + + ci-steward: + if: | + github.event.pull_request.state == 'OPEN' && github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.user.login == vars.STEWARD_BOT + name: (Scala Steward) Enable auto-merge + runs-on: ubuntu-latest + steps: + - name: Enable auto-merge for this PR + run: gh pr merge --auto --merge ${{github.event.pull_request.number}} -R "$GITHUB_REPOSITORY" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + test: + needs: [ci-steward] + if: | + always() && !contains(needs.*.result, 'failure') && github.event.pull_request.state == 'OPEN' && + github.actor != 'dependabot[bot]' + name: Run "sbt ci-test" on JDK ${{ matrix.jdk }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + jdk: + - 11 + - 17 + steps: + - name: Checkout project + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ github.head_ref }} + fetch-depth: 0 + + - uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4.0.0 + with: + distribution: "liberica" + java-version: ${{ matrix.jdk }} + cache: "sbt" + + - name: Run `sbt ci-test` + run: sbt ci-test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..3707eb12 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,95 @@ +# Don't edit this file! +# It is automatically updated after every release of https://github.com/alejandrohdezma/sbt-ci +# If you want to suggest a change, please open a PR or issue in that repository + +# This workflow performs two tasks: +# +# - Creates a release of the project by running `sbt ci-publish` (this task should be added to the project as a command +# alias containing the necessary steps to do a release). Examples of this `ci-publish` alias can be found +# [here](https://github.com/search?q=org%3Aalejandrohdezma+%22ci-publish%22+path%3Abuild.sbt++NOT+is%3Aarchived&type=code). +# +# - Runs `sbt ci-docs` on the project and pushes a commit with the changes (the `ci-docs` task should be added to the +# project as a command alias containing the necessary steps to update documentation: re-generate docs files, +# publish websites, update headers...). Examples of this `ci-docs` alias can be found +# [here](https://github.com/search?q=org%3Aalejandrohdezma+%22ci-docs%22+path%3Abuild.sbt++NOT+is%3Aarchived&type=code). +# +# This workflow will launch on pushed tags. Alternatively one can launch it manually using a "workflow dispatch" to +# create a snapshot release (this won't trigger the documentation update). + +name: Release + +on: + workflow_dispatch: + push: + tags: [v*] + +jobs: + release: + name: Release a new version of the artifact + runs-on: ubuntu-latest + steps: + - name: Checkout project + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + + - name: Check latest tag follows semantic versioning + if: github.event_name == 'push' + uses: alejandrohdezma/actions/check-semver-tag@v1 + + - uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4.0.0 + with: + distribution: "liberica" + java-version: "11" + cache: "sbt" + + - name: Run `sbt ci-publish` + run: sbt ci-publish + env: + GITHUB_TOKEN: ${{ secrets.ADMIN_GITHUB_TOKEN }} + PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} + PGP_SECRET: ${{ secrets.PGP_SECRET }} + SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} + SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} + + documentation: + needs: [release] + name: Updates documentation and version policy after latest release + if: github.event_name == 'push' + runs-on: ubuntu-latest + steps: + - name: Checkout project + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + ref: main + token: ${{ secrets.ADMIN_GITHUB_TOKEN }} + + - uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4.0.0 + with: + distribution: "liberica" + java-version: "17" + cache: "sbt" + + - name: Run `sbt ci-docs` + run: sbt ci-docs + env: + GITHUB_TOKEN: ${{ secrets.ADMIN_GITHUB_TOKEN }} + GIT_DEPLOY_KEY: ${{ secrets.GIT_DEPLOY_KEY }} + + - name: Commit changes by `sbt ci-docs` + uses: alejandrohdezma/actions/commit-and-push@v1 + with: + message: Run `sbt ci-docs` [skip ci] + branch: main + + - name: Reset `versionPolicyIntention` + run: sed -i -r 's/Compatibility\.(None|BinaryCompatible)/Compatibility.BinaryAndSourceCompatible/g' build.sbt + + - name: Commit `versionPolicyIntention` reset + uses: alejandrohdezma/actions/commit-and-push@v1 + with: + message: Reset `versionPolicyIntention` [skip ci] + branch: main + + diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..c7fe899f --- /dev/null +++ b/.gitignore @@ -0,0 +1,89 @@ +# Don't edit this file! +# It is automatically updated after every release of https://github.com/alejandrohdezma/sbt-ci +# If you want to suggest a change, please open a PR or issue in that repository + +# Default .gitignore for the project. + +### Intellij ### + +.idea +out/ + +### Java ### + +*.class +*.log + +### macOS ### + +.DS_Store + +### SBT ### + +dist/* +target/ +lib_managed/ +src_managed/ +project/boot/ +project/plugins/project/ +.history +.cache +.lib/ + +### Scala ### + +.bloop/ +**/.bloop/ +project/.bloop/ +.bsp +*.metals +.metals/ +**/metals.sbt +project/metals.sbt + +### Vim ### + +# Swap +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist + +# Auto-generated tag files +tags + +# Persistent undo +[._]*.un~ + +# Coc configuration directory +.vim + +### VisualStudioCode ### + +.vscode + +### Docusaurus ### + +.docusaurus +node_modules +website/build + +### sbt-scalafix-defaults ### +# Scalafix configuration file is automatically downloaded by `sbt-scalafix-defaults` +# https://github.com/alejandrohdezma/sbt-scalafix-defaults + +.scalafix.conf + +### sbt-scalafmt-defaults ### +# Scalafmt configuration file is automatically downloaded by `sbt-scalafmt-defaults` +# https://github.com/alejandrohdezma/sbt-scalafmt-defaults + +.scalafmt.conf diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 00000000..1c0c291a --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (C) 2019-2024 Permutive Ltd. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/project/build.properties b/project/build.properties new file mode 100644 index 00000000..abbbce5d --- /dev/null +++ b/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.9.8 diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100644 index 00000000..3b5ffaba --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1,16 @@ +addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.12.0") +addSbtPlugin("com.alejandrohdezma" % "sbt-ci" % "2.14.1") +addSbtPlugin("com.alejandrohdezma" % "sbt-fix" % "0.7.1") +addSbtPlugin("com.alejandrohdezma" % "sbt-github-mdoc" % "0.11.13") +addSbtPlugin("com.alejandrohdezma" % "sbt-github-header" % "0.11.13") +addSbtPlugin("com.alejandrohdezma" % "sbt-scalafix-defaults" % "0.12.0") +addSbtPlugin("com.alejandrohdezma" % "sbt-scalafmt-defaults" % "0.9.0") +addSbtPlugin("com.alejandrohdezma" % "sbt-mdoc-toc" % "0.4.1") +addSbtPlugin("com.alejandrohdezma" % "sbt-modules" % "0.3.1") +addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.2") +addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") +addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.10.0") +addSbtPlugin("ch.epfl.scala" % "sbt-version-policy" % "3.2.0") +addSbtPlugin("org.typelevel" % "sbt-tpolecat" % "0.5.0") +addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.5.2") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2") From e9998c8f5b83ded3f135465d2bf4a76e5b4a5538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Herna=CC=81ndez?= Date: Mon, 1 Apr 2024 18:03:54 +0200 Subject: [PATCH 3/9] =?UTF-8?q?Recreate=20the=20library=20as=20new=20`fs2-?= =?UTF-8?q?pubsub`=20=F0=9F=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.sbt | 13 + .../main/scala/fs2/pubsub/AckDeadline.scala | 79 ++++ .../src/main/scala/fs2/pubsub/AckId.scala | 56 +++ .../scala/fs2/pubsub/MessageDecoder.scala | 90 +++++ .../scala/fs2/pubsub/MessageEncoder.scala | 82 ++++ .../src/main/scala/fs2/pubsub/MessageId.scala | 57 +++ .../main/scala/fs2/pubsub/PubSubClient.scala | 311 +++++++++++++++ .../scala/fs2/pubsub/PubSubPublisher.scala | 366 ++++++++++++++++++ .../main/scala/fs2/pubsub/PubSubRecord.scala | 327 ++++++++++++++++ .../scala/fs2/pubsub/PubSubSubscriber.scala | 243 ++++++++++++ .../main/scala/fs2/pubsub/Subscription.scala | 57 +++ .../src/main/scala/fs2/pubsub/Topic.scala | 56 +++ .../main/scala/fs2/pubsub/circe/package.scala | 31 ++ .../main/scala/fs2/pubsub/dsl/client.scala | 128 ++++++ .../main/scala/fs2/pubsub/dsl/publisher.scala | 125 ++++++ .../scala/fs2/pubsub/dsl/subscriber.scala | 235 +++++++++++ .../exceptions/PubSubRequestError.scala | 75 ++++ .../src/main/scala/fs2/pubsub/package.scala | 25 ++ .../scala/fs2/pubsub/PubSubClientSuite.scala | 45 +++ .../fs2/pubsub/PubSubPublisherSuite.scala | 69 ++++ .../fs2/pubsub/PubSubSubscriberSuite.scala | 55 +++ .../test/scala/fs2/pubsub/PubSubSuite.scala | 177 +++++++++ project/Dependencies.scala | 21 + 23 files changed, 2723 insertions(+) create mode 100644 build.sbt create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/AckDeadline.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/AckId.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/MessageDecoder.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/MessageEncoder.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/MessageId.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubClient.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubPublisher.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubRecord.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubSubscriber.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/Subscription.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/Topic.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/circe/package.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/dsl/client.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/dsl/publisher.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/dsl/subscriber.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/exceptions/PubSubRequestError.scala create mode 100644 modules/fs2-pubsub/src/main/scala/fs2/pubsub/package.scala create mode 100644 modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubClientSuite.scala create mode 100644 modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubPublisherSuite.scala create mode 100644 modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubSubscriberSuite.scala create mode 100644 modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubSuite.scala create mode 100644 project/Dependencies.scala diff --git a/build.sbt b/build.sbt new file mode 100644 index 00000000..38f3b71e --- /dev/null +++ b/build.sbt @@ -0,0 +1,13 @@ +ThisBuild / scalaVersion := "2.13.12" +ThisBuild / crossScalaVersions := Seq("2.12.18", "2.13.12", "3.3.1") +ThisBuild / organization := "com.permutive" +ThisBuild / versionPolicyIntention := Compatibility.None + +addCommandAlias("ci-test", "fix --check; versionPolicyCheck; mdoc; publishLocal; +test") +addCommandAlias("ci-docs", "github; mdoc; headerCreateAll") +addCommandAlias("ci-publish", "versionCheck; github; ci-release") + +lazy val `fs2-pubsub` = module + .settings(libraryDependencies ++= Dependencies.`fs2-pubsub`) + .settings(Test / fork := true) + .settings(Test / run / fork := true) diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/AckDeadline.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/AckDeadline.scala new file mode 100644 index 00000000..4b64449c --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/AckDeadline.scala @@ -0,0 +1,79 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import scala.concurrent.duration._ + +import cats._ +import cats.syntax.all._ + +import io.circe.Codec +import io.circe.Decoder +import io.circe.Encoder + +/** Represents the duration for which the Pub/Sub system will wait for the subscriber to acknowledge the message before + * redelivering it. + */ +final class AckDeadline private (val value: FiniteDuration) extends AnyVal { + + override def toString(): String = value.toString() // scalafix:ok + +} + +object AckDeadline { + + lazy val zero: AckDeadline = new AckDeadline(0.seconds) + + /** Creates a new `AckDeadline` from the provided `FiniteDuration`. + * + * @param duration + * the duration for which the Pub/Sub system will wait for the subscriber to acknowledge the message + * @return + * a `Right` with the `AckDeadline` if the duration is between 0 and 600 seconds (both included), a `Left` + * otherwise + */ + def from(duration: FiniteDuration): Either[String, AckDeadline] = + if (duration >= 0.seconds && duration <= 600.seconds) new AckDeadline(duration).asRight + else show"AckDeadline must be between 0 and 600 seconds (both included), but was: $duration".asLeft + + def unapply(value: AckDeadline): Some[FiniteDuration] = Some(value.value) + + // Cats instances + + implicit val DeadlineShow: Show[AckDeadline] = Show[FiniteDuration].contramap(_.value) + + implicit val DeadlineEqHashOrder: Eq[AckDeadline] with Hash[AckDeadline] with Order[AckDeadline] = + new Eq[AckDeadline] with Hash[AckDeadline] with Order[AckDeadline] { + + override def hash(x: AckDeadline): Int = Hash[FiniteDuration].hash(x.value) + + override def compare(x: AckDeadline, y: AckDeadline): Int = Order[FiniteDuration].compare(x.value, y.value) + + } + + // Circe instances + + implicit val DeadlineCodec: Codec[AckDeadline] = + Codec.from( + Decoder[String].map(Duration(_)).emap { + case duration: FiniteDuration => AckDeadline.from(duration) + case other => show"$other wasn't a valid `FiniteDuration`".asLeft + }, + Encoder[String].contramap(_.value.toString()) // scalafix:ok + ) + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/AckId.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/AckId.scala new file mode 100644 index 00000000..c3948412 --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/AckId.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import cats._ +import cats.syntax.all._ + +import io.circe.Codec +import io.circe.Decoder +import io.circe.Encoder + +/** Represents the unique identifier for the acknowledgment of a message. */ +final class AckId(val value: String) extends AnyVal { + + override def toString(): String = value // scalafix:ok + +} + +object AckId { + + def apply(string: String): AckId = new AckId(string) + + def unapply(value: AckId): Some[String] = Some(value.value) + + // Cats instances + + implicit val AckIdShow: Show[AckId] = Show[String].contramap(_.value) + + implicit val AckIdEqHashOrder: Eq[AckId] with Hash[AckId] with Order[AckId] = + new Eq[AckId] with Hash[AckId] with Order[AckId] { + + override def hash(x: AckId): Int = Hash[String].hash(x.value) + + override def compare(x: AckId, y: AckId): Int = Order[String].compare(x.value, y.value) + + } + + // Circe instances + + implicit val AckIdCodec: Codec[AckId] = Codec.from(Decoder[String].map(AckId(_)), Encoder[String].contramap(_.value)) + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/MessageDecoder.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/MessageDecoder.scala new file mode 100644 index 00000000..281eea73 --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/MessageDecoder.scala @@ -0,0 +1,90 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import java.nio.charset.StandardCharsets.UTF_8 + +import cats.Functor +import cats.syntax.all._ + +import io.circe.Json + +/** Represents a typeclass capable of decoding a target messaage type from an array of bytes. + * + * @tparam A + * the type of message to be decoded + */ +trait MessageDecoder[A] { + + /** Decodes the input array of bytes into the specified message type. + * + * @param message + * the byte array to be decoded + * @return + * either the decoded message of type `A` or an exception if the decoding fails + */ + def decode(message: Array[Byte]): Either[Throwable, A] + + /** Maps the decoded message of type `A` to a new type `B` using the provided function. + * + * @param f + * the function to map the decoded message to another type + * @return + * a new `MessageDecoder` instance for the mapped type `B` + */ + def map[B](f: A => B): MessageDecoder[B] = MessageDecoder.functor.map(this)(f) + + /** Maps the decoded message of type A to a new type B using the provided function that may result in an `Either`. + * + * @param f + * the function that may result in an `Either` value for mapping to type `B` + * @return + * a new `MessageDecoder` instance for the mapped type `B` + */ + def emap[B](f: A => Either[Throwable, B]): MessageDecoder[B] = this.decode(_).flatMap(f) + +} + +object MessageDecoder { + + def apply[A: MessageDecoder]: MessageDecoder[A] = implicitly + + /** Creates a new `MessageDecoder` instance for the specified type `A` using the provided decoding function. + * + * @param f + * the decoding function for the specified type `A` + * @tparam A + * the type of message to be decoded + * @return + * a new `MessageDecoder` instance for the specified type `A` + */ + def instance[A](f: Array[Byte] => Either[Throwable, A]): MessageDecoder[A] = bytes => f(bytes) + + implicit val string: MessageDecoder[String] = bytes => Either.catchNonFatal(new String(bytes, UTF_8)) + + implicit val byteArray: MessageDecoder[Array[Byte]] = _.asRight + + implicit val json: MessageDecoder[Json] = io.circe.jawn.decodeByteArray[Json](_) + + implicit val functor: Functor[MessageDecoder] = new Functor[MessageDecoder] { + + override def map[A, B](fa: MessageDecoder[A])(f: A => B): MessageDecoder[B] = + (message: Array[Byte]) => fa.decode(message).map(f) + + } + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/MessageEncoder.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/MessageEncoder.scala new file mode 100644 index 00000000..751e5f9d --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/MessageEncoder.scala @@ -0,0 +1,82 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import java.nio.charset.StandardCharsets + +import cats.Contravariant + +import io.circe.Json + +/** Represents a typeclass capable of encoding a value of type `A` into an array of bytes. + * + * @tparam A + * the type of message to be encoded + */ +trait MessageEncoder[A] { + + /** Encodes the input value of type `A` into an array of bytes. + * + * @param a + * the value to be encoded + * @return + * the encoded byte array representing the input value + */ + def encode(a: A): Array[Byte] + + /** Contramaps the encoding process from a type `B` to the original type `A` using the provided function. + * + * @param f + * the function to contramap the encoding process from type `B` to type `A` + * @return + * a new `MessageEncoder` instance for the contramapped type `B` + */ + def contramap[B](f: B => A): MessageEncoder[B] = b => this.encode(f(b)) + +} + +object MessageEncoder { + + def apply[A: MessageEncoder]: MessageEncoder[A] = implicitly + + /** Creates a new `MessageEncoder` instance for the specified type `A` using the provided encoding function. + * + * @param f + * the encoding function for the specified type `A` + * @tparam A + * the type of message to be encoded + * @return + * a new `MessageEncoder` instance for the specified type `A` + */ + def instance[A](f: A => Array[Byte]): MessageEncoder[A] = a => f(a) + + implicit val string: MessageEncoder[String] = _.getBytes(StandardCharsets.UTF_8) + + implicit val json: MessageEncoder[Json] = string.contramap(_.noSpaces) + + implicit def option[A: MessageEncoder]: MessageEncoder[Option[A]] = { + case Some(a) => MessageEncoder[A].encode(a) + case None => Array() + } + + implicit val MessageEncoderContravariant: Contravariant[MessageEncoder] = new Contravariant[MessageEncoder] { + + override def contramap[A, B](fa: MessageEncoder[A])(f: B => A): MessageEncoder[B] = b => fa.encode(f(b)) + + } + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/MessageId.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/MessageId.scala new file mode 100644 index 00000000..00bcbe97 --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/MessageId.scala @@ -0,0 +1,57 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import cats._ +import cats.syntax.all._ + +import io.circe.Codec +import io.circe.Decoder +import io.circe.Encoder + +/** Represents the unique identifier for a message. */ +final class MessageId(val value: String) extends AnyVal { + + override def toString(): String = value // scalafix:ok + +} + +object MessageId { + + def apply(string: String): MessageId = new MessageId(string) + + def unapply(value: MessageId): Some[String] = Some(value.value) + + // Cats instances + + implicit val MessageIdShow: Show[MessageId] = Show[String].contramap(_.value) + + implicit val MessageIdEqHashOrder: Eq[MessageId] with Hash[MessageId] with Order[MessageId] = + new Eq[MessageId] with Hash[MessageId] with Order[MessageId] { + + override def hash(x: MessageId): Int = Hash[String].hash(x.value) + + override def compare(x: MessageId, y: MessageId): Int = Order[String].compare(x.value, y.value) + + } + + // Circe instances + + implicit val MessageIdCodec: Codec[MessageId] = + Codec.from(Decoder[String].map(MessageId(_)), Encoder[String].contramap(_.value)) + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubClient.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubClient.scala new file mode 100644 index 00000000..6e580230 --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubClient.scala @@ -0,0 +1,311 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import java.time.Instant +import java.util.Base64 + +import scala.util.control.NonFatal + +import cats.effect.Temporal +import cats.syntax.all._ + +import com.permutive.common.types.gcp.http4s._ +import fs2.Chunk +import fs2.pubsub.dsl.client._ +import fs2.pubsub.exceptions.PubSubRequestError +import io.circe.Decoder +import io.circe.Json +import io.circe.syntax._ +import org.http4s.Method._ +import org.http4s.Uri +import org.http4s.circe._ +import org.http4s.client.dsl.Http4sClientDsl +import org.http4s.client.middleware.Retry +import org.http4s.headers.`Idempotency-Key` + +/** Represents a class designed to handle connections to Pub/Sub APIs and perform various operations on them. */ +trait PubSubClient[F[_]] { + + /** Publishes a sequence of records to the specified topic in Pub/Sub. + * + * @param topic + * the topic to publish the records to + * @param records + * a sequence of publisher records to be published + * @return + * a list of message IDs associated with the published records + */ + def publish[A: MessageEncoder](topic: Topic, records: Seq[PubSubRecord.Publisher[A]]): F[List[MessageId]] + + /** Reads a specified number of messages from the subscription in Pub/Sub. + * + * @param subscription + * the subscription to read messages from + * @param maxMessages + * the maximum number of messages to read + * @return + * a list of subscriber records containing the read messages + */ + def read( + subscription: Subscription, + maxMessages: Int + ): F[List[PubSubRecord.Subscriber[F, Array[Byte]]]] + + /** Acknowledges the receipt of a message in the subscription in Pub/Sub. + * + * @param subscription + * the subscription containing the message to acknowledge + * @param ackId + * the acknowledgment ID of the message to be acknowledged + */ + def ack(subscription: Subscription, ackId: AckId): F[Unit] = ack(subscription, Chunk(ackId)) + + /** Acknowledges the receipt of multiple messages in the subscription in Pub/Sub. + * + * @param subscription + * the subscription containing the messages to acknowledge + * @param ackIds + * the acknowledgment IDs of the messages to be acknowledged + */ + + def ack(subscription: Subscription, ackIds: Chunk[AckId]): F[Unit] + + /** Negatively acknowledges the receipt of a message in the subscription in Pub/Sub. + * + * @param subscription + * the subscription containing the message to negatively acknowledge + * @param ackId + * the acknowledgment ID of the message to be negatively acknowledged + */ + def nack(subscription: Subscription, ackId: AckId): F[Unit] = nack(subscription, Chunk(ackId)) + + /** Negatively acknowledges the receipt of multiple messages in the subscription in Pub/Sub. + * + * @param subscription + * the subscription containing the messages to negatively acknowledge + * @param ackIds + * the acknowledgment IDs of the messages to be negatively acknowledged + */ + def nack(subscription: Subscription, ackIds: Chunk[AckId]): F[Unit] = + modifyDeadline(subscription: Subscription, ackIds, AckDeadline.zero) + + /** Modifies the deadline for acknowledging a specific message in the subscription in Pub/Sub. + * + * @param subscription + * the subscription containing the message for which the deadline will be modified + * @param ackId + * the acknowledgment ID of the message for which the deadline will be modified + * @param by + * the duration by which to modify the deadline + */ + def modifyDeadline(subscription: Subscription, ackId: AckId, by: AckDeadline): F[Unit] = + modifyDeadline(subscription, Chunk(ackId), by) + + /** Modifies the deadline for acknowledging multiple messages in the subscription in Pub/Sub. + * + * @param subscription + * the subscription containing the messages for which the deadline will be modified + * @param ackIds + * the acknowledgment IDs of the messages for which the deadline will be modified + * @param by + * the deadline duration by which to modify the deadlines + */ + def modifyDeadline(subscription: Subscription, ackIds: Chunk[AckId], by: AckDeadline): F[Unit] + + /** Returns a `PubSubPublisher.Builder` to configure and create a publisher for a specific message type within + * Pub/Sub. + * + * @tparam A + * the type of message for which to create the publisher + * @return + * a `PubSubPublisher.Builder` instance for the specified message type + */ + def publisher[A: MessageEncoder]: PubSubPublisher.Builder.FromPubSubClient[F, A] = + PubSubPublisher.fromPubSubClient(this) + + /** Returns a `PubSubSubscriber.Builder` to configure and create a subscriber for handling messages within Pub/Sub. */ + def subscriber(implicit F: Temporal[F]): PubSubSubscriber.Builder.FromPubSubClient[F] = + PubSubSubscriber.fromPubSubClient(this) + +} + +object PubSubClient { + + /** Represents the configuration used by a `PubSubClient`. + * + * @param projectId + * the ID of the GCP project + * @param uri + * URI of the Pub/Sub API + */ + sealed abstract class Config private ( + val projectId: ProjectId, + val uri: Uri + ) { + + /** Sets the ID of the GCP project */ + def withProjectId(projectId: ProjectId): Config = copy(projectId = projectId) + + /** Sets the URI of the Pub/Sub API */ + def withUri(uri: Uri): Config = copy(uri = uri) + + private def copy( + projectId: ProjectId = this.projectId, + uri: Uri = this.uri + ): Config = Config(projectId, uri) + + } + + object Config { + + def apply(projectId: ProjectId, uri: Uri): Config = new Config(projectId, uri) {} + + } + + /** Starts creating an HTTP Pub/Sub client in a step-by-step fashion. + * + * @tparam F + * the effect type + */ + def http[F[_]: Temporal]: PubSubClientStep[F] = { projectId => uri => underlying => retryPolicy => + new PubSubClient[F] with Http4sClientDsl[F] { + + private val httpClient = Retry.create(retryPolicy, logRetries = false)(underlying) + + override def publish[A: MessageEncoder]( + topic: Topic, + records: Seq[PubSubRecord.Publisher[A]] + ): F[List[MessageId]] = { + val body = Json.obj( + "messages" := records.map { record => + val data = MessageEncoder[A].encode(record.data) + + Json.obj( + "data" := Base64.getEncoder().encodeToString(data), + "attributes" := record.attributes + ) + } + ) + + val request = POST(body, uri / "v1" / "projects" / projectId / "topics" / show"$topic:publish") + + httpClient + .expectOr[Json](request)(PubSubRequestError.from(_, request).widen) + .flatMap(_.hcursor.get[List[MessageId]]("messageIds").liftTo[F]) + .adaptError { + case e: PubSubRequestError => e + case NonFatal(e) => PubSubRequestError("Failed to publish records to PubSub", request, e) + } + } + + def read( + subscription: Subscription, + maxMessages: Int + ): F[List[PubSubRecord.Subscriber[F, Array[Byte]]]] = { + val body = Json.obj( + "maxMessages" := maxMessages + ) + + val request = POST( + body, + uri / "v1" / "projects" / projectId / "subscriptions" / show"$subscription:pull", + `Idempotency-Key`("pull") + ) + + implicit val decoder: Decoder[PubSubRecord.Subscriber[F, Array[Byte]]] = cursor => + for { + ackId <- cursor.get[AckId]("ackId") + message = cursor.downField("message") + data <- message.get[Option[String]]("data") + attributes <- message.get[Option[Map[String, String]]]("attributes") + messageId <- message.get[Option[MessageId]]("messageId") + publishTime <- message.get[Option[Instant]]("publishTime") + } yield PubSubRecord.Subscriber( + data.map(Base64.getDecoder().decode), + attributes.orEmpty, + messageId, + publishTime, + ackId, + ack(subscription, ackId), + nack(subscription, ackId), + modifyDeadline(subscription, ackId, _) + ) + + httpClient + .expectOr[Json](request)(PubSubRequestError.from(_, request).widen) + .flatMap(_.hcursor.get[Option[List[PubSubRecord.Subscriber[F, Array[Byte]]]]]("receivedMessages").liftTo[F]) + .map(_.getOrElse(Nil)) + .adaptError { + case e: PubSubRequestError => e + case NonFatal(e) => PubSubRequestError("Failed to pull messages from PubSub", request, e) + } + } + + final def ack(subscription: Subscription, ackIds: Chunk[AckId]): F[Unit] = { + val body = Json.obj("ackIds" := ackIds.toList) + + val request = + POST( + body, + uri / "v1" / "projects" / projectId / "subscriptions" / show"$subscription:acknowledge", + `Idempotency-Key`("acknowledge") + ) + + httpClient + .expectOr[Unit](request)(PubSubRequestError.from(_, request).widen) + .void + .adaptError { + case e: PubSubRequestError => e + case NonFatal(e) => PubSubRequestError("Failed to acknowledge messages in PubSub", request, e) + } + } + + final def modifyDeadline(subscription: Subscription, ackIds: Chunk[AckId], by: AckDeadline): F[Unit] = { + val body = Json.obj("ackIds" := ackIds.toList, "ackDeadlineSeconds" := by.value.toSeconds) + + val request = + POST( + body, + uri / "v1" / "projects" / projectId / "subscriptions" / show"$subscription:modifyAckDeadline", + `Idempotency-Key`("modifyAckDeadline") + ) + + httpClient + .expectOr[Unit](request)(PubSubRequestError.from(_, request).widen) + .void + .adaptError { + case e: PubSubRequestError => e + case NonFatal(e) => PubSubRequestError("Failed to modify ACK deadline in PubSub", request, e) + } + } + + } + + } + + // format: off + type FromConfigBuilder[F[_]] = + ClientStep[F, RetryPolicyStep[F, PubSubClient[F]]] + // format: on + + // format: off + type Builder[F[_]] = + ProjectIdStep[UriStep[ClientStep[F, RetryPolicyStep[F, PubSubClient[F]]]]] + // format: on + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubPublisher.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubPublisher.scala new file mode 100644 index 00000000..3f576863 --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubPublisher.scala @@ -0,0 +1,366 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import scala.concurrent.duration._ + +import cats.Applicative +import cats.Functor +import cats.effect.Resource +import cats.effect._ +import cats.effect.syntax.all._ +import cats.syntax.all._ + +import fs2.Chunk +import fs2.concurrent.Channel +import fs2.pubsub.dsl.client._ +import fs2.pubsub.dsl.publisher._ +import org.http4s.Uri + +/** Represents a class defining a Pub/Sub publisher responsible for producing messages of type `A`. + * + * @tparam F + * the effect type + * @tparam A + * the type of messages to be published + */ +trait PubSubPublisher[F[_], A] { + + /** Produces a single message with the provided data and optional attributes. + * + * @param data + * the data of type `A` to be sent as a message + * @param attributes + * optional key-value pairs representing message attributes + * @return + * the message ID associated with the published message + */ + def publishOne(data: A, attributes: (String, String)*)(implicit F: Functor[F]): F[MessageId] = + publishOne(data, attributes.toMap) + + /** Produces a single message with the provided data and map of attributes. + * + * @param data + * the data of type `A` to be sent as a message + * @param attributes + * a map of key-value pairs representing message attributes + * @return + * the message ID associated with the published message + */ + def publishOne(data: A, attributes: Map[String, String])(implicit F: Functor[F]): F[MessageId] = + publishOne(PubSubRecord.Publisher(data).withAttributes(attributes)) + + /** Produces a single message based on the given publisher `PubSubRecord` instance. + * + * @param record + * the publisher `PubSubRecord` instance representing the message to be published + * @return + * the message ID associated with the published message + */ + def publishOne(record: PubSubRecord.Publisher[A])(implicit F: Functor[F]): F[MessageId] = + publishMany(Seq(record)).map(_.head) + + /** Produces multiple messages based on the given sequence of `PubSubRecord.Publisher`. + * + * @param records + * a sequence of publisher `PubSubRecord` instances representing messages to be published + * @return + * a list of message IDs associated with the published messages + */ + def publishMany(records: Seq[PubSubRecord.Publisher[A]]): F[List[MessageId]] + + /** Starts configuring an asynchronous `PubSub` publisher from this `PubSubPublisher`. */ + def batching(implicit F: Temporal[F]): PubSubPublisher.Async.Builder.Default[F, A] = batchSize => + maxLatency => PubSubPublisher.Async.fromPubSubPublisher(batchSize, maxLatency)(this) + +} + +object PubSubPublisher { + + /** Represents the configuration parameters for a `PubSubPublisher`. + * + * @param projectId + * the ID of the GCP project + * @param topic + * the topic into which messages will be published + * @param uri + * URI of the Pub/Sub API + */ + sealed abstract class Config private ( + val projectId: ProjectId, + val topic: Topic, + val uri: Uri + ) { + + /** Sets the ID of the GCP project */ + def withProjectId(projectId: ProjectId): Config = copy(projectId = projectId) + + /** Sets the topic into which messages will be published */ + def withTopic(topic: Topic): Config = copy(topic = topic) + + /** Sets the URI of the Pub/Sub API */ + def withUri(uri: Uri): Config = copy(uri = uri) + + private def copy( + projectId: ProjectId = this.projectId, + topic: Topic = this.topic, + uri: Uri = this.uri + ): Config = Config(projectId, topic, uri) + + } + + object Config { + + /** Creates the configuration parameters for a `PubSubPublisher`. + * + * @param projectId + * the ID of the GCP project + * @param topic + * the topic into which messages will be published + * @param uri + * URI of the Pub/Sub API + */ + def apply(projectId: ProjectId, topic: Topic, uri: Uri): Config = new Config(projectId, topic, uri) {} + + } + + /** Starts creating an HTTP Pub/Sub publisher in a step-by-step fashion. + * + * @tparam F + * the effect type + * @tparam A + * the type of messages to be sent to Pub/Sub + */ + def http[F[_]: Temporal, A: MessageEncoder]: PubSubPublisherStep[F, A] = { + projectId => topic => uri => client => retryPolicy => + PubSubClient.http + .projectId(projectId) + .uri(uri) + .httpClient(client) + .retryPolicy(retryPolicy) + .publisher + .topic(topic) + } + + /** Creates a builder for configuring a `PubSubPublisher` by leveraging an existing `PubSubClient`. + * + * @tparam F + * the effect type + * @tparam A + * the type of messages to be sent to Pub/Sub + * @param pubSubClient + * the Pub/Sub client used for producing messages + * @return + * a builder instance sourcing configuration from the provided `PubSubClient` + */ + def fromPubSubClient[F[_], A: MessageEncoder](pubSubClient: PubSubClient[F]): Builder.FromPubSubClient[F, A] = + topic => records => pubSubClient.publish[A](topic, records) + + object Builder { + + type Default[F[_], A] = ProjectIdStep[TopicStep[UriStep[ClientStep[F, RetryPolicyStep[F, PubSubPublisher[F, A]]]]]] + + type FromConfig[F[_], A] = ClientStep[F, RetryPolicyStep[F, PubSubPublisher[F, A]]] + + type FromPubSubClient[F[_], A] = TopicStep[PubSubPublisher[F, A]] + + } + + /** Represents a Pub/Sub publisher capable of producing messages of type `A` asynchronously. + * + * @tparam F + * the effect type + * @tparam A + * the type of messages to be handled asynchronously + */ + trait Async[F[_], A] { + + /** Produces a single message with the provided data and optional attributes. + * + * @param data + * the data of type `A` to be sent as a message + * @param attributes + * optional key-value pairs representing message attributes + */ + def publishOne(data: A, attributes: (String, String)*)(implicit F: Applicative[F]): F[Unit] = + publishOne(data, attributes.toMap) + + /** Produces a single message with the provided data and map of attributes. + * + * @param data + * the data of type `A` to be sent as a message + * @param attributes + * a map of key-value pairs representing message attributes + */ + def publishOne(data: A, attributes: Map[String, String])(implicit F: Applicative[F]): F[Unit] = + publishOne(PubSubRecord.Publisher(data).withAttributes(attributes)) + + /** Produces a single message with the provided data, callback function, and optional attributes. + * + * @param data + * the data of type `A` to be sent as a message + * @param callback + * the callback function to execute after producing the message + * @param attributes + * optional key-value pairs representing message attributes + */ + def publishOne(data: A, callback: Either[Throwable, Unit] => F[Unit], attributes: (String, String)*): F[Unit] = + publishOne(data, callback, attributes.toMap) + + /** Produces a single message with the provided data, callback function, and map of attributes. + * + * @param data + * the data of type `A` to be sent as a message + * @param callback + * the callback function to execute after producing the message + * @param attributes + * a map of key-value pairs representing message attributes + */ + def publishOne(data: A, callback: Either[Throwable, Unit] => F[Unit], attributes: Map[String, String]): F[Unit] = + publishOne(PubSubRecord.Publisher(data).withAttributes(attributes).withCallback(callback)) + + /** Produces a single message based on the given `PubSubRecord.Publisher` containing a callback. + * + * @param record + * the publisher `PubSubRecord` instance with a callback function for handling published message + */ + def publishOne(record: PubSubRecord.Publisher.WithCallback[F, A]): F[Unit] = + publishMany(Seq(record)) + + /** Produces a single message based on the provided publisher `PubSubRecord` instance. + * + * @param record + * the publisher `PubSubRecord` instance representing the message to be published + */ + def publishOne(record: PubSubRecord.Publisher[A])(implicit F: Applicative[F]): F[Unit] = + publishOne(record.withCallback(_ => Applicative[F].unit)) + + /** Produces multiple messages based on the given sequence of publisher `PubSubRecord` with callback instances. + * + * @param records + * a sequence of publisher `PubSubRecord` with callback instances + */ + def publishMany(records: Seq[PubSubRecord.Publisher.WithCallback[F, A]]): F[Unit] + + } + + object Async { + + /** Represents the configuration parameters for an async `PubSubPublisher`. + * + * @param projectId + * the ID of the GCP project + * @param topic + * the topic into which messages will be published + * @param uri + * URI of the Pub/Sub API + * @param batchSize + * the maximum size of message batches for processing + * @param maxLatency + * the maximum duration allowed for latency in message processing + */ + sealed abstract class Config private ( + val projectId: ProjectId, + val topic: Topic, + val uri: Uri, + val batchSize: Int, + val maxLatency: FiniteDuration + ) { + + /** Sets the ID of the GCP project */ + def withProjectId(projectId: ProjectId): Config = copy(projectId = projectId) + + /** Sets the topic into which messages will be published */ + def withTopic(topic: Topic): Config = copy(topic = topic) + + /** Sets the URI of the Pub/Sub API */ + def withUri(uri: Uri): Config = copy(uri = uri) + + /** Sets the maximum size of message batches for processing */ + def withBatchSize(batchSize: Int): Config = copy(batchSize = batchSize) + + /** Sets the maximum duration allowed for latency in message processing */ + def withMaxLatency(maxLatency: FiniteDuration): Config = copy(maxLatency = maxLatency) + + private def copy( + projectId: ProjectId = this.projectId, + topic: Topic = this.topic, + uri: Uri = this.uri, + batchSize: Int = this.batchSize, + maxLatency: FiniteDuration = this.maxLatency + ): Config = Config(projectId, topic, uri, batchSize, maxLatency) + + } + + object Config { + + /** Creates the configuration parameters for an async `PubSubPublisher`. + * + * @param projectId + * the ID of the GCP project + * @param topic + * the topic into which messages will be published + * @param uri + * URI of the Pub/Sub API + * @param batchSize + * the maximum size of message batches for processing. Defaults to 100 + * @param maxLatency + * the maximum duration allowed for latency in message processing. Defaults to 1 second + */ + def apply( + projectId: ProjectId, + topic: Topic, + uri: Uri, + batchSize: Int = 100, + maxLatency: FiniteDuration = 1.second + ): Config = new Config(projectId, topic, uri, batchSize, maxLatency) {} + + } + + object Builder { + + type Default[F[_], A] = + BatchSizeStep[MaxLatencyStep[Resource[F, PubSubPublisher.Async[F, A]]]] + + type FromConfig[F[_], A] = + ClientStep[F, RetryPolicyStep[F, Resource[F, PubSubPublisher.Async[F, A]]]] + + } + + private[pubsub] def fromPubSubPublisher[F[_]: Temporal, A]( + batchSize: Int, + maxLatency: FiniteDuration + )(underlying: PubSubPublisher[F, A]): Resource[F, PubSubPublisher.Async[F, A]] = { + val publish = (records: Chunk[PubSubRecord.Publisher.WithCallback[F, A]]) => + underlying + .publishMany(records.toList.map(_.noCallback)) + .void + .attempt + .flatMap(result => records.traverse_(_.callback(result))) + + val channelAndFiber = Channel + .unbounded[F, PubSubRecord.Publisher.WithCallback[F, A]] + .mproduct(_.stream.groupWithin(batchSize, maxLatency).evalMap(publish).compile.drain.start) + + Resource + .make(channelAndFiber) { case (channel, fiber) => channel.close.void >> fiber.join.void } + .map { case (channel, _) => _.toList.traverse_(channel.send).void } + } + + } + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubRecord.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubRecord.scala new file mode 100644 index 00000000..b76c1b69 --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubRecord.scala @@ -0,0 +1,327 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import java.time.Instant + +import cats.Show +import cats.syntax.all._ + +object PubSubRecord { + + /** Represents a message that has been received from a Pub/Sub subscription. + * + * @param value + * the message payload + * @param attributes + * the message attributes + * @param messageId + * the unique identifier for the message + * @param publishTime + * the time at which the message was published + * @param ackId + * the unique identifier for the acknowledgment of the message + * @param ack + * the function to acknowledge the message + * @param nack + * the function to negatively acknowledge the message + * @param extendDeadline + * the function to extend the deadline for acknowledging the message + */ + sealed abstract class Subscriber[F[_], +A] private ( + val value: Option[A], + val attributes: Map[String, String], + val messageId: Option[MessageId], + val publishTime: Option[Instant], + val ackId: AckId, + val ack: F[Unit], + val nack: F[Unit], + val extendDeadline: AckDeadline => F[Unit] + ) { + + private def copy[B]( + value: Option[B] = this.value, + attributes: Map[String, String] = this.attributes, + messageId: Option[MessageId] = this.messageId, + publishTime: Option[Instant] = this.publishTime, + ackId: AckId = this.ackId, + ack: F[Unit] = this.ack, + nack: F[Unit] = this.nack, + extendDeadline: AckDeadline => F[Unit] = this.extendDeadline + ): Subscriber[F, B] = Subscriber(value, attributes, messageId, publishTime, ackId, ack, nack, extendDeadline) + + @SuppressWarnings(Array("scalafix:DisableSyntax.==", "scalafix:Disable.equals")) + override def equals(obj: Any): Boolean = obj match { + case record: Subscriber[_, _] => + this.value == record.value && this.attributes === record.attributes && this.messageId === record.messageId && + this.publishTime == record.publishTime && this.ackId === record.ackId + case _ => false + } + + @SuppressWarnings(Array("scalafix:Disable.toString")) + override def toString(): String = s"PubSubRecord.Subscriber($value, $attributes, $messageId, $publishTime, $ackId)" + + /** Updates the function to acknowledge the message */ + def withAck(f: F[Unit]): PubSubRecord.Subscriber[F, A] = copy(ack = f) + + /** Updates the function to negatively acknowledge the message */ + def withNack(f: F[Unit]): PubSubRecord.Subscriber[F, A] = copy(nack = f) + + /** Contramaps the value of the message from type `B` to type `A` using the provided function. + * + * @param f + * the function to contramap the value of the message from type `B` to type `A` + * @return + * a new `PubSubRecord.Subscriber` instance for the contramapped type `B` + */ + def map[B](f: A => B): Subscriber[F, B] = copy(value = value.map(f)) + + /** Maps the value of the message from type `A` to type `B` using the provided function that may result in an + * `Either`. + * + * @param f + * the function that may result in an `Either` value for mapping to type `B` + * @return + * a new `PubSubRecord.Subscriber` instance for the mapped type `B` + */ + def emap[B](f: A => Either[Throwable, B]): Either[Throwable, Subscriber[F, B]] = + value + .map(f(_).map(v => copy(value = v.some))) + .getOrElse(copy(value = None).asRight) + + } + + object Subscriber { + + /** Creates a new `PubSubRecord.Subscriber` from the provided parameters. + * + * @param value + * the message payload + * @param attributes + * the message attributes + * @param messageId + * the unique identifier for the message + * @param publishTime + * the time at which the message was published + * @param ackId + * the unique identifier for the acknowledgment of the message + * @param ack + * the function to acknowledge the message + * @param nack + * the function to negatively acknowledge the message + * @param extendDeadline + * the function to extend the deadline for acknowledging the message + * @return + * a new `PubSubRecord.Subscriber` instance + */ + def apply[F[_], A]( + value: Option[A], + attributes: Map[String, String], + messageId: Option[MessageId], + publishTime: Option[Instant], + ackId: AckId, + ack: F[Unit], + nack: F[Unit], + extendDeadline: AckDeadline => F[Unit] + ): PubSubRecord.Subscriber[F, A] = + new PubSubRecord.Subscriber(value, attributes, messageId, publishTime, ackId, ack, nack, extendDeadline) {} + + // format: off + def unapply[F[_], A](record: PubSubRecord.Subscriber[F, A]): Some[(Option[A], Map[String, String], Option[MessageId], Option[Instant], AckId, F[Unit], F[Unit], AckDeadline => F[Unit])] = + Some((record.value, record.attributes, record.messageId, record.publishTime, record.ackId, record.ack, record.nack, record.extendDeadline)) + // format: on + + implicit def show[F[_], A: Show]: Show[PubSubRecord.Subscriber[F, A]] = record => + show"PubSubRecord.Subscriber(${record.value}, ${record.attributes}, ${record.messageId}, ${s"${record.publishTime}"}, ${record.ackId})" + + implicit class PubSubRecordSubscriberSyntax[F[_]](subscriber: PubSubRecord.Subscriber[F, Array[Byte]]) { + + /** Decodes the message payload of the `PubSubRecord.Subscriber` into the specified message type. + * + * @tparam B + * the type of message to be decoded + * @return + * either the decoded message of type `B` or an exception if the decoding fails + */ + def as[B: MessageDecoder]: Either[Throwable, Subscriber[F, B]] = subscriber.emap(MessageDecoder[B].decode) + + } + + } + + /** Represents a message that is to be published to a Pub/Sub topic. + * + * @param data + * the message payload + * @param attributes + * the message attributes + */ + sealed abstract class Publisher[A] private (val data: A, val attributes: Map[String, String]) { + + private def copy[B]( + data: B = this.data, + attributes: Map[String, String] = this.attributes + ): Publisher[B] = Publisher(data, attributes) + + @SuppressWarnings(Array("scalafix:DisableSyntax.==", "scalafix:Disable.equals")) + override def equals(obj: Any): Boolean = obj match { + case record: Publisher[_] => this.data == record.data && this.attributes === record.attributes + case _ => false + } + + @SuppressWarnings(Array("scalafix:Disable.toString")) + override def toString(): String = s"PubSubRecord.Publisher($data, $attributes)" + + /** Updates the message payload of the record */ + def withData[B](data: B): Publisher[B] = + copy(data = data) + + /** Adds a new attribute to the record */ + def withAttribute(key: String, value: String): Publisher[A] = + withAttributes((key, value)) + + /** Adds multiple attributes to the record */ + def withAttributes(first: (String, String), rest: (String, String)*): Publisher[A] = + withAttributes((first +: rest).toMap) + + /** Adds multiple attributes to the record */ + def withAttributes(attributes: Map[String, String]): Publisher[A] = + copy(attributes = this.attributes ++ attributes) + + /** Adds a callback to the record. The resulting record can only be published with an async `PubSubPublisher` + * + * @see + * [[PubSubPublisher.Async]] + */ + def withCallback[F[_]](callback: Either[Throwable, Unit] => F[Unit]): Publisher.WithCallback[F, A] = + Publisher.WithCallback[F, A](this.data, this.attributes, callback) + + } + + object Publisher { + + /** Creates a new `PubSubRecord.Publisher` from the provided parameters. + * + * @param data + * the message payload + * @return + * a new `PubSubRecord.Publisher` instance + */ + def apply[A](data: A): Publisher[A] = Publisher[A](data, Map.empty[String, String]) + + /** Creates a new `PubSubRecord.Publisher` from the provided parameters. + * + * @param data + * the message payload + * @param attributes + * the message attributes + * @return + * a new `PubSubRecord.Publisher` instance + */ + def apply[A](data: A, attributes: Map[String, String]): Publisher[A] = new Publisher[A](data, attributes) {} + + def unapply[A](record: PubSubRecord.Publisher[A]): Some[(A, Map[String, String])] = + Some((record.data, record.attributes)) + + implicit def show[A: Show]: Show[PubSubRecord.Publisher[A]] = record => + show"PubSubRecord.Publisher(${record.data}, ${record.attributes})" + + /** Represents a message that is to be published asynchronously to a Pub/Sub topic. + * + * @param data + * the message payload + * @param attributes + * the message attributes + * @param callback + * the callback to be executed after the message has been published + */ + sealed abstract class WithCallback[F[_], A] private ( + val data: A, + val attributes: Map[String, String], + val callback: Either[Throwable, Unit] => F[Unit] + ) { + + private def copy[B]( + data: B = this.data, + attributes: Map[String, String] = this.attributes, + callback: Either[Throwable, Unit] => F[Unit] = this.callback + ): WithCallback[F, B] = WithCallback[F, B](data, attributes, callback) + + @SuppressWarnings(Array("scalafix:DisableSyntax.==", "scalafix:Disable.equals")) + override def equals(obj: Any): Boolean = obj match { + case record: Publisher.WithCallback[_, _] => this.data == record.data && this.attributes === record.attributes + case _ => false + } + + @SuppressWarnings(Array("scalafix:Disable.toString")) + override def toString(): String = s"PubSubRecord.Publisher.WithCallback($data, $attributes)" + + /** Removes the callback from the record. The resulting record can only be published with a normal + * `PubSubPublisher`. + */ + def noCallback: Publisher[A] = Publisher[A](this.data, this.attributes) + + /** Adds an attribute to the record. */ + def withAttribute(key: String, value: String): Publisher.WithCallback[F, A] = + withAttributes((key, value)) + + /** Adds multiple attributes to the record */ + def withAttributes(first: (String, String), rest: (String, String)*): Publisher.WithCallback[F, A] = + withAttributes((first +: rest).toMap) + + /** Adds multiple attributes to the record */ + def withAttributes(attributes: Map[String, String]): Publisher.WithCallback[F, A] = + copy[A](attributes = this.attributes ++ attributes) + + /** Updates the record's callback */ + def withCallback(callback: Either[Throwable, Unit] => F[Unit]): Publisher.WithCallback[F, A] = + copy[A](callback = callback) + + } + + object WithCallback { + + /** Creates a new `PubSubRecord.Publisher.WithCallback` from the provided parameters. + * + * @param data + * the message payload + * @param attributes + * the message attributes + * @param callback + * the callback to be executed after the message has been published + * @return + * a new `PubSubRecord.Publisher.WithCallback` instance + */ + def apply[F[_], A]( + data: A, + attributes: Map[String, String], + callback: Either[Throwable, Unit] => F[Unit] + ): WithCallback[F, A] = new WithCallback[F, A](data, attributes, callback) {} + + def unapply[F[_], A]( + record: PubSubRecord.Publisher.WithCallback[F, A] + ): Some[(A, Map[String, String], Either[Throwable, Unit] => F[Unit])] = + Some((record.data, record.attributes, record.callback)) + + implicit def show[A: Show]: Show[PubSubRecord.Publisher[A]] = record => + show"PubSubRecord.Publisher.WithCallback(${record.data}, ${record.attributes})" + + } + + } + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubSubscriber.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubSubscriber.scala new file mode 100644 index 00000000..7206743e --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubSubscriber.scala @@ -0,0 +1,243 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import scala.concurrent.duration._ + +import cats.Applicative +import cats.effect.Resource +import cats.effect.Temporal +import cats.effect.syntax.all._ +import cats.syntax.all._ + +import fs2.Chunk +import fs2.Stream +import fs2.concurrent.Channel +import fs2.pubsub.dsl.client._ +import fs2.pubsub.dsl.subscriber._ +import org.http4s.Uri + +/** Contains method for creating Pub/Sub subscribers. */ +object PubSubSubscriber { + + /** Represents the configuration used by a `PubSubClient`. + * + * @param projectId + * the ID of the GCP project + * @param subscription + * the subscription from which messages will be read + * @param uri + * URI of the Pub/Sub API + * @param batchSize + * the maximum size of message batches for processing + * @param maxLatency + * the maximum duration allowed for latency in message processing + * @param readMaxMessages + * the maximum number of messages to read at once + * @param readConcurrency + * the concurrency level for reading messages + */ + sealed abstract class Config private ( + val projectId: ProjectId, + val subscription: Subscription, + val uri: Uri, + val batchSize: Int, + val maxLatency: FiniteDuration, + val readMaxMessages: Int, + val readConcurrency: Int + ) { + + /** Sets the ID of the GCP project */ + def withProjectId(projectId: ProjectId): Config = copy(projectId = projectId) + + /** Sets the subscription from which messages will be read */ + def withSubscription(subscription: Subscription): Config = copy(subscription = subscription) + + /** Sets the URI of the Pub/Sub API */ + def withUri(uri: Uri): Config = copy(uri = uri) + + /** Sets the maximum size of message batches for processing */ + def withBatchSize(batchSize: Int): Config = copy(batchSize = batchSize) + + /** Sets the maximum duration allowed for latency in message processing */ + def withMaxLatency(maxLatency: FiniteDuration): Config = copy(maxLatency = maxLatency) + + /** Sets the maximum number of messages to read at once */ + def withReadMaxMessages(readMaxMessages: Int): Config = copy(readMaxMessages = readMaxMessages) + + /** Sets the concurrency level for reading messages */ + def withReadConcurrency(readConcurrency: Int): Config = copy(readConcurrency = readConcurrency) + + private def copy( + projectId: ProjectId = this.projectId, + subscription: Subscription = this.subscription, + uri: Uri = this.uri, + batchSize: Int = this.batchSize, + maxLatency: FiniteDuration = this.maxLatency, + readMaxMessages: Int = this.readMaxMessages, + readConcurrency: Int = this.readConcurrency + ): Config = + Config(projectId, subscription, uri, batchSize, maxLatency, readMaxMessages, readConcurrency) + + } + + object Config { + + /** Creates the configuration used by a `PubSubClient`. + * + * @param projectId + * the ID of the GCP project + * @param subscription + * the subscription from which messages will be read + * @param uri + * URI of the Pub/Sub API + * @param batchSize + * the maximum size of message batches for processing. Defaults to 100 + * @param maxLatency + * the maximum duration allowed for latency in message processing. Defaults to 1 second + * @param readMaxMessages + * the maximum number of messages to read at once. Defaults to 1000 + * @param readConcurrency + * the concurrency level for reading messages. Defaults to 1 (no concurrency) + */ + def apply( + projectId: ProjectId, + subscription: Subscription, + uri: Uri, + batchSize: Int = 100, + maxLatency: FiniteDuration = 1.second, + readMaxMessages: Int = 1000, + readConcurrency: Int = 1 + ): Config = new Config(projectId, subscription, uri, batchSize, maxLatency, readMaxMessages, readConcurrency) {} + + } + + /** Starts creating an HTTP Pub/Sub subscriber in a step-by-step fashion. + * + * @tparam F + * the effect type + */ + def http[F[_]: Temporal]: PubSubSubscriberStep[F] = { projectId => subscription => uri => client => retryPolicy => + PubSubClient.http + .projectId(projectId) + .uri(uri) + .httpClient(client) + .retryPolicy(retryPolicy) + .subscriber + .subscription(subscription) + } + + /** Creates a builder for configuring a Pub/Sub subscriber by leveraging an existing `PubSubClient`. + * + * @tparam F + * the effect type + * @param pubSubClient + * the Pub/Sub client used for reading messages + * @return + * a builder instance sourcing configuration from the provided `PubSubClient` + */ + def fromPubSubClient[F[_]: Temporal](pubSubClient: PubSubClient[F]): Builder.FromPubSubClient[F] = { + subscription => (errorHandler: PartialFunction[(PubSubSubscriber.Operation[F], Throwable), F[Unit]]) => batchSize => + maxLatency => readMaxMessages => readConcurrency => + def ackChannel(processRecords: Chunk[PubSubRecord.Subscriber[F, Array[Byte]]] => F[Unit]) = + Stream.resource(Resource { + Channel + .unbounded[F, PubSubRecord.Subscriber[F, Array[Byte]]] + .mproduct(_.stream.groupWithin(batchSize, maxLatency).evalMap(processRecords).compile.drain.start) + .map { case (channel, fiber) => (channel, channel.close.void >> fiber.join.void) } + }) + + val stream = for { + ack <- ackChannel { records => + pubSubClient + .ack(subscription, records.map(_.ackId)) + .handleErrorWith { + case e if errorHandler.isDefinedAt((Operation.Ack(records), e)) => + errorHandler((Operation.Ack(records), e)) + case _ => Applicative[F].unit + } + } + nack <- ackChannel { records => + pubSubClient + .nack(subscription, records.map(_.ackId)) + .handleErrorWith { + case e if errorHandler.isDefinedAt((Operation.Nack(records), e)) => + errorHandler((Operation.Nack(records), e)) + case _ => Applicative[F].unit + } + } + record <- + Stream + .emit(pubSubClient.read(subscription, readMaxMessages)) + .repeat + .covary[F] + .mapAsyncUnordered(readConcurrency)(identity) + .flatMap(Stream.emits) + } yield record.withAck(ack.send(record).void).withNack(nack.send(record).void) + + SubscriberStep(stream, errorHandler) + } + + object Builder { + + type Default[F[_]] = + ProjectIdStep[SubscriptionStep[UriStep[ClientStep[F, RetryPolicyStep[F, ErrorHandlerStep[F, WithDefaults[F]]]]]]] + + type WithDefaults[F[_]] = + BatchSizeStep[MaxLatencyStep[ReadMaxMessagesStep[ReadConcurrencyStep[SubscriberStep[F]]]]] + + type FromConfig[F[_]] = + ClientStep[F, RetryPolicyStep[F, ErrorHandlerStep[F, SubscriberStep[F]]]] + + type FromPubSubClient[F[_]] = + SubscriptionStep[ErrorHandlerStep[F, WithDefaults[F]]] + + } + + /** Represents various operations that can fail while consuming messages from Pub/Sub. + * + * @tparam F + * the effect type for the operations + */ + sealed trait Operation[F[_]] + + object Operation { + + /** Represents the operation to acknowledge specific records. + * + * @param records + * the chunk of subscriber records to be acknowledged + */ + final case class Ack[F[_]](val records: Chunk[PubSubRecord.Subscriber[F, Array[Byte]]]) extends Operation[F] + + /** Represents the operation to negatively acknowledge specific records. + * + * @param records + * the chunk of subscriber records to be negatively acknowledged + */ + final case class Nack[F[_]](val records: Chunk[PubSubRecord.Subscriber[F, Array[Byte]]]) extends Operation[F] + + /** Represents the operation to decode a single record. + * + * @param record + * the subscriber record to be decoded + */ + final case class Decode[F[_]](val record: PubSubRecord.Subscriber[F, Array[Byte]]) extends Operation[F] + + } + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/Subscription.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/Subscription.scala new file mode 100644 index 00000000..e5a93581 --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/Subscription.scala @@ -0,0 +1,57 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import cats._ +import cats.syntax.all._ + +import io.circe.Codec +import io.circe.Decoder +import io.circe.Encoder + +/** Represents a subscription to a Pub/Sub topic. */ +final class Subscription(val value: String) extends AnyVal { + + override def toString(): String = value // scalafix:ok + +} + +object Subscription { + + def apply(string: String): Subscription = new Subscription(string) + + def unapply(value: Subscription): Some[String] = Some(value.value) + + // Cats instances + + implicit val SubscriptionShow: Show[Subscription] = Show[String].contramap(_.value) + + implicit val SubscriptionEqHashOrder: Eq[Subscription] with Hash[Subscription] with Order[Subscription] = + new Eq[Subscription] with Hash[Subscription] with Order[Subscription] { + + override def hash(x: Subscription): Int = Hash[String].hash(x.value) + + override def compare(x: Subscription, y: Subscription): Int = Order[String].compare(x.value, y.value) + + } + + // Circe instances + + implicit val SubscriptionCodec: Codec[Subscription] = + Codec.from(Decoder[String].map(Subscription(_)), Encoder[String].contramap(_.value)) + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/Topic.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/Topic.scala new file mode 100644 index 00000000..a48484b4 --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/Topic.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import cats._ +import cats.syntax.all._ + +import io.circe.Codec +import io.circe.Decoder +import io.circe.Encoder + +/** Represents a Pub/Sub topic. */ +final class Topic(val value: String) extends AnyVal { + + override def toString(): String = value // scalafix:ok + +} + +object Topic { + + def apply(string: String): Topic = new Topic(string) + + def unapply(value: Topic): Some[String] = Some(value.value) + + // Cats instances + + implicit val TopicShow: Show[Topic] = Show[String].contramap(_.value) + + implicit val TopicEqHashOrder: Eq[Topic] with Hash[Topic] with Order[Topic] = + new Eq[Topic] with Hash[Topic] with Order[Topic] { + + override def hash(x: Topic): Int = Hash[String].hash(x.value) + + override def compare(x: Topic, y: Topic): Int = Order[String].compare(x.value, y.value) + + } + + // Circe instances + + implicit val TopicCodec: Codec[Topic] = Codec.from(Decoder[String].map(Topic(_)), Encoder[String].contramap(_.value)) + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/circe/package.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/circe/package.scala new file mode 100644 index 00000000..37d35cdd --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/circe/package.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import io.circe.Decoder +import io.circe.Encoder +import io.circe.syntax._ + +package object circe { + + implicit def CirceDecoder2MessageDecoder[A: Decoder]: MessageDecoder[A] = + MessageDecoder.json.emap(_.as[A]) + + implicit def CirceEncoder2MessageEncoder[A: Encoder]: MessageEncoder[A] = + MessageEncoder.json.contramap(_.asJson) + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/dsl/client.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/dsl/client.scala new file mode 100644 index 00000000..e4b48e14 --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/dsl/client.scala @@ -0,0 +1,128 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub.dsl + +import scala.concurrent.duration._ + +import cats.effect.Temporal + +import com.permutive.common.types.gcp.ProjectId +import fs2.pubsub.PubSubClient +import org.http4s.Uri +import org.http4s.client.Client +import org.http4s.client.middleware.RetryPolicy +import org.http4s.client.middleware.RetryPolicy.exponentialBackoff +import org.http4s.syntax.all._ + +object client { + + trait PubSubClientStep[F[_]] extends PubSubClient.Builder[F] { + + /** Create a `PubSubClient` from a configuration. + * + * @param config + * the configuration to use + * @param F + * the effect type + * @return + * a builder for the client + */ + def fromConfig(config: PubSubClient.Config)(implicit F: Temporal[F]): PubSubClient.FromConfigBuilder[F] = { + PubSubClient + .http[F] + .projectId(config.projectId) + .uri(config.uri) + } + + } + + trait ProjectIdStep[A] { + + /** Sets the GCP project ID. + * + * @param projectId + * the GCP project ID + * @return + * the next step in the builder + */ + def projectId(projectId: ProjectId): A + + } + + trait UriStep[A] { + + /** Sets the Pub/Sub URI. + * + * @param uri + * the Pub/Sub URI + * @return + * the next step in the builder + */ + def uri(uri: Uri): A + + /** Configures the builder to use the default Pub/Sub URI. + * + * @return + * the next step in the builder + */ + def defaultUri: A = uri(uri"https://pubsub.googleapis.com") + + } + + trait ClientStep[F[_], A] { + + /** Sets the HTTP client to use. If the API requires authentication, the client should be configured with the + * necessary credentials. You can use `permutive-engineering/gcp-auth` to create an authenticated client. + * + * @param client + * the HTTP client + * @return + * the next step in the builder + */ + def httpClient(client: Client[F]): A + + } + + trait RetryPolicyStep[F[_], A] { + + /** Sets the retry policy to use for the client. + * + * @param retryPolicy + * the retry policy + * @return + * the next step in the builder + */ + def retryPolicy(retryPolicy: RetryPolicy[F]): A + + /** Configures the builder to not retry requests. + * + * @return + * the next step in the builder + */ + def noRetry: A = retryPolicy((_, _, _) => None) + + /** Configures the builder to retry requests recklessly. This will retry requests with exponential backoff of 5 + * seconds (up to a maximum of 3 requests). + * + * @return + * the next step in the builder + */ + def defaultRetry: A = retryPolicy(RetryPolicy(exponentialBackoff(5.seconds, maxRetry = 3))) + + } + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/dsl/publisher.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/dsl/publisher.scala new file mode 100644 index 00000000..37e19f3c --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/dsl/publisher.scala @@ -0,0 +1,125 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub.dsl + +import scala.concurrent.duration.FiniteDuration + +import cats.effect.Temporal + +import fs2.pubsub.MessageEncoder +import fs2.pubsub.PubSubPublisher +import fs2.pubsub.Topic + +object publisher { + + trait PubSubPublisherStep[F[_], A] extends PubSubPublisher.Builder.Default[F, A] { + + /** Create a `PubSubPublisher` from a configuration. + * + * @param config + * the configuration to use + * @param F + * the effect type + * @param A + * the message type + * @return + * a builder for the publisher + */ + def fromConfig( + config: PubSubPublisher.Config + )(implicit + F: Temporal[F], + A: MessageEncoder[A] + ): PubSubPublisher.Builder.FromConfig[F, A] = { + PubSubPublisher + .http[F, A] + .projectId(config.projectId) + .topic(config.topic) + .uri(config.uri) + } + + /** Create an async `PubSubPublisher` from a configuration. + * + * @param config + * the configuration to use + * @param F + * the effect type + * @param A + * the message type + * @return + * a builder for the publisher + */ + def fromConfig( + config: PubSubPublisher.Async.Config + )(implicit + F: Temporal[F], + A: MessageEncoder[A] + ): PubSubPublisher.Async.Builder.FromConfig[F, A] = { client => retryPolicy => + PubSubPublisher + .http[F, A] + .projectId(config.projectId) + .topic(config.topic) + .uri(config.uri) + .httpClient(client) + .retryPolicy(retryPolicy) + .batching + .batchSize(config.batchSize) + .maxLatency(config.maxLatency) + } + + } + + trait TopicStep[A] { + + /** Sets the Pub/Sub topic to which messages will be published. + * + * @param topic + * the Pub/Sub topic + * @return + * the next step in the builder + */ + def topic(topic: Topic): A + + } + + trait BatchSizeStep[A] { + + /** Sets the maximum size of message batches for publishing. + * + * @param batchSize + * the maximum size of message batches + * @return + * the next step in the builder + */ + def batchSize(batchSize: Int): A + + } + + trait MaxLatencyStep[A] { + + /** Sets the maximum duration allowed for latency in message publishing. + * + * @param maxLatency + * the maximum duration allowed for latency + * @return + * the next step in the builder + */ + def maxLatency(maxLatency: FiniteDuration): A + + } + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/dsl/subscriber.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/dsl/subscriber.scala new file mode 100644 index 00000000..dccd5ed8 --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/dsl/subscriber.scala @@ -0,0 +1,235 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub.dsl + +import scala.concurrent.duration.FiniteDuration +import scala.concurrent.duration._ + +import cats.Applicative +import cats.effect.Temporal +import cats.syntax.all._ + +import fs2.Stream +import fs2.pubsub.MessageDecoder +import fs2.pubsub.PubSubRecord +import fs2.pubsub.PubSubSubscriber +import fs2.pubsub.PubSubSubscriber.Operation +import fs2.pubsub.Subscription + +object subscriber { + + trait PubSubSubscriberStep[F[_]] extends PubSubSubscriber.Builder.Default[F] { + + /** Create a `PubSubSubscriber` from a configuration. + * + * @param config + * the configuration to use + * @param F + * the effect type + * @return + * a builder for the subscriber + */ + def fromConfig(config: PubSubSubscriber.Config)(implicit + F: Temporal[F] + ): PubSubSubscriber.Builder.FromConfig[F] = { client => retryPolicy => errorHandler => + PubSubSubscriber + .http[F] + .projectId(config.projectId) + .subscription(config.subscription) + .uri(config.uri) + .httpClient(client) + .retryPolicy(retryPolicy) + .errorHandler(errorHandler) + .batchSize(config.batchSize) + .maxLatency(config.maxLatency) + .readMaxMessages(config.readMaxMessages) + .readConcurrency(config.readConcurrency) + + } + + } + + trait SubscriptionStep[A] { + + /** Sets the subscription from which messages will be read. + * + * @param subscription + * the subscription + * @return + * the next step in the builder + */ + def subscription(subscription: Subscription): A + + } + + trait BatchSizeStep[A] { + + /** Sets the maximum size of message batches when acknowledging. + * + * @param batchSize + * the maximum size of message batches + * @return + * the next step in the builder + */ + def batchSize(batchSize: Int): A + + } + + object BatchSizeStep { + + implicit class BatchSizeStepOps[F[_]](step: PubSubSubscriber.Builder.WithDefaults[F]) { + + /** Configures the builder with the default values: + * + * - `batchSize` = 100 + * - `maxLatency` = 1 second + * - `readMaxMessages` = 1000 + * - `readConcurrency` = 1 + */ + def withDefaults: SubscriberStep[F] = + step + .batchSize(100) + .maxLatency(1.second) + .readMaxMessages(1000) + .readConcurrency(1) + + } + + } + + trait MaxLatencyStep[A] { + + /** Sets the maximum duration allowed for latency in message acknowledging. + * + * @param maxLatency + * the maximum duration allowed for latency in message acknowledging + * @return + * the next step in the builder + */ + def maxLatency(maxLatency: FiniteDuration): A + + } + + trait ReadMaxMessagesStep[A] { + + /** Sets the maximum number of messages to read at once. + * + * @param maxMessages + * the maximum number of messages + * @return + * the next step in the builder + */ + def readMaxMessages(maxMessages: Int): A + + } + + trait ReadConcurrencyStep[A] { + + /** Sets the concurrency level for reading messages. + * + * @param readConcurrency + * the concurrency level + * @return + * the next step in the builder + */ + def readConcurrency(readConcurrency: Int): A + + } + + sealed abstract class SubscriberStep[F[_]] private ( + stream: Stream[F, PubSubRecord.Subscriber[F, Array[Byte]]], + errorHandler: PartialFunction[(PubSubSubscriber.Operation[F], Throwable), F[Unit]] + ) { + + /** Returns a stream of raw Pub/Sub records. */ + def raw(implicit F: Applicative[F]): Stream[F, PubSubRecord.Subscriber[F, Array[Byte]]] = + decodeTo[Array[Byte]].subscribe + + /** Returns a stream of Pub/Sub records decoded to a specific type. */ + def decodeTo[A: MessageDecoder](implicit F: Applicative[F]) = SubscribeStep { + stream + .evalMapChunk[F, Option[PubSubRecord.Subscriber[F, A]]] { record => + record + .as[A] + .fold( + { + case e if errorHandler.isDefinedAt((Operation.Decode(record), e)) => + errorHandler((Operation.Decode(record), e)).as(None) + case _ => Option.empty[PubSubRecord.Subscriber[F, A]].pure[F] + }, + _.some.pure[F] + ) + } + .unNone + } + + } + + object SubscriberStep { + + private[pubsub] def apply[F[_]]( + stream: Stream[F, PubSubRecord.Subscriber[F, Array[Byte]]], + errorHandler: PartialFunction[(PubSubSubscriber.Operation[F], Throwable), F[Unit]] + ): SubscriberStep[F] = + new SubscriberStep[F](stream, errorHandler) {} + + } + + trait ErrorHandlerStep[F[_], A] { + + /** Sets the error handler for the subscriber. The error handler is called whenever an error occurs while processing + * a message. Possible scenarios are: decoding a message, acknowledging a message, or negatively acknowledging a + * message. + * + * @param handler + * the error handler + * @return + * the next step in the builder + */ + def errorHandler(handler: PartialFunction[(PubSubSubscriber.Operation[F], Throwable), F[Unit]]): A + + /** Disables error handling for the subscriber. + * + * @return + * the next step in the builder + */ + def noErrorHandling(implicit F: Applicative[F]): A = errorHandler { case (_, _) => Applicative[F].unit } + + } + + sealed abstract class SubscribeStep[F[_], A](stream: Stream[F, PubSubRecord.Subscriber[F, A]]) { + + /** Subscribes to the Pub/Sub topic and returns a stream of subscriber records. It is up to the user to acknowledge + * records. + */ + def subscribe: Stream[F, PubSubRecord.Subscriber[F, A]] = stream + + /** Subscribes to the Pub/Sub topic and returns a stream of decoded messages. Records will be acknowledge after + * being decoded + */ + def subscribeAndAck: Stream[F, Option[A]] = subscribe.evalTap(_.ack).map(_.value) + + } + + object SubscribeStep { + + private[pubsub] def apply[F[_], A](stream: Stream[F, PubSubRecord.Subscriber[F, A]]): SubscribeStep[F, A] = + new SubscribeStep[F, A](stream) {} + + } + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/exceptions/PubSubRequestError.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/exceptions/PubSubRequestError.scala new file mode 100644 index 00000000..39cbdd89 --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/exceptions/PubSubRequestError.scala @@ -0,0 +1,75 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub.exceptions + +import cats.effect.Concurrent +import cats.syntax.all._ + +import org.http4s.Method +import org.http4s.Request +import org.http4s.Response +import org.http4s.Uri + +/** Represents an error that occurred while making a request to Pub/Sub. + * + * @param message + * the error message + * @param method + * the HTTP method used + * @param uri + * the URI of the request + * @param cause + * the cause of the error (optional) + */ +sealed abstract class PubSubRequestError private ( + val message: String, + val method: Method, + val uri: Uri, + val cause: Option[Throwable] +) extends RuntimeException( + s"""Request to PubSub failed. + | + |URI: $uri + |Method: $method + | + |Failure: $message.""".stripMargin, + cause.orNull + ) + +object PubSubRequestError { + + /** Create a `PubSubRequestError` from a response and request. + * + * @param response + * the response + * @param request + * the request + * @return + * the error + */ + def from[F[_]: Concurrent](response: Response[F], request: Request[F]) = + response.bodyText.compile.string.redeem(_ => apply("Unknown error", request), PubSubRequestError(_, request)) + + /** Create a `PubSubRequestError` from a message and request. */ + def apply[F[_]](message: String, request: Request[F]): PubSubRequestError = + new PubSubRequestError(message.trim(), request.method, request.uri, None) {} + + /** Create a `PubSubRequestError` from a message, request, and cause. */ + def apply[F[_]](message: String, request: Request[F], cause: Throwable): PubSubRequestError = + new PubSubRequestError(message.trim(), request.method, request.uri, cause.some) {} + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/package.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/package.scala new file mode 100644 index 00000000..d422e4e1 --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/package.scala @@ -0,0 +1,25 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2 + +package object pubsub { + + type ProjectId = com.permutive.common.types.gcp.ProjectId + + val ProjectId = com.permutive.common.types.gcp.ProjectId + +} diff --git a/modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubClientSuite.scala b/modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubClientSuite.scala new file mode 100644 index 00000000..bdd69d4c --- /dev/null +++ b/modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubClientSuite.scala @@ -0,0 +1,45 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import cats.effect.IO + +import munit.FunSuite +import org.http4s.HttpApp +import org.http4s.client.Client +import org.http4s.syntax.all._ + +class PubSubClientSuite extends FunSuite { + + test("PubSubClient can be created from configuration class") { + val config = PubSubClient.Config( + projectId = ProjectId.random(), + uri = uri"localhost:8080" + ) + + val client: Client[IO] = Client.fromHttpApp(HttpApp.notFound[IO]) + + val publisher = PubSubClient + .http[IO] + .fromConfig(config) + .httpClient(client) + .noRetry + + assert(publisher.isInstanceOf[PubSubClient[IO]]) + } + +} diff --git a/modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubPublisherSuite.scala b/modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubPublisherSuite.scala new file mode 100644 index 00000000..ae48b920 --- /dev/null +++ b/modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubPublisherSuite.scala @@ -0,0 +1,69 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import scala.concurrent.duration._ + +import cats.effect.IO +import cats.effect.Resource + +import munit.FunSuite +import org.http4s.HttpApp +import org.http4s.client.Client +import org.http4s.syntax.all._ + +class PubSubPublisherSuite extends FunSuite { + + test("PubSubPublisher can be created from configuration class") { + val config = PubSubPublisher.Config( + projectId = ProjectId.random(), + topic = Topic("my-topic"), + uri = uri"localhost:8080" + ) + + val client: Client[IO] = Client.fromHttpApp(HttpApp.notFound[IO]) + + val publisher = PubSubPublisher + .http[IO, String] + .fromConfig(config) + .httpClient(client) + .noRetry + + assert(publisher.isInstanceOf[PubSubPublisher[IO, String]]) + } + + test("PubSubPublisher.Async can be created from configuration class") { + val config = PubSubPublisher.Async.Config( + projectId = ProjectId.random(), + topic = Topic("my-topic"), + uri = uri"localhost:8080", + batchSize = 100, + maxLatency = 1.second + ) + + val client: Client[IO] = Client.fromHttpApp(HttpApp.notFound[IO]) + + val publisher = PubSubPublisher + .http[IO, String] + .fromConfig(config) + .httpClient(client) + .noRetry + + assert(publisher.isInstanceOf[Resource[IO, PubSubPublisher.Async[IO, String]]]) + } + +} diff --git a/modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubSubscriberSuite.scala b/modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubSubscriberSuite.scala new file mode 100644 index 00000000..af4c23dd --- /dev/null +++ b/modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubSubscriberSuite.scala @@ -0,0 +1,55 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import scala.concurrent.duration._ + +import cats.effect.IO + +import fs2.Stream +import munit.FunSuite +import org.http4s.HttpApp +import org.http4s.client.Client +import org.http4s.syntax.all._ + +class PubSubSubscriberSuite extends FunSuite { + + test("PubSubSubscriber can be created from configuration class") { + val config = PubSubSubscriber.Config( + projectId = ProjectId.random(), + subscription = Subscription("my-subscription"), + uri = uri"localhost:8080", + batchSize = 10, + maxLatency = 1.second, + readMaxMessages = 100, + readConcurrency = 3 + ) + + val client: Client[IO] = Client.fromHttpApp(HttpApp.notFound[IO]) + + val subscriber = PubSubSubscriber + .http[IO] + .fromConfig(config) + .httpClient(client) + .noRetry + .noErrorHandling + .raw + + assert(subscriber.isInstanceOf[Stream[IO, PubSubRecord.Subscriber[IO, Array[Byte]]]]) + } + +} diff --git a/modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubSuite.scala b/modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubSuite.scala new file mode 100644 index 00000000..2b0a04a1 --- /dev/null +++ b/modules/fs2-pubsub/src/test/scala/fs2/pubsub/PubSubSuite.scala @@ -0,0 +1,177 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import scala.concurrent.duration._ + +import cats.effect.IO +import cats.effect.std.Queue +import cats.syntax.all._ + +import com.dimafeng.testcontainers.GenericContainer +import com.dimafeng.testcontainers.munit.fixtures.TestContainersFixtures +import com.permutive.common.types.gcp.http4s._ +import fs2.Chunk +import io.circe.Json +import io.circe.syntax._ +import munit.CatsEffectSuite +import org.http4s.Method._ +import org.http4s.Uri +import org.http4s.circe._ +import org.http4s.client.dsl.io._ +import org.http4s.ember.client.EmberClientBuilder +import org.testcontainers.containers.wait.strategy.Wait + +class PubSubSuite extends CatsEffectSuite with TestContainersFixtures { + + afterProducing(records = 1) + .test("it should send and receive a message, acknowledging as expected") { subscriber => + val result = subscriber + .evalTap(_.ack) + .map(_.value) + .interruptAfter(2.seconds) + .compile + .toList + + val expected = List("ping".some) + + assertIO(result, expected) + } + + afterProducing(records = 5) + .test("it should preserve chunksize in the underlying stream") { subscriber => + val result = subscriber.chunks + .evalTap(_.traverse(_.ack)) + .interruptAfter(2.seconds) + .map(_.map(_.value)) + .compile + .toList + + assertIO(result, List(Chunk("ping".some, "ping".some, "ping".some, "ping".some, "ping".some))) + } + + afterProducing(records = 1, withAckDeadlineSeconds = 2) + .test("it should extend the deadline for a message") { subscriber => + val deadline = AckDeadline.from(10.seconds).toOption.get + + val result = subscriber + .evalTap(_.extendDeadline(deadline)) + .evalTap(_ => IO.sleep(3.seconds)) + .evalTap(_.ack) + .interruptAfter(5.seconds) + .compile + .count + + assertIO(result, 1L) + } + + afterProducing(records = 1) + .test("it should nack a message properly") { subscriber => + val result = subscriber + .evalScan(false) { case (nackedAlready, record) => + if (nackedAlready) record.ack.as(true) else record.nack.as(true) + } + .void + .interruptAfter(2.seconds) + .compile + .count + + assertIO(result, 3L) + } + + ////////////// + // Fixtures // + ////////////// + + val projects = List.fill(4)("example-topic:example-subscription").zipWithIndex.map { case (topics, index) => + s"test-project-${index + 1},$topics" + } + + val projectsFixture = ResourceSuiteLocalFixture( + "Projects", + Queue + .unbounded[IO, ProjectId] + .flatTap(queue => projects.map(_.split(",").head).traverse(ProjectId.fromStringF[IO](_).flatMap(queue.offer))) + .toResource + ) + + def afterProducing(records: Int, withAckDeadlineSeconds: Int = 10) = ResourceFunFixture { + IO(projectsFixture().take).flatten.toResource + .product(EmberClientBuilder.default[IO].build) + .evalTap { case (projectId, client) => + val body = Json.obj( + "subscription" := Json.obj( + "topic" := "example-topic", + "ackDeadlineSeconds" := withAckDeadlineSeconds + ), + "updateMask" := "ackDeadlineSeconds" + ) + + val request = + PATCH(body, container.uri / "v1" / "projects" / projectId / "subscriptions" / "example-subscription") + + client.expect[Unit](request) + } + .map { case (projectId, client) => + val pubSubClient = PubSubClient + .http[IO] + .projectId(projectId) + .uri(container.uri) + .httpClient(client) + .noRetry + + val publisher = pubSubClient + .publisher[String] + .topic(Topic("example-topic")) + + val subscriber = pubSubClient.subscriber + .subscription(Subscription("example-subscription")) + .errorHandler { + case (PubSubSubscriber.Operation.Ack(_), t) => IO.println(t) + case (PubSubSubscriber.Operation.Nack(_), t) => IO.println(t) + case (PubSubSubscriber.Operation.Decode(record), t) => IO.println(t) >> record.ack + } + .withDefaults + .decodeTo[String] + .subscribe + + (publisher, subscriber) + } + .evalTap { + case (publisher, _) if records === 1 => publisher.publishOne("ping") + case (publisher, _) => publisher.publishMany(List.fill(records)(PubSubRecord.Publisher("ping"))) + } + ._2F + } + + case object container + extends GenericContainer( + "thekevjames/gcloud-pubsub-emulator:450.0.0", + exposedPorts = Seq(8681, 8682), + waitStrategy = Wait.forListeningPort().some, + env = projects.zipWithIndex.map { case (project, index) => s"PUBSUB_PROJECT${index + 1}" -> project }.toMap + ) { + + def uri = Uri.unsafeFromString(s"http://localhost:${mappedPort(8681)}") + + } + + val containerFixture = new ForAllContainerFixture[GenericContainer](container) + + override def munitFixtures = super.munitFixtures :+ projectsFixture :+ containerFixture + +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala new file mode 100644 index 00000000..ebb2bdf8 --- /dev/null +++ b/project/Dependencies.scala @@ -0,0 +1,21 @@ +import sbt._ +import sbt.Keys._ + +object Dependencies { + + lazy val `fs2-pubsub` = Seq( + "co.fs2" %% "fs2-core" % "3.9.4", + "com.permutive" %% "common-types-gcp-http4s" % "0.0.2", + "io.circe" %% "circe-parser" % "0.14.6", + "org.http4s" %% "http4s-circe" % "0.23.16", + "org.http4s" %% "http4s-client" % "0.23.16", + "org.http4s" %% "http4s-dsl" % "0.23.16" + ) ++ Seq( + "com.dimafeng" %% "testcontainers-scala-munit" % "0.41.0", + "com.permutive" %% "gcp-auth" % "0.1.0", + "org.http4s" %% "http4s-ember-client" % "0.23.25", + "org.slf4j" % "slf4j-nop" % "2.0.10", + "org.typelevel" %% "munit-cats-effect" % "2.0.0-M4" + ).map(_ % Test) + +} From ccc6f493626f8966d8d7e942c9598ad5e9d6b5b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Herna=CC=81ndez?= Date: Mon, 1 Apr 2024 18:04:17 +0200 Subject: [PATCH 4/9] Add module with `pureconfig` integration --- build.sbt | 4 ++ .../scala/fs2/pubsub/pureconfig/package.scala | 43 +++++++++++++++++++ project/Dependencies.scala | 5 +++ 3 files changed, 52 insertions(+) create mode 100644 modules/fs2-pubsub-pureconfig/src/main/scala/fs2/pubsub/pureconfig/package.scala diff --git a/build.sbt b/build.sbt index 38f3b71e..07fbd725 100644 --- a/build.sbt +++ b/build.sbt @@ -11,3 +11,7 @@ lazy val `fs2-pubsub` = module .settings(libraryDependencies ++= Dependencies.`fs2-pubsub`) .settings(Test / fork := true) .settings(Test / run / fork := true) + +lazy val `fs2-pubsub-pureconfig` = module + .dependsOn(`fs2-pubsub`) + .settings(libraryDependencies ++= Dependencies.`fs2-pubsub-pureconfig`) diff --git a/modules/fs2-pubsub-pureconfig/src/main/scala/fs2/pubsub/pureconfig/package.scala b/modules/fs2-pubsub-pureconfig/src/main/scala/fs2/pubsub/pureconfig/package.scala new file mode 100644 index 00000000..f6e9c602 --- /dev/null +++ b/modules/fs2-pubsub-pureconfig/src/main/scala/fs2/pubsub/pureconfig/package.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import _root_.pureconfig.ConfigReader +import _root_.pureconfig.ConfigReader._ +import _root_.pureconfig.module.http4s._ +import com.permutive.common.types.gcp.pureconfig._ + +package object pureconfig { + + implicit val TopicConfigReader: ConfigReader[Topic] = ConfigReader[String].map(Topic(_)) + + implicit val SubscriptionConfigReader: ConfigReader[Subscription] = ConfigReader[String].map(Subscription(_)) + + implicit val PubSubClientConfigConfigReader: ConfigReader[PubSubClient.Config] = + forProduct2("project-id", "uri")(PubSubClient.Config.apply) + + implicit val PubSubPublisherConfigConfigReader: ConfigReader[PubSubPublisher.Config] = + forProduct3("project-id", "topic", "uri")(PubSubPublisher.Config.apply) + + implicit val PubSubPublisherAsyncConfigConfigReader: ConfigReader[PubSubPublisher.Async.Config] = + forProduct5("project-id", "topic", "uri", "batch-size", "max-latency")(PubSubPublisher.Async.Config.apply) + + implicit val PubSubSubscriberConfigConfigReader: ConfigReader[PubSubSubscriber.Config] = + forProduct7("project-id", "subscription", "uri", "batch-size", "max-latency", "read-max-messages", + "read-concurrency")(PubSubSubscriber.Config.apply) + +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index ebb2bdf8..507c691d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -18,4 +18,9 @@ object Dependencies { "org.typelevel" %% "munit-cats-effect" % "2.0.0-M4" ).map(_ % Test) + lazy val `fs2-pubsub-pureconfig` = Seq( + "com.github.pureconfig" %% "pureconfig-http4s" % "0.17.5", + "com.permutive" %% "common-types-gcp-pureconfig" % "0.0.2" + ) + } From 37ff1033307561fab7b19208b6ab3ac213fc6e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Herna=CC=81ndez?= Date: Mon, 1 Apr 2024 18:06:11 +0200 Subject: [PATCH 5/9] Add gRPC constructors for every algebra --- build.sbt | 2 + .../fs2/pubsub/GrpcConstructors.scala | 27 +++ .../fs2/pubsub/GrpcConstructors.scala | 178 ++++++++++++++++++ .../main/scala/fs2/pubsub/PubSubClient.scala | 2 +- .../scala/fs2/pubsub/PubSubPublisher.scala | 2 +- .../scala/fs2/pubsub/PubSubSubscriber.scala | 2 +- project/Dependencies.scala | 2 + 7 files changed, 212 insertions(+), 3 deletions(-) create mode 100644 modules/fs2-pubsub/src/main/scala-2.12/fs2/pubsub/GrpcConstructors.scala create mode 100644 modules/fs2-pubsub/src/main/scala-2.13+/fs2/pubsub/GrpcConstructors.scala diff --git a/build.sbt b/build.sbt index 07fbd725..7ddadaac 100644 --- a/build.sbt +++ b/build.sbt @@ -9,6 +9,8 @@ addCommandAlias("ci-publish", "versionCheck; github; ci-release") lazy val `fs2-pubsub` = module .settings(libraryDependencies ++= Dependencies.`fs2-pubsub`) + .settings(libraryDependencies += scalaVersion.value.on(2, 13)(Dependencies.grpc)) + .settings(libraryDependencies += scalaVersion.value.on(3)(Dependencies.grpc)) .settings(Test / fork := true) .settings(Test / run / fork := true) diff --git a/modules/fs2-pubsub/src/main/scala-2.12/fs2/pubsub/GrpcConstructors.scala b/modules/fs2-pubsub/src/main/scala-2.12/fs2/pubsub/GrpcConstructors.scala new file mode 100644 index 00000000..668ce3f4 --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala-2.12/fs2/pubsub/GrpcConstructors.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +object GrpcConstructors { + + trait Client {} + + trait Subscriber {} + + trait Publisher {} + +} diff --git a/modules/fs2-pubsub/src/main/scala-2.13+/fs2/pubsub/GrpcConstructors.scala b/modules/fs2-pubsub/src/main/scala-2.13+/fs2/pubsub/GrpcConstructors.scala new file mode 100644 index 00000000..6bc000dd --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala-2.13+/fs2/pubsub/GrpcConstructors.scala @@ -0,0 +1,178 @@ +/* + * Copyright 2019-2024 Permutive Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.pubsub + +import java.util.Base64 + +import cats.effect.Temporal +import cats.syntax.all._ + +import com.google.protobuf.ByteString +import com.google.pubsub.v1.pubsub.AcknowledgeRequest +import com.google.pubsub.v1.pubsub.ModifyAckDeadlineRequest +import com.google.pubsub.v1.pubsub.PublishRequest +import com.google.pubsub.v1.pubsub.Publisher +import com.google.pubsub.v1.pubsub.PubsubMessage +import com.google.pubsub.v1.pubsub.PullRequest +import com.google.pubsub.v1.pubsub.ReceivedMessage +import com.google.pubsub.v1.pubsub.Subscriber +import fs2.Chunk +import fs2.pubsub.dsl.client.PubSubClientStep +import fs2.pubsub.dsl.publisher.PubSubPublisherStep +import fs2.pubsub.dsl.subscriber.PubSubSubscriberStep +import org.http4s.Headers +import org.http4s.client.Client +import org.http4s.client.middleware.Retry +import org.http4s.headers.`Content-Type` +import org.http4s.syntax.all._ + +object GrpcConstructors { + + trait Publisher { + + /** Starts creating a gRPC Pub/Sub publisher in a step-by-step fashion. + * + * @tparam F + * the effect type + * @tparam A + * the type of messages to be sent to Pub/Sub + */ + def grpc[F[_]: Temporal, A: MessageEncoder]: PubSubPublisherStep[F, A] = { + projectId => topic => uri => client => retryPolicy => + PubSubClient.grpc + .projectId(projectId) + .uri(uri) + .httpClient(client) + .retryPolicy(retryPolicy) + .publisher + .topic(topic) + } + + } + + trait Subscriber { + + /** Starts creating a gRPC Pub/Sub subscriber in a step-by-step fashion. + * + * @tparam F + * the effect type + */ + def grpc[F[_]: Temporal]: PubSubSubscriberStep[F] = { projectId => subscription => uri => client => retryPolicy => + PubSubClient.grpc + .projectId(projectId) + .uri(uri) + .httpClient(client) + .retryPolicy(retryPolicy) + .subscriber + .subscription(subscription) + } + + } + + trait Client { + + /** Starts creating a gRPC Pub/Sub client in a step-by-step fashion. + * + * @tparam F + * the effect type + */ + def grpc[F[_]: Temporal]: PubSubClientStep[F] = { projectId => uri => underlying => retryPolicy => + new PubSubClient[F] { + + private val httpClient = Retry.create(retryPolicy, logRetries = false) { + Client[F](request => underlying.run(request.putHeaders(`Content-Type`(mediaType"application/grpc")))) + } + + val subscriber = Subscriber.fromClient(httpClient, uri) + val publisher = Publisher.fromClient(httpClient, uri) + + override def publish[A: MessageEncoder]( + topic: Topic, + records: Seq[PubSubRecord.Publisher[A]] + ): F[List[MessageId]] = { + val toPubSubMessage = (record: PubSubRecord.Publisher[A]) => + PubsubMessage( + data = ByteString.copyFromUtf8(Base64.getEncoder().encodeToString(MessageEncoder[A].encode(record.data))), + attributes = record.attributes + ) + + val request = PublishRequest.of( + topic = show"projects/$projectId/topics/$topic", + messages = records.map(toPubSubMessage) + ) + + publisher + .publish(request, Headers.empty) + .map(_.messageIds.map(MessageId(_)).toList) + } + + override def read( + subscription: Subscription, + maxMessages: Int + ): F[List[PubSubRecord.Subscriber[F, Array[Byte]]]] = { + val request = PullRequest.of( + subscription = show"projects/$projectId/subscriptions/$subscription", + returnImmediately = false, + maxMessages = maxMessages + ) + + val toPubSubRecord = (message: ReceivedMessage) => + PubSubRecord.Subscriber( + message.message.map(m => m.data.toByteArray()).map(Base64.getDecoder().decode), + message.message.map(_.attributes).orEmpty, + message.message.map(_.messageId).map(MessageId(_)), + message.message.flatMap(_.publishTime.map(_.asJavaInstant)), + AckId(message.ackId), + ack(subscription, AckId(message.ackId)), + nack(subscription, AckId(message.ackId)), + modifyDeadline(subscription, AckId(message.ackId), _) + ) + + subscriber + .pull(request, Headers.empty) + .map(_.receivedMessages.map(toPubSubRecord).toList) + } + + override def ack(subscription: Subscription, ackIds: Chunk[AckId]): F[Unit] = { + val request = AcknowledgeRequest.of( + subscription = show"projects/$projectId/subscriptions/$subscription", + ackIds = ackIds.map(_.value).toList + ) + + subscriber + .acknowledge(request, Headers.empty) + .void + } + + override def modifyDeadline(subscription: Subscription, ackIds: Chunk[AckId], by: AckDeadline): F[Unit] = { + val request = ModifyAckDeadlineRequest.of( + subscription = show"projects/$projectId/subscriptions/$subscription", + ackIds = ackIds.map(_.value).toList, + ackDeadlineSeconds = by.value.toSeconds.toInt + ) + + subscriber + .modifyAckDeadline(request, Headers.empty) + .void + } + + } + } + + } + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubClient.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubClient.scala index 6e580230..c1b3dc50 100644 --- a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubClient.scala +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubClient.scala @@ -144,7 +144,7 @@ trait PubSubClient[F[_]] { } -object PubSubClient { +object PubSubClient extends GrpcConstructors.Client { /** Represents the configuration used by a `PubSubClient`. * diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubPublisher.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubPublisher.scala index 3f576863..dd08657b 100644 --- a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubPublisher.scala +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubPublisher.scala @@ -89,7 +89,7 @@ trait PubSubPublisher[F[_], A] { } -object PubSubPublisher { +object PubSubPublisher extends GrpcConstructors.Publisher { /** Represents the configuration parameters for a `PubSubPublisher`. * diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubSubscriber.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubSubscriber.scala index 7206743e..c0359f90 100644 --- a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubSubscriber.scala +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubSubscriber.scala @@ -32,7 +32,7 @@ import fs2.pubsub.dsl.subscriber._ import org.http4s.Uri /** Contains method for creating Pub/Sub subscribers. */ -object PubSubSubscriber { +object PubSubSubscriber extends GrpcConstructors.Subscriber { /** Represents the configuration used by a `PubSubClient`. * diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 507c691d..aeb865a8 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -3,6 +3,8 @@ import sbt.Keys._ object Dependencies { + lazy val grpc = "io.chrisdavenport" %% "http4s-grpc-google-cloud-pubsub-v1" % "1.108.0+0.0.6" + lazy val `fs2-pubsub` = Seq( "co.fs2" %% "fs2-core" % "3.9.4", "com.permutive" %% "common-types-gcp-http4s" % "0.0.2", From cfff94dfede9699391d04fdcbce6bd4f5c4d41d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Herna=CC=81ndez?= Date: Mon, 1 Apr 2024 18:14:04 +0200 Subject: [PATCH 6/9] Add documentation --- .github/docs/README.md | 250 +++++++++++++++++++++++++++++++++++++ build.sbt | 6 + project/Dependencies.scala | 10 ++ 3 files changed, 266 insertions(+) create mode 100644 .github/docs/README.md diff --git a/.github/docs/README.md b/.github/docs/README.md new file mode 100644 index 00000000..4d361e0a --- /dev/null +++ b/.github/docs/README.md @@ -0,0 +1,250 @@ +@DESCRIPTION@ + +```scala mdoc:toc +``` + +## Installation + +Add the following line to your `build.sbt` file: + +```sbt +libraryDependencies += "@ORGANIZATION@" %% "fs2-pubsub" % "@VERSION@" +``` + +The library is published for Scala versions: @SUPPORTED_SCALA_VERSIONS@. + +## Usage + +To start using the library, you'll need an http4s `Client` with permission to +call Pub/Sub APIs in GCP. You can create one using [`gcp-auth`]: + +```scala mdoc:silent +import org.http4s.ember.client.EmberClientBuilder +import cats.effect.IO +import cats.syntax.all._ +import com.permutive.gcp.auth.TokenProvider + +val client = EmberClientBuilder + .default[IO] + .withHttp2 + .build + .mproduct(client => TokenProvider.userAccount(client).toResource) + .map { case (client, tokenProvider) => tokenProvider.clientMiddleware(client) } +``` + +```scala mdoc:reset:invisible +import org.http4s.client.Client +import org.http4s.HttpApp +import cats.effect.IO + +val client: Client[IO] = Client.fromHttpApp[IO](HttpApp.notFound[IO]) +``` + +### Publishing messages to a Pub/Sub topic + +To publish messages to Pub/Sub, you can use the `PubsubPublisher` class: + +```scala mdoc:silent +import fs2.pubsub._ + +val publisher: PubSubPublisher[IO, String] = PubSubPublisher + .http[IO, String] + .projectId(ProjectId("my-project")) + .topic(Topic("my-topic")) + .defaultUri + .httpClient(client) + .noRetry +``` + +Then you can use any of the `PubSubPublisher` methods to send messages to Pub/Sub. + +```scala mdoc:silent +// Producing a single message + +publisher.publishOne("message") +``` + +```scala mdoc:silent +// Producing multiple messages + +val records = List( + PubSubRecord.Publisher("message1"), + PubSubRecord.Publisher("message2"), + PubSubRecord.Publisher("message3") +) + +publisher.publishMany(records) +``` + +```scala mdoc:silent +// Producing a message with attributes + +publisher.publishOne("message", "key" -> "value") +``` + +```scala mdoc:silent +// Producing a message using the record type + +val record = PubSubRecord.Publisher("message").withAttribute("key", "value") + +publisher.publishOne(record) +``` + +#### Configuring the publisher + +There are several configuration options available for the publisher: + +- `projectId`: The GCP project ID. +- `topic`: The Pub/Sub topic name. +- `uri`: The URI of the Pub/Sub API. By default, it uses the Google Cloud +Pub/Sub API. +- `httpClient`: The http4s `Client` to use for making requests to the +Pub/Sub API. +- `retry`: The retry policy to use when sending messages to Pub/Sub. By +default, it retries up to 3 times with exponential backoff. + +These configurations can either by provided by using a configuration object +(`PubSubPublisher.Config`) or by using the builder pattern. + +#### Using gRPC (only available on 2.13 or 3.x) + +You can use `PubSubPublisher.grpc` to create a publisher that uses gRPC to connect +to Pub/Sub. + +This type of publisher is only available on Scala `2.13` or `3.x`. + +#### Publishing messages asynchronously (in batches) + +In order to publish messages asynchronously, you can use the `PubSubPublisher.Async`. +You can create an instance of this class from a regular `PubSubPublisher` by using the +`batching` method: + +```scala mdoc:silent +import cats.effect.Resource +import scala.concurrent.duration._ + +val asyncPublisher: Resource[IO, PubSubPublisher.Async[IO, String]] = + publisher + .batching + .batchSize(10) + .maxLatency(1.second) +``` + +Then you can use any of the `PubSubPublisher.Async` methods to send messages to Pub/Sub. +These methods are the same ones you'll find in the regular `PubSubPublisher`, with +the difference that they return a `F[Unit]` instead of a `F[MessageId]` and that +they expect a `PubSubRecord.Publisher.WithCallback` instead of a regular +`PubSubRecord.Publisher`. + +In order to construct such class you can either use the `PubSubRecord.Publisher.WithCallback` +constructor or use the `withCallback` method on a regular `PubSubRecord.Publisher`: + +```scala mdoc:silent +val recordWithCallback = PubSubRecord.Publisher("message").withCallback { _ => + IO(println("Message sent!")) +} +``` + +### Subscribing to a Pub/Sub subscription + +To subscribe to a Pub/Sub subscription, you can use the `PubSubSubscriber` class: + +```scala mdoc:silent +import fs2.Stream + +val subscriber: Stream[IO, Option[String]] = PubSubSubscriber + .http[IO] + .projectId(ProjectId("my-project")) + .subscription(Subscription("my-subscription")) + .defaultUri + .httpClient(client) + .noRetry + .noErrorHandling + .withDefaults + .decodeTo[String] + .subscribeAndAck +``` + +#### Configuring the subscriber + +There are several configuration options available for the subscriber: + +- `projectId`: The GCP project ID. +- `subscription`: The Pub/Sub subscription name. +- `uri`: The URI of the Pub/Sub API. By default, it uses the Google Cloud +Pub/Sub API. +- `httpClient`: The http4s `Client` to use for making requests to the +Pub/Sub API. +- `retry`: The retry policy to use when receiving messages from Pub/Sub. By +default, it retries up to 3 times with exponential backoff. +- `errorHandling`: The error handling policy to use when performing operations +such as decoding messages or acknowledging them. +- `batchSize`: The maximum number of messages to acknowledge at once. +- `maxLatency`: The maximum time to wait for a batch of messages before +acknowledging them. +- `maxMessages`: The maximum number of messages to receive in a single batch. +- `readConcurrency`: The number of concurrent reads from the subscription. + +These configurations can either by provided by using a configuration object +(`PubSubSubscriber.Config`) or by using the builder pattern. + +#### Using gRPC (only available on 2.13 or 3.x) + +You can use `PubSubSubscriber.grpc` to create a subscriber that uses gRPC to connect +to Pub/Sub. + +This type of subscriber is only available on Scala `2.13` or `3.x`. + +#### Creating a raw subscriber + +There are two types of subscribers available in the library: raw and decoded. + +The raw subscriber returns the raw message received from Pub/Sub, while the +decoded subscriber decodes the message to a specific type. + +The former is useful when you want to handle the message yourself, while the +latter is useful when you want to work with a specific type. You can create +a raw subscriber by using the `raw` method instead of `decodeTo`. + +### Pureconfig integration + +The library provides a way to load the configuration from a `ConfigSource` using +[`pureconfig`]. + +You just need to add the following line to your `build.sbt` file: + +```sbt +libraryDependencies += "@ORGANIZATION@" %% "fs2-pubsub-pureconfig" % "@VERSION@" +``` + +And then add the following import when you want to use the `pureconfig` integration: + +```scala mdoc:reset:invisible +import org.http4s.client.Client +import org.http4s.HttpApp +import cats.effect.IO + +val client: Client[IO] = Client.fromHttpApp[IO](HttpApp.notFound[IO]) +``` + +```scala mdoc:compile-only +import pureconfig.ConfigSource + +import fs2.pubsub.PubSubPublisher +import fs2.pubsub.pureconfig._ + +val config = ConfigSource.default.loadOrThrow[PubSubPublisher.Config] + +PubSubPublisher + .http[IO, String] + .fromConfig(config) + .httpClient(client) + .noRetry +``` + +## Contributors to this project + +@CONTRIBUTORS_TABLE@ + +[`gcp-auth`]: https://github.com/permutive-engineering/gcp-auth/ +[`pureconfig`]: https://pureconfig.github.io/ \ No newline at end of file diff --git a/build.sbt b/build.sbt index 7ddadaac..ebd55579 100644 --- a/build.sbt +++ b/build.sbt @@ -7,6 +7,12 @@ addCommandAlias("ci-test", "fix --check; versionPolicyCheck; mdoc; publishLocal; addCommandAlias("ci-docs", "github; mdoc; headerCreateAll") addCommandAlias("ci-publish", "versionCheck; github; ci-release") +lazy val documentation = project + .enablePlugins(MdocPlugin) + .dependsOn(`fs2-pubsub-pureconfig`) + .settings(mdocAutoDependency := false) + .settings(libraryDependencies ++= Dependencies.documentation) + lazy val `fs2-pubsub` = module .settings(libraryDependencies ++= Dependencies.`fs2-pubsub`) .settings(libraryDependencies += scalaVersion.value.on(2, 13)(Dependencies.grpc)) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index aeb865a8..111ffc43 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,8 +1,18 @@ import sbt._ import sbt.Keys._ +import mdoc.BuildInfo object Dependencies { + lazy val documentation = Seq( + ("org.scalameta" %% "mdoc" % BuildInfo.version).excludeAll( + ExclusionRule(organization = "com.thesamet.scalapb", name = "lenses_2.13"), + ExclusionRule(organization = "com.thesamet.scalapb", name = "scalapb-runtime_2.13") + ), + "com.permutive" %% "gcp-auth" % "0.2.0", + "org.http4s" %% "http4s-ember-client" % "0.23.25" + ) + lazy val grpc = "io.chrisdavenport" %% "http4s-grpc-google-cloud-pubsub-v1" % "1.108.0+0.0.6" lazy val `fs2-pubsub` = Seq( From 1cba2cd4431ade0377bbe56a572e771b5a0df729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Herna=CC=81ndez?= Date: Mon, 1 Apr 2024 18:14:25 +0200 Subject: [PATCH 7/9] Execute `ci-docs` --- README.md | 254 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..ca171645 --- /dev/null +++ b/README.md @@ -0,0 +1,254 @@ +Google Cloud Pub/Sub stream-based client built on top of cats-effect, fs2 and http4s. + +--- + +- [Installation](#installation) +- [Usage](#usage) + - [Publishing messages to a Pub/Sub topic](#publishing-messages-to-a-pubsub-topic) + - [Configuring the publisher](#configuring-the-publisher) + - [Using gRPC (only available on 2.13 or 3.x)](#using-grpc-only-available-on-213-or-3x) + - [Publishing messages asynchronously (in batches)](#publishing-messages-asynchronously-in-batches) + - [Subscribing to a Pub/Sub subscription](#subscribing-to-a-pubsub-subscription) + - [Configuring the subscriber](#configuring-the-subscriber) + - [Using gRPC (only available on 2.13 or 3.x)](#using-grpc-only-available-on-213-or-3x) + - [Creating a raw subscriber](#creating-a-raw-subscriber) + - [Pureconfig integration](#pureconfig-integration) +- [Contributors to this project](#contributors-to-this-project) + +## Installation + +Add the following line to your `build.sbt` file: + +```sbt +libraryDependencies += "com.permutive" %% "fs2-pubsub" % "0.22.0" +``` + +The library is published for Scala versions: `2.12`, `2.13` and `3`. + +## Usage + +To start using the library, you'll need an http4s `Client` with permission to +call Pub/Sub APIs in GCP. You can create one using [`gcp-auth`]: + +```scala +import org.http4s.ember.client.EmberClientBuilder +import cats.effect.IO +import cats.syntax.all._ +import com.permutive.gcp.auth.TokenProvider + +val client = EmberClientBuilder + .default[IO] + .withHttp2 + .build + .mproduct(client => TokenProvider.userAccount(client).toResource) + .map { case (client, tokenProvider) => tokenProvider.clientMiddleware(client) } +``` + + +### Publishing messages to a Pub/Sub topic + +To publish messages to Pub/Sub, you can use the `PubsubPublisher` class: + +```scala +import fs2.pubsub._ + +val publisher: PubSubPublisher[IO, String] = PubSubPublisher + .http[IO, String] + .projectId(ProjectId("my-project")) + .topic(Topic("my-topic")) + .defaultUri + .httpClient(client) + .noRetry +``` + +Then you can use any of the `PubSubPublisher` methods to send messages to Pub/Sub. + +```scala +// Producing a single message + +publisher.publishOne("message") +``` + +```scala +// Producing multiple messages + +val records = List( + PubSubRecord.Publisher("message1"), + PubSubRecord.Publisher("message2"), + PubSubRecord.Publisher("message3") +) + +publisher.publishMany(records) +``` + +```scala +// Producing a message with attributes + +publisher.publishOne("message", "key" -> "value") +``` + +```scala +// Producing a message using the record type + +val record = PubSubRecord.Publisher("message").withAttribute("key", "value") + +publisher.publishOne(record) +``` + +#### Configuring the publisher + +There are several configuration options available for the publisher: + +- `projectId`: The GCP project ID. +- `topic`: The Pub/Sub topic name. +- `uri`: The URI of the Pub/Sub API. By default, it uses the Google Cloud +Pub/Sub API. +- `httpClient`: The http4s `Client` to use for making requests to the +Pub/Sub API. +- `retry`: The retry policy to use when sending messages to Pub/Sub. By +default, it retries up to 3 times with exponential backoff. + +These configurations can either by provided by using a configuration object +(`PubSubPublisher.Config`) or by using the builder pattern. + +#### Using gRPC (only available on 2.13 or 3.x) + +You can use `PubSubPublisher.grpc` to create a publisher that uses gRPC to connect +to Pub/Sub. + +This type of publisher is only available on Scala `2.13` or `3.x`. + +#### Publishing messages asynchronously (in batches) + +In order to publish messages asynchronously, you can use the `PubSubPublisher.Async`. +You can create an instance of this class from a regular `PubSubPublisher` by using the +`batching` method: + +```scala +import cats.effect.Resource +import scala.concurrent.duration._ + +val asyncPublisher: Resource[IO, PubSubPublisher.Async[IO, String]] = + publisher + .batching + .batchSize(10) + .maxLatency(1.second) +``` + +Then you can use any of the `PubSubPublisher.Async` methods to send messages to Pub/Sub. +These methods are the same ones you'll find in the regular `PubSubPublisher`, with +the difference that they return a `F[Unit]` instead of a `F[MessageId]` and that +they expect a `PubSubRecord.Publisher.WithCallback` instead of a regular +`PubSubRecord.Publisher`. + +In order to construct such class you can either use the `PubSubRecord.Publisher.WithCallback` +constructor or use the `withCallback` method on a regular `PubSubRecord.Publisher`: + +```scala +val recordWithCallback = PubSubRecord.Publisher("message").withCallback { _ => + IO(println("Message sent!")) +} +``` + +### Subscribing to a Pub/Sub subscription + +To subscribe to a Pub/Sub subscription, you can use the `PubSubSubscriber` class: + +```scala +import fs2.Stream + +val subscriber: Stream[IO, Option[String]] = PubSubSubscriber + .http[IO] + .projectId(ProjectId("my-project")) + .subscription(Subscription("my-subscription")) + .defaultUri + .httpClient(client) + .noRetry + .noErrorHandling + .withDefaults + .decodeTo[String] + .subscribeAndAck +``` + +#### Configuring the subscriber + +There are several configuration options available for the subscriber: + +- `projectId`: The GCP project ID. +- `subscription`: The Pub/Sub subscription name. +- `uri`: The URI of the Pub/Sub API. By default, it uses the Google Cloud +Pub/Sub API. +- `httpClient`: The http4s `Client` to use for making requests to the +Pub/Sub API. +- `retry`: The retry policy to use when receiving messages from Pub/Sub. By +default, it retries up to 3 times with exponential backoff. +- `errorHandling`: The error handling policy to use when performing operations +such as decoding messages or acknowledging them. +- `batchSize`: The maximum number of messages to acknowledge at once. +- `maxLatency`: The maximum time to wait for a batch of messages before +acknowledging them. +- `maxMessages`: The maximum number of messages to receive in a single batch. +- `readConcurrency`: The number of concurrent reads from the subscription. + +These configurations can either by provided by using a configuration object +(`PubSubSubscriber.Config`) or by using the builder pattern. + +#### Using gRPC (only available on 2.13 or 3.x) + +You can use `PubSubSubscriber.grpc` to create a subscriber that uses gRPC to connect +to Pub/Sub. + +This type of subscriber is only available on Scala `2.13` or `3.x`. + +#### Creating a raw subscriber + +There are two types of subscribers available in the library: raw and decoded. + +The raw subscriber returns the raw message received from Pub/Sub, while the +decoded subscriber decodes the message to a specific type. + +The former is useful when you want to handle the message yourself, while the +latter is useful when you want to work with a specific type. You can create +a raw subscriber by using the `raw` method instead of `decodeTo`. + +### Pureconfig integration + +The library provides a way to load the configuration from a `ConfigSource` using +[`pureconfig`]. + +You just need to add the following line to your `build.sbt` file: + +```sbt +libraryDependencies += "com.permutive" %% "fs2-pubsub-pureconfig" % "0.22.0" +``` + +And then add the following import when you want to use the `pureconfig` integration: + + +```scala +import pureconfig.ConfigSource + +import fs2.pubsub.PubSubPublisher +import fs2.pubsub.pureconfig._ + +val config = ConfigSource.default.loadOrThrow[PubSubPublisher.Config] + +PubSubPublisher + .http[IO, String] + .fromConfig(config) + .httpClient(client) + .noRetry +``` + +## Contributors to this project + +| CremboC | bastewart | TimWSpence | travisbrown | ChristianJohnston97 | chrisjl154 | janstenpickle | +| :--: | :--: | :--: | :--: | :--: | :--: | :--: | +| CremboC | bastewart | TimWSpence | travisbrown | ChristianJohnston97 | chrisjl154 | janstenpickle | + +| marcelocarlos | desbo | kythyra | mcgizzle | istreeter | Joe8Bit | arunas-cesonis | +| :--: | :--: | :--: | :--: | :--: | :--: | :--: | +| marcelocarlos | desbo | kythyra | mcgizzle | istreeter | Joe8Bit | arunas-cesonis | + +[`gcp-auth`]: https://github.com/permutive-engineering/gcp-auth/ +[`pureconfig`]: https://pureconfig.github.io/ \ No newline at end of file From 582f283eef0ac0539ba1f1d7b766ad4fc9ec0d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Herna=CC=81ndez?= Date: Thu, 23 May 2024 19:30:37 +0200 Subject: [PATCH 8/9] Generate gRPC sources ourselves instead of using `googleapis-http4s` --- build.sbt | 10 +++++-- .../main/protobuf/google/package-opts.proto | 20 ++++++++++++++ .../fs2/pubsub/GrpcConstructors.scala | 2 +- .../pubsub/{ => grpc}/GrpcConstructors.scala | 26 ++++++++++++------- .../fs2/pubsub/grpc/internal/package.scala | 22 ++++++++++++++++ .../main/scala/fs2/pubsub/PubSubClient.scala | 1 + .../scala/fs2/pubsub/PubSubPublisher.scala | 1 + .../scala/fs2/pubsub/PubSubSubscriber.scala | 1 + project/Dependencies.scala | 13 +++++++--- project/plugins.sbt | 1 + 10 files changed, 82 insertions(+), 15 deletions(-) create mode 100644 modules/fs2-pubsub/src/main/protobuf/google/package-opts.proto rename modules/fs2-pubsub/src/main/scala-2.13+/fs2/pubsub/{ => grpc}/GrpcConstructors.scala (89%) create mode 100644 modules/fs2-pubsub/src/main/scala-2.13+/fs2/pubsub/grpc/internal/package.scala diff --git a/build.sbt b/build.sbt index ebd55579..1ac27bc6 100644 --- a/build.sbt +++ b/build.sbt @@ -14,9 +14,15 @@ lazy val documentation = project .settings(libraryDependencies ++= Dependencies.documentation) lazy val `fs2-pubsub` = module + .enablePlugins(Http4sGrpcPlugin) .settings(libraryDependencies ++= Dependencies.`fs2-pubsub`) - .settings(libraryDependencies += scalaVersion.value.on(2, 13)(Dependencies.grpc)) - .settings(libraryDependencies += scalaVersion.value.on(3)(Dependencies.grpc)) + .settings(libraryDependencies ++= scalaVersion.value.on(2, 13)(Dependencies.grpc).getOrElse(Nil)) + .settings(libraryDependencies ++= scalaVersion.value.on(3)(Dependencies.grpc).getOrElse(Nil)) + .settings(libraryDependencies -= scalaVersion.value.on(2, 12)(Dependencies.`http4s-grpc`)) + .settings(PB.generate / excludeFilter := "package.proto") + .settings(scalacOptions += "-Wconf:src=src_managed/.*:s") + .settings(Compile / PB.targets += scalapb.gen(grpc = false) -> (Compile / sourceManaged).value / "scalapb") + .settings(Compile / PB.targets := (if (scalaVersion.value.startsWith("2.12")) Nil else (Compile / PB.targets).value)) .settings(Test / fork := true) .settings(Test / run / fork := true) diff --git a/modules/fs2-pubsub/src/main/protobuf/google/package-opts.proto b/modules/fs2-pubsub/src/main/protobuf/google/package-opts.proto new file mode 100644 index 00000000..89b62c1a --- /dev/null +++ b/modules/fs2-pubsub/src/main/protobuf/google/package-opts.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package google; + +import "scalapb/scalapb.proto"; + +option (scalapb.options) = { + scope: PACKAGE; + package_name: "fs2.pubsub.grpc.internal"; + flat_package: true; + aux_message_options: [ + { + target: "*"; + options: { + annotations: "@scala.annotation.nowarn private[grpc]"; + companion_annotations: "@scala.annotation.nowarn private[grpc]"; + } + } + ] +}; diff --git a/modules/fs2-pubsub/src/main/scala-2.12/fs2/pubsub/GrpcConstructors.scala b/modules/fs2-pubsub/src/main/scala-2.12/fs2/pubsub/GrpcConstructors.scala index 668ce3f4..0f74f404 100644 --- a/modules/fs2-pubsub/src/main/scala-2.12/fs2/pubsub/GrpcConstructors.scala +++ b/modules/fs2-pubsub/src/main/scala-2.12/fs2/pubsub/GrpcConstructors.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package fs2.pubsub +package fs2.pubsub.grpc object GrpcConstructors { diff --git a/modules/fs2-pubsub/src/main/scala-2.13+/fs2/pubsub/GrpcConstructors.scala b/modules/fs2-pubsub/src/main/scala-2.13+/fs2/pubsub/grpc/GrpcConstructors.scala similarity index 89% rename from modules/fs2-pubsub/src/main/scala-2.13+/fs2/pubsub/GrpcConstructors.scala rename to modules/fs2-pubsub/src/main/scala-2.13+/fs2/pubsub/grpc/GrpcConstructors.scala index 6bc000dd..8fe42c9f 100644 --- a/modules/fs2-pubsub/src/main/scala-2.13+/fs2/pubsub/GrpcConstructors.scala +++ b/modules/fs2-pubsub/src/main/scala-2.13+/fs2/pubsub/grpc/GrpcConstructors.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package fs2.pubsub +package fs2.pubsub.grpc import java.util.Base64 @@ -22,18 +22,26 @@ import cats.effect.Temporal import cats.syntax.all._ import com.google.protobuf.ByteString -import com.google.pubsub.v1.pubsub.AcknowledgeRequest -import com.google.pubsub.v1.pubsub.ModifyAckDeadlineRequest -import com.google.pubsub.v1.pubsub.PublishRequest -import com.google.pubsub.v1.pubsub.Publisher -import com.google.pubsub.v1.pubsub.PubsubMessage -import com.google.pubsub.v1.pubsub.PullRequest -import com.google.pubsub.v1.pubsub.ReceivedMessage -import com.google.pubsub.v1.pubsub.Subscriber import fs2.Chunk +import fs2.pubsub.AckDeadline +import fs2.pubsub.AckId +import fs2.pubsub.MessageEncoder +import fs2.pubsub.MessageId +import fs2.pubsub.PubSubClient +import fs2.pubsub.PubSubRecord +import fs2.pubsub.Subscription +import fs2.pubsub.Topic import fs2.pubsub.dsl.client.PubSubClientStep import fs2.pubsub.dsl.publisher.PubSubPublisherStep import fs2.pubsub.dsl.subscriber.PubSubSubscriberStep +import fs2.pubsub.grpc.internal.AcknowledgeRequest +import fs2.pubsub.grpc.internal.ModifyAckDeadlineRequest +import fs2.pubsub.grpc.internal.PublishRequest +import fs2.pubsub.grpc.internal.Publisher +import fs2.pubsub.grpc.internal.PubsubMessage +import fs2.pubsub.grpc.internal.PullRequest +import fs2.pubsub.grpc.internal.ReceivedMessage +import fs2.pubsub.grpc.internal.Subscriber import org.http4s.Headers import org.http4s.client.Client import org.http4s.client.middleware.Retry diff --git a/modules/fs2-pubsub/src/main/scala-2.13+/fs2/pubsub/grpc/internal/package.scala b/modules/fs2-pubsub/src/main/scala-2.13+/fs2/pubsub/grpc/internal/package.scala new file mode 100644 index 00000000..659b8768 --- /dev/null +++ b/modules/fs2-pubsub/src/main/scala-2.13+/fs2/pubsub/grpc/internal/package.scala @@ -0,0 +1,22 @@ +package fs2.pubsub.grpc + +import scalapb.TypeMapper + +package object internal { + + implicit val FloatValueTypeMapper: TypeMapper[FloatValue, Float] = + TypeMapper[FloatValue, Float](_.value)(FloatValue.apply(_)) + + implicit val Int64ValueTypeMapper: TypeMapper[Int64Value, Long] = + TypeMapper[Int64Value, Long](_.value)(Int64Value.apply(_)) + + implicit val UInt64ValueTypeMapper: TypeMapper[UInt64Value, Long] = + TypeMapper[UInt64Value, Long](_.value)(UInt64Value.apply(_)) + + implicit val Int32ValueTypeMapper: TypeMapper[Int32Value, Int] = + TypeMapper[Int32Value, Int](_.value)(Int32Value.apply(_)) + + implicit val UInt32ValueTypeMapper: TypeMapper[UInt32Value, Int] = + TypeMapper[UInt32Value, Int](_.value)(UInt32Value.apply(_)) + +} diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubClient.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubClient.scala index c1b3dc50..df3f255b 100644 --- a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubClient.scala +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubClient.scala @@ -28,6 +28,7 @@ import com.permutive.common.types.gcp.http4s._ import fs2.Chunk import fs2.pubsub.dsl.client._ import fs2.pubsub.exceptions.PubSubRequestError +import fs2.pubsub.grpc.GrpcConstructors import io.circe.Decoder import io.circe.Json import io.circe.syntax._ diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubPublisher.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubPublisher.scala index dd08657b..63d47748 100644 --- a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubPublisher.scala +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubPublisher.scala @@ -29,6 +29,7 @@ import fs2.Chunk import fs2.concurrent.Channel import fs2.pubsub.dsl.client._ import fs2.pubsub.dsl.publisher._ +import fs2.pubsub.grpc.GrpcConstructors import org.http4s.Uri /** Represents a class defining a Pub/Sub publisher responsible for producing messages of type `A`. diff --git a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubSubscriber.scala b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubSubscriber.scala index c0359f90..e4b0263d 100644 --- a/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubSubscriber.scala +++ b/modules/fs2-pubsub/src/main/scala/fs2/pubsub/PubSubSubscriber.scala @@ -29,6 +29,7 @@ import fs2.Stream import fs2.concurrent.Channel import fs2.pubsub.dsl.client._ import fs2.pubsub.dsl.subscriber._ +import fs2.pubsub.grpc.GrpcConstructors import org.http4s.Uri /** Contains method for creating Pub/Sub subscribers. */ diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 111ffc43..094a284d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,11 +1,10 @@ import sbt._ import sbt.Keys._ -import mdoc.BuildInfo object Dependencies { lazy val documentation = Seq( - ("org.scalameta" %% "mdoc" % BuildInfo.version).excludeAll( + ("org.scalameta" %% "mdoc" % mdoc.BuildInfo.version).excludeAll( ExclusionRule(organization = "com.thesamet.scalapb", name = "lenses_2.13"), ExclusionRule(organization = "com.thesamet.scalapb", name = "scalapb-runtime_2.13") ), @@ -13,7 +12,15 @@ object Dependencies { "org.http4s" %% "http4s-ember-client" % "0.23.25" ) - lazy val grpc = "io.chrisdavenport" %% "http4s-grpc-google-cloud-pubsub-v1" % "1.108.0+0.0.6" + lazy val `http4s-grpc` = "io.chrisdavenport" %% "http4s-grpc" % "0.0.4" + + lazy val grpc = Seq( + "com.google.api.grpc" % "proto-google-cloud-pubsub-v1" % "1.108.0", + "com.google.api.grpc" % "proto-google-common-protos" % "2.31.0", + "com.google.protobuf" % "protobuf-java" % "3.25.2" + ).map(_ % "protobuf-src" intransitive ()) ++ Seq( + "com.thesamet.scalapb" %% "scalapb-runtime" % scalapb.compiler.Version.scalapbVersion % "protobuf" + ) lazy val `fs2-pubsub` = Seq( "co.fs2" %% "fs2-core" % "3.9.4", diff --git a/project/plugins.sbt b/project/plugins.sbt index 3b5ffaba..2508be85 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -14,3 +14,4 @@ addSbtPlugin("ch.epfl.scala" % "sbt-version-policy" % "3. addSbtPlugin("org.typelevel" % "sbt-tpolecat" % "0.5.0") addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.5.2") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2") +addSbtPlugin("io.chrisdavenport" % "sbt-http4s-grpc" % "0.0.4") From 8cff8d92e599af1c8a0f1cce94eba786f6a563b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Herna=CC=81ndez?= Date: Fri, 24 May 2024 10:20:20 +0200 Subject: [PATCH 9/9] Add small migration guide from `fs2-google-pubsub` --- .github/docs/README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/docs/README.md b/.github/docs/README.md index 4d361e0a..ecbd3184 100644 --- a/.github/docs/README.md +++ b/.github/docs/README.md @@ -242,9 +242,49 @@ PubSubPublisher .noRetry ``` +## Migrating from `fs2-google-pubsub` + +The most important thing you need to take into account while migrating is that +the library no longer creates an authenticated `Client` for you. You need to +provide one yourself using [`permutive-engineering/gcp-auth`][`gcp-auth`]. + +You can use the following table to +find the equivalent classes and methods in `fs2-pubsub`: + +| `fs2-google-pubsub` | `fs2-pubsub` | +|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------| +| [`com.permutive.pubsub.consumer.ConsumerRecord`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/ConsumerRecord.scala) | `fs2.pubsub.PubSubRecord.Publisher` | +| [`com.permutive.pubsub.consumer.ConsumerRecord`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/ConsumerRecord.scala) | `fs2.pubsub.PubSubRecord.Publisher` | +| [`com.permutive.pubsub.consumer.decoder.MessageDecoder`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/decoder/MessageDecoder.scala) | `fs2.pubsub.MessageDecoder` | +| [`com.permutive.pubsub.consumer.grpc.PubsubGoogleConsumer`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/PubsubGoogleConsumer.scala) | `fs2.pubsub.PubSubSubscriber.grpc` | +| [`com.permutive.pubsub.consumer.grpc.PubsubGoogleConsumerConfig`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/consumer/grpc/PubsubGoogleConsumerConfig.scala) | `fs2.pubsub.PubSubSubscriber.Config` | +| [`com.permutive.pubsub.consumer.http.PubsubHttpConsumer`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubHttpConsumer.scala) | `fs2.pubsub.PubSubSubscriber.http` | +| [`com.permutive.pubsub.consumer.http.PubsubHttpConsumerConfig`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubHttpConsumerConfig.scala) | `fs2.pubsub.PubSubSubscriber.Config` | +| [`com.permutive.pubsub.consumer.http.PubsubMessage`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/consumer/http/PubsubMessage.scala) | `fs2.pubsub.PubSubRecord.Subscriber` | +| [`com.permutive.pubsub.consumer.Model.ProjectId`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/Model.scala) | `fs2.pubsub.ProjectId` | +| [`com.permutive.pubsub.consumer.Model.Subscription`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/consumer/Model.scala) | `fs2.pubsub.Subscription` | +| [`com.permutive.pubsub.http.crypto.*`](https://github.com/permutive-engineering/fs2-google-pubsub/tree/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/crypto) | [`permutive-engineering/gcp-auth`][`gcp-auth`] | +| [`com.permutive.pubsub.http.oauth.*`](https://github.com/permutive-engineering/fs2-google-pubsub/tree/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/oauth) | [`permutive-engineering/gcp-auth`][`gcp-auth`] | +| [`com.permutive.pubsub.http.util.RefreshableEffect`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/http/util/RefreshableEffect.scala) | [`permutive-engineering/refreshable`][`refreshable`] | +| [`com.permutive.pubsub.producer.AsyncPubsubProducer`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/AsyncPubsubProducer.scala) | `fs2.pubsub.PubSubPublisher.Async` | +| [`com.permutive.pubsub.producer.encoder.MessageEncoder`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/encoder/MessageEncoder.scala) | `fs2.pubsub.MessageEncoder` | +| [`com.permutive.pubsub.producer.grpc.GooglePubsubProducer`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/GooglePubsubProducer.scala) | `fs2.pubsub.PubSubPublisher.grpc` | +| [`com.permutive.pubsub.producer.grpc.PubsubProducerConfig`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub-grpc/src/main/scala/com/permutive/pubsub/producer/grpc/PubsubProducerConfig.scala) | `fs2.pubsub.PubSubPublisher.Config` | +| [`com.permutive.pubsub.producer.http.BatchingHttpProducerConfig`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/BatchingHttpProducerConfig.scala) | `fs2.pubsub.PubSubPublisher.Async.Config` | +| [`com.permutive.pubsub.producer.http.BatchingHttpPubsubProducer`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/BatchingHttpPubsubProducer.scala) | `fs2.pubsub.PubSubPublisher.Async.http` | +| [`com.permutive.pubsub.producer.http.HttpPubsubProducer`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/HttpPubsubProducer.scala) | `fs2.pubsub.PubSubPublisher.http` | +| [`com.permutive.pubsub.producer.http.PubsubHttpProducerConfig`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub-http/src/main/scala/com/permutive/pubsub/producer/http/PubsubHttpProducerConfig.scala) | `fs2.pubsub.PubSubPublisher.Config` | +| [`com.permutive.pubsub.producer.Model.AsyncRecord`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/Model.scala) | `fs2.pubsub.PubSubRecord.Subscriber.WithCallback` | +| [`com.permutive.pubsub.producer.Model.MessageId`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/Model.scala) | `fs2.pubsub.MessageId` | +| [`com.permutive.pubsub.producer.Model.ProjectId`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/Model.scala) | `fs2.pubsub.ProjectId` | +| [`com.permutive.pubsub.producer.Model.SimpleRecord`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/Model.scala) | `fs2.pubsub.PubSubRecord.Subscriber` | +| [`com.permutive.pubsub.producer.Model.Topic`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/Model.scala) | `fs2.pubsub.Topic` | +| [`com.permutive.pubsub.producer.PubsubProducer`](https://github.com/permutive-engineering/fs2-google-pubsub/blob/a54ad7e698c89aaa6cc280ad482faa7f7ee210e2/fs2-google-pubsub/src/main/scala/com/permutive/pubsub/producer/PubsubProducer.scala) | `fs2.pubsub.PubSubPublisher` | + ## Contributors to this project @CONTRIBUTORS_TABLE@ [`gcp-auth`]: https://github.com/permutive-engineering/gcp-auth/ +[`refreshable`]: https://github.com/permutive-engineering/refreshable/ [`pureconfig`]: https://pureconfig.github.io/ \ No newline at end of file