This repository has been archived by the owner on Oct 23, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added automated integration testing to the library, running tests against a dockerized (localstack) Kinesis instance.
- Loading branch information
Showing
11 changed files
with
359 additions
and
68 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| AWS_REGION=us-east-1 | ||
| AWS_ACCESS_KEY_ID=<AWS_ACCESS_KEY_ID> | ||
| AWS_SECRET_ACCESS_KEY=<AWS_SECRET_ACCESS_KEY> | ||
| AWS_CBOR_DISABLE=true | ||
| APPLICATION_NAME=myReallyAwesomeApplication | ||
| KINESIS_STREAM_NAME=my-stream | ||
| KPL_KINESIS_ENDPOINT=localhost | ||
| KPL_KINESIS_PORT=4568 | ||
| KPL_CLOUDWATCH_ENDPOINT=localhost | ||
| KPL_CLOUDWATCH_PORT=4582 | ||
| KPL_VERIFY_CERTIFICATE=false | ||
| KCL_KINESIS_ENDPOINT=https://localhost:4568 | ||
| KCL_DYNAMODB_ENDPOINT=https://localhost:4569 | ||
| ADDITIONAL_JAVA_OPTS=-Dcom.amazonaws.sdk.disableCertChecking=true | ||
| LOCALSTACK_SERVICES=kinesis,dynamodb,sqs,cloudwatch,cloudformation | ||
| LOCALSTACK_HOSTNAME=localhost | ||
| LOCALSTACK_USE_SSL=true | ||
| LOCALSTACK_KINESIS_ERROR_PROBABILITY=0.0 | ||
| LOCALSTACK_DYNAMODB_ERROR_PROBABILITY=0.0 | ||
| LOCALSTACK_LAMBDA_EXECUTOR=local | ||
| LOCALSTACK_LAMBDA_REMOTE_DOCKER=false | ||
| LOCALSTACK_DATA_DIR=/tmp/localstack/data | ||
| LOCALSTACK_DOCKER_IMAGE_TAG=0.7.5 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| version: "2.3" | ||
|
|
||
| services: | ||
| localstack: | ||
| image: markglh/initialised-localstack:0.8.3 | ||
| volumes: | ||
| - ./templates:/opt/bootstrap/templates | ||
| #network_mode: "host" | ||
| environment: | ||
| - "SERVICES=${LOCALSTACK_SERVICES:-kinesis,dynamodb,cloudwatch,cloudformation}" | ||
| - "DEFAULT_REGION=${AWS_REGION:-us-east-1}" | ||
| - "HOSTNAME=${LOCALSTACK_HOSTNAME:-localhost}" | ||
| - "HOSTNAME_EXTERNAL=${LOCALSTACK_HOSTNAME_EXTERNAL:-localhost}" | ||
| - "USE_SSL=true" | ||
| #- "DATA_DIR=${LOCALSTACK_DATA_DIR:-/tmp/localstack/data}" | ||
| ports: | ||
| - "4567-4582:4567-4582" | ||
| - "8080:8080" | ||
|
|
||
| # Ensures the health check runs | ||
| dummy-service: | ||
| image: alpine:3.5 | ||
| depends_on: | ||
| localstack: | ||
| condition: service_healthy | ||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| AWSTemplateFormatVersion: '2010-09-09' | ||
| Description: CloudFormation Int Test Template | ||
| Resources: | ||
| KinesisStream1: | ||
| Type: AWS::Kinesis::Stream | ||
| Properties: | ||
| Name: int-test-stream-1 | ||
| ShardCount: 1 | ||
| KinesisStream2: | ||
| Type: AWS::Kinesis::Stream | ||
| Properties: | ||
| Name: int-test-stream-2 | ||
| ShardCount: 1 | ||
| KinesisStream3: | ||
| Type: AWS::Kinesis::Stream | ||
| Properties: | ||
| Name: int-test-stream-3 | ||
| ShardCount: 1 | ||
| KinesisStream4: | ||
| Type: AWS::Kinesis::Stream | ||
| Properties: | ||
| Name: int-test-stream-4 | ||
| ShardCount: 1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,59 +1 @@ | ||
|
|
||
| kamon { | ||
| metric.filters { | ||
| akka-actor { | ||
| includes = ["test-system/user/simple-consumer/consumer-worker", "test-system/user/simple-consumer/consumer-worker*/checkpoint-worker*"] | ||
| excludes = ["test-system/system/**"] | ||
| } | ||
|
|
||
| akka-dispatcher { | ||
| includes = ["test-system/akka.actor.default-dispatcher", "test-system/kinesis.akka.default-dispatcher"] | ||
| } | ||
|
|
||
| trace { | ||
| includes = [ "**" ] | ||
| excludes = [] | ||
| } | ||
|
|
||
| #akka-router { | ||
| # includes = [ "test-system/user/some-router" ] | ||
| #} | ||
| } | ||
|
|
||
| statsd { | ||
| # Hostname and port in which your StatsD is running. Remember that StatsD packets are sent using UDP and | ||
| # setting unreachable hosts and/or not open ports wont be warned by the Kamon, your data wont go anywhere. | ||
| hostname = "127.0.0.1" | ||
| port = 8125 | ||
|
|
||
| # Interval between metrics data flushes to StatsD. It's value must be equal or greater than the | ||
| # kamon.metrics.tick-interval setting. | ||
| flush-interval = 1 second | ||
|
|
||
| # Max packet size for UDP metrics data sent to StatsD. | ||
| max-packet-size = 1024 bytes | ||
|
|
||
| # Subscription patterns used to select which metrics will be pushed to StatsD. Note that first, metrics | ||
| # collection for your desired entities must be activated under the kamon.metrics.filters settings. | ||
| includes { | ||
| actor = ["*"] | ||
| trace = ["*"] | ||
| dispatcher = ["*"] | ||
| } | ||
|
|
||
| simple-metric-key-generator { | ||
| # Application prefix for all metrics pushed to StatsD. The default namespacing scheme for metrics follows | ||
| # this pattern: | ||
| # application.host.entity.entity-name.metric-name | ||
| application = "kinesis-test" | ||
| } | ||
| } | ||
|
|
||
| # modules can be disabled at startup using yes/no arguments. | ||
| modules { | ||
| kamon-log-reporter.auto-start = no | ||
| kamon-system-metrics.auto-start = no | ||
| kamon-statsd.auto-start = no | ||
| kamon-akka.auto-start = no | ||
| } | ||
| } |
126 changes: 126 additions & 0 deletions
126
src/it/scala/com/weightwatchers/reactive.kinesis/KinesisProducerIntegrationSpec.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,126 @@ | ||
| /* | ||
| * Copyright 2017 WeightWatchers | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.weightwatchers.reactive.kinesis | ||
|
|
||
| import java.io.File | ||
|
|
||
| import com.amazonaws.services.kinesis.producer.{KinesisProducer => AWSKinesisProducer} | ||
| import com.typesafe.config.ConfigFactory | ||
| import com.weightwatchers.reactive.kinesis.common.{KinesisTestConsumer, TestCredentials} | ||
| import com.weightwatchers.reactive.kinesis.consumer.KinesisConsumer.ConsumerConf | ||
| import com.weightwatchers.reactive.kinesis.models.ProducerEvent | ||
| import com.weightwatchers.reactive.kinesis.producer.{KinesisProducer, ProducerConf} | ||
| import org.scalatest.concurrent.Eventually | ||
| import org.scalatest.mockito.MockitoSugar | ||
| import org.scalatest.time.{Millis, Seconds, Span} | ||
| import org.scalatest.{BeforeAndAfterAll, FreeSpec, Matchers} | ||
|
|
||
| import scala.concurrent.duration._ | ||
| import scala.language.postfixOps | ||
| import scala.util.Random | ||
|
|
||
| //scalastyle:off magic.number | ||
| class KinesisProducerIntegrationSpec | ||
| extends FreeSpec | ||
| with Matchers | ||
| with MockitoSugar | ||
| with BeforeAndAfterAll | ||
| with Eventually { | ||
|
|
||
| implicit val ece = scala.concurrent.ExecutionContext.global | ||
|
|
||
| val defaultKinesisConfig = | ||
| ConfigFactory.parseFile(new File("src/main/resources/reference.conf")).getConfig("kinesis") | ||
|
|
||
| val kinesisConfig = ConfigFactory | ||
| .parseString( | ||
| """ | ||
| |kinesis { | ||
| | | ||
| | application-name = "ScalaProducerTestSpec" | ||
| | | ||
| | testProducer { | ||
| | stream-name = "int-test-stream-1" | ||
| | | ||
| | kpl { | ||
| | Region = us-east-1 | ||
| | | ||
| | CloudwatchEndpoint = localhost | ||
| | CloudwatchPort = 4582 | ||
| | | ||
| | KinesisEndpoint = localhost | ||
| | KinesisPort = 4568 | ||
| | | ||
| | VerifyCertificate = false | ||
| | } | ||
| | } | ||
| | | ||
| | testConsumer { | ||
| | stream-name = "int-test-stream-1" | ||
| | | ||
| | kcl { | ||
| | AWSCredentialsProvider = "com.weightwatchers.reactive.kinesis.common.TestCredentials|foo|bar" | ||
| | regionName = us-east-1 | ||
| | KinesisEndpoint = "https://localhost:4568" | ||
| | DynamoDBEndpoint = "https://localhost:4569" | ||
| | | ||
| | | ||
| | metricsLevel = None | ||
| | } | ||
| | | ||
| | } | ||
| |} | ||
| """.stripMargin | ||
| ) | ||
| .getConfig("kinesis") | ||
| .withFallback(defaultKinesisConfig) | ||
|
|
||
| implicit override val patienceConfig: PatienceConfig = | ||
| PatienceConfig(timeout = Span(5, Seconds), interval = Span(100, Millis)) | ||
|
|
||
| val consumer: KinesisTestConsumer = | ||
| KinesisTestConsumer.from(ConsumerConf(kinesisConfig, "testConsumer"), Some(100 millis)) | ||
|
|
||
| override protected def afterAll(): Unit = { | ||
| consumer.shutdown | ||
| } | ||
|
|
||
| "The KinesisProducer" - { | ||
|
|
||
| "Should publish a message to a stream" in { | ||
|
|
||
| val producerConf = | ||
| ProducerConf(kinesisConfig, "testProducer", Some(TestCredentials.Credentials)) | ||
| val producer = KinesisProducer(producerConf) | ||
|
|
||
| val existingRecordCount = consumer.retrieveRecords(producerConf.streamName, 10).size | ||
|
|
||
| val event = ProducerEvent("1234", Random.alphanumeric.take(10).mkString) | ||
| producer.addUserRecord(event) | ||
|
|
||
| eventually { | ||
| val records: Seq[String] = consumer.retrieveRecords(producerConf.streamName, 10) | ||
| records.size shouldBe (existingRecordCount + 1) | ||
| records should contain( | ||
| new String(event.payload.array(), java.nio.charset.StandardCharsets.UTF_8) | ||
| ) | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| //scalastyle:on |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.