From 7f35b61fcc9be064d50c8054de619156faaf4cd8 Mon Sep 17 00:00:00 2001 From: Ravi Suhag Date: Sat, 15 Jul 2023 08:56:11 -0500 Subject: [PATCH] chore: rename to raystack --- .github/ISSUE_TEMPLATE/config.yml | 8 - .github/workflows/package.yml | 4 +- Dockerfile | 2 +- README.md | 22 +- build.gradle | 24 +- docs/docs/advance/filters.md | 29 +- docs/docs/concepts/architecture.md | 10 +- docs/docs/concepts/monitoring.md | 6 +- docs/docs/concepts/overview.md | 13 +- docs/docs/contribute/contribution.md | 10 +- docs/docs/contribute/development.md | 22 +- docs/docs/guides/create_firehose.md | 12 +- docs/docs/guides/deployment.md | 14 +- docs/docs/reference/core-faqs.md | 6 +- docs/docs/reference/faq.md | 343 +++++++++--------- docs/docs/reference/metrics.md | 157 ++++---- docs/docs/roadmap.md | 4 +- docs/docs/sinks/bigquery-sink.md | 27 +- docs/docs/sinks/bigtable-sink.md | 4 +- docs/docs/sinks/http-sink.md | 10 +- docs/docs/sinks/redis-sink.md | 16 +- docs/docs/support.md | 11 +- docs/docusaurus.config.js | 10 +- docs/static/assets/overview.svg | 2 +- docs/static/overview.svg | 4 +- env/local.properties | 4 +- gradlew.bat | 12 +- .../writer/local/policy/WriterPolicy.java | 7 - .../common/blobstorage/BlobStorageType.java | 6 - .../raystack}/firehose/config/AppConfig.java | 20 +- .../firehose/config/BlobSinkConfig.java | 12 +- .../raystack}/firehose/config/DlqConfig.java | 10 +- .../config/DlqKafkaProducerConfig.java | 2 +- .../firehose/config/ErrorConfig.java | 6 +- .../firehose/config/EsSinkConfig.java | 6 +- .../firehose/config/FilterConfig.java | 16 +- .../raystack}/firehose/config/GCSConfig.java | 2 +- .../firehose/config/GrpcSinkConfig.java | 2 +- .../firehose/config/HttpSinkConfig.java | 26 +- .../firehose/config/InfluxSinkConfig.java | 4 +- .../firehose/config/JdbcSinkConfig.java | 2 +- .../firehose/config/KafkaConsumerConfig.java | 6 +- .../firehose/config/MongoSinkConfig.java | 6 +- .../firehose/config/PromSinkConfig.java | 6 +- .../raystack}/firehose/config/S3Config.java | 2 +- .../firehose/config/SinkPoolConfig.java | 2 +- .../BlobSinkFilePartitionTypeConverter.java | 4 +- .../BlobSinkLocalFileWriterTypeConverter.java | 4 +- .../converter/BlobStorageTypeConverter.java | 4 +- .../converter/ConsumerModeConverter.java | 4 +- .../converter/DlqWriterTypeConverter.java | 4 +- .../converter/EsSinkMessageTypeConverter.java | 4 +- .../FilterDataSourceTypeConverter.java | 4 +- .../converter/FilterEngineTypeConverter.java | 4 +- .../FilterMessageFormatTypeConverter.java | 4 +- .../HttpSinkDataFormatTypeConverter.java | 4 +- .../HttpSinkParameterDataFormatConverter.java | 4 +- ...tpSinkParameterPlacementTypeConverter.java | 4 +- .../HttpSinkParameterSourceTypeConverter.java | 4 +- .../HttpSinkRequestMethodConverter.java | 4 +- .../converter/InputSchemaTypeConverter.java | 4 +- .../config/converter/LabelMapConverter.java | 2 +- .../MongoSinkMessageTypeConverter.java | 4 +- .../ProtoIndexToFieldMapConverter.java | 2 +- .../converter/RangeToHashMapConverter.java | 2 +- .../SchemaRegistryHeadersConverter.java | 2 +- .../SchemaRegistryRefreshConverter.java | 4 +- .../converter/SetErrorTypeConverter.java | 4 +- .../config/converter/SinkTypeConverter.java | 4 +- .../config/enums/EsSinkMessageType.java | 2 +- .../config/enums/EsSinkRequestType.java | 2 +- .../config/enums/FilterDataSourceType.java | 2 +- .../config/enums/FilterEngineType.java | 2 +- .../config/enums/FilterMessageFormatType.java | 2 +- .../config/enums/HttpSinkDataFormatType.java | 2 +- .../enums/HttpSinkParameterPlacementType.java | 2 +- .../enums/HttpSinkParameterSourceType.java | 2 +- .../enums/HttpSinkRequestMethodType.java | 2 +- .../config/enums/InputSchemaType.java | 2 +- .../config/enums/KafkaConsumerMode.java | 2 +- .../config/enums/MongoSinkMessageType.java | 2 +- .../config/enums/MongoSinkRequestType.java | 2 +- .../firehose/config/enums/SinkType.java | 2 +- .../consumer/FirehoseAsyncConsumer.java | 22 +- .../firehose/consumer/FirehoseConsumer.java | 2 +- .../consumer/FirehoseConsumerFactory.java | 75 ++-- .../firehose/consumer/FirehoseFilter.java | 14 +- .../consumer/FirehoseSyncConsumer.java | 22 +- .../kafka/ConsumerAndOffsetManager.java | 10 +- .../consumer/kafka/FirehoseKafkaConsumer.java | 16 +- .../consumer/kafka/OffsetManager.java | 4 +- .../firehose/consumer/kafka/OffsetNode.java | 2 +- .../converter/ProtoTimeConverter.java | 2 +- .../firehose/error/ErrorHandler.java | 8 +- .../raystack}/firehose/error/ErrorScope.java | 2 +- .../exception/ConfigurationException.java | 2 +- .../firehose/exception/DefaultException.java | 2 +- .../exception/DeserializerException.java | 2 +- .../exception/EmptyMessageException.java | 2 +- .../FirehoseConsumerFailedException.java | 2 +- .../exception/JsonParseException.java | 2 +- .../firehose/exception/NeedToRetry.java | 2 +- .../firehose/exception/OAuth2Exception.java | 2 +- .../firehose/exception/SinkException.java | 2 +- .../exception/SinkTaskFailedException.java | 2 +- .../exception/UnknownFieldsException.java | 2 +- .../raystack}/firehose/filter/Filter.java | 4 +- .../firehose/filter/FilterException.java | 2 +- .../firehose/filter/FilteredMessages.java | 4 +- .../raystack}/firehose/filter/NoOpFilter.java | 6 +- .../firehose/filter/jexl/JexlFilter.java | 16 +- .../firehose/filter/json/JsonFilter.java | 22 +- .../firehose/filter/json/JsonFilterUtil.java | 8 +- .../raystack}/firehose/launch/Main.java | 24 +- .../raystack}/firehose/launch/Task.java | 4 +- .../message/FirehoseMessageUtils.java | 9 +- .../raystack}/firehose/message/Message.java | 8 +- .../firehose/metrics/BigQueryMetrics.java | 2 +- .../firehose/metrics/BlobStorageMetrics.java | 2 +- .../metrics/FirehoseInstrumentation.java | 34 +- .../raystack}/firehose/metrics/Metrics.java | 2 +- .../parser/KafkaEnvironmentVariables.java | 2 +- .../firehose/proto/ProtoMessage.java | 8 +- .../firehose/proto/ProtoToFieldMapper.java | 6 +- .../raystack}/firehose/proto/ProtoUtils.java | 2 +- .../serializer/JsonWrappedProtoByte.java | 6 +- .../serializer/MessageJsonSerializer.java | 4 +- .../serializer/MessageSerializer.java | 6 +- .../firehose/serializer/MessageToJson.java | 8 +- .../serializer/MessageToTemplatizedJson.java | 12 +- .../raystack}/firehose/sink/AbstractSink.java | 18 +- .../raystack/firehose/sink/GenericSink.java} | 29 +- .../raystack}/firehose/sink/Sink.java | 5 +- .../raystack}/firehose/sink/SinkFactory.java | 66 ++-- .../firehose/sink/SinkFactoryUtils.java | 4 +- .../raystack}/firehose/sink/SinkPool.java | 19 +- .../sink/bigquery/BigquerySinkUtils.java | 2 +- .../firehose/sink/blob/BlobSink.java | 28 +- .../firehose/sink/blob/BlobSinkFactory.java | 36 +- .../firehose/sink/blob/Constants.java | 2 +- .../sink/blob/message/KafkaMetadataUtils.java | 8 +- .../blob/message/MessageDeSerializer.java | 20 +- .../firehose/sink/blob/message/Record.java | 4 +- .../blob/proto/KafkaMetadataProtoMessage.java | 2 +- .../proto/KafkaMetadataProtoMessageUtils.java | 2 +- .../NestedKafkaMetadataProtoMessage.java | 2 +- .../proto/TimestampMetadataProtoMessage.java | 2 +- .../sink/blob/writer/WriterOrchestrator.java | 30 +- .../blob/writer/WriterOrchestratorStatus.java | 2 +- .../blob/writer/local/LocalFileChecker.java | 24 +- .../blob/writer/local/LocalFileMetadata.java | 2 +- .../blob/writer/local/LocalFileWriter.java | 6 +- .../local/LocalFileWriterFailedException.java | 2 +- .../writer/local/LocalParquetFileWriter.java | 11 +- .../sink/blob/writer/local/LocalStorage.java | 10 +- .../local/path/TimePartitionedPathUtils.java | 8 +- .../local/policy/SizeBasedRotatingPolicy.java | 4 +- .../local/policy/TimeBasedRotatingPolicy.java | 4 +- .../writer/local/policy/WriterPolicy.java | 7 + .../writer/remote/BlobStorageChecker.java | 8 +- .../remote/BlobStorageFailedException.java | 2 +- .../blob/writer/remote/BlobStorageWorker.java | 8 +- .../BlobStorageWriterFutureHandler.java | 24 +- .../sink/common/AbstractHttpSink.java | 17 +- .../sink/common/KeyOrMessageParser.java | 8 +- .../sink/common/blobstorage/BlobStorage.java | 2 +- .../blobstorage/BlobStorageException.java | 2 +- .../blobstorage/BlobStorageFactory.java | 10 +- .../common/blobstorage/BlobStorageType.java | 6 + .../blobstorage/gcs/GoogleCloudStorage.java | 10 +- .../blobstorage/gcs/error/GCSErrorType.java | 2 +- .../sink/common/blobstorage/s3/S3.java | 8 +- .../firehose/sink/dlq/DLQWriterType.java | 2 +- .../firehose/sink/dlq/DlqWriter.java | 4 +- .../firehose/sink/dlq/DlqWriterFactory.java | 24 +- .../dlq/blobstorage/BlobStorageDlqWriter.java | 10 +- .../sink/dlq/blobstorage/DlqMessage.java | 2 +- .../sink/dlq/kafka/KafkaDlqWriter.java | 8 +- .../firehose/sink/dlq/log/LogDlqWriter.java | 10 +- .../firehose/sink/elasticsearch/EsSink.java | 17 +- .../sink/elasticsearch/EsSinkFactory.java | 20 +- .../request/EsRequestHandler.java | 10 +- .../request/EsRequestHandlerFactory.java | 17 +- .../request/EsUpdateRequestHandler.java | 10 +- .../request/EsUpsertRequestHandler.java | 10 +- .../firehose/sink/grpc/GrpcSink.java | 16 +- .../firehose/sink/grpc/GrpcSinkFactory.java | 14 +- .../firehose/sink/grpc/client/GrpcClient.java | 8 +- .../firehose/sink/http/HttpSink.java | 19 +- .../firehose/sink/http/HttpSinkFactory.java | 20 +- .../sink/http/auth/OAuth2AccessToken.java | 2 +- .../firehose/sink/http/auth/OAuth2Client.java | 4 +- .../sink/http/auth/OAuth2Credential.java | 4 +- .../sink/http/factory/SerializerFactory.java | 26 +- .../request/HttpRequestMethodFactory.java | 6 +- .../sink/http/request/RequestFactory.java | 40 +- .../sink/http/request/body/JsonBody.java | 8 +- .../request/create/BatchRequestCreator.java | 22 +- .../create/IndividualRequestCreator.java | 22 +- .../http/request/create/RequestCreator.java | 6 +- .../request/entity/RequestEntityBuilder.java | 4 +- .../http/request/header/HeaderBuilder.java | 8 +- .../request/method/HttpDeleteWithBody.java | 2 +- .../http/request/types/DynamicUrlRequest.java | 28 +- .../types/ParameterizedHeaderRequest.java | 35 +- .../types/ParameterizedUriRequest.java | 35 +- .../sink/http/request/types/Request.java | 16 +- .../http/request/types/SimpleRequest.java | 30 +- .../sink/http/request/uri/UriBuilder.java | 8 +- .../sink/http/request/uri/UriParser.java | 6 +- .../firehose/sink/influxdb/InfluxSink.java | 16 +- .../sink/influxdb/InfluxSinkFactory.java | 12 +- .../sink/influxdb/builder/PointBuilder.java | 6 +- .../sink/jdbc/HikariJdbcConnectionPool.java | 2 +- .../sink/jdbc/JdbcConnectionPool.java | 2 +- .../firehose/sink/jdbc/JdbcMapper.java | 4 +- .../firehose/sink/jdbc/JdbcSink.java | 12 +- .../firehose/sink/jdbc/JdbcSinkFactory.java | 16 +- .../firehose/sink/jdbc/QueryTemplate.java | 8 +- .../sink/jdbc/field/JdbcDefaultField.java | 2 +- .../firehose/sink/jdbc/field/JdbcField.java | 2 +- .../sink/jdbc/field/JdbcFieldFactory.java | 8 +- .../sink/jdbc/field/JdbcMapField.java | 2 +- .../field/message/JdbcCollectionField.java | 4 +- .../message/JdbcDefaultMessageField.java | 4 +- .../field/message/JdbcTimestampField.java | 4 +- .../firehose/sink/mongodb/MongoSink.java | 12 +- .../sink/mongodb/MongoSinkFactory.java | 24 +- .../sink/mongodb/client/MongoSinkClient.java | 21 +- .../mongodb/client/MongoSinkClientUtil.java | 4 +- .../mongodb/request/MongoRequestHandler.java | 10 +- .../request/MongoRequestHandlerFactory.java | 19 +- .../request/MongoUpdateRequestHandler.java | 10 +- .../request/MongoUpsertRequestHandler.java | 10 +- .../mongodb/util/MongoSinkFactoryUtil.java | 4 +- .../firehose/sink/prometheus/PromSink.java | 19 +- .../sink/prometheus/PromSinkConstants.java | 2 +- .../sink/prometheus/PromSinkFactory.java | 18 +- .../prometheus/builder/HeaderBuilder.java | 4 +- .../prometheus/builder/PrometheusLabel.java | 2 +- .../prometheus/builder/PrometheusMetric.java | 2 +- .../builder/RequestEntityBuilder.java | 2 +- .../prometheus/builder/TimeSeriesBuilder.java | 8 +- .../builder/TimeSeriesBuilderUtils.java | 11 +- .../builder/WriteRequestBuilder.java | 6 +- .../sink/prometheus/request/PromRequest.java | 14 +- .../request/PromRequestCreator.java | 18 +- .../firehose/sinkdecorator/BackOff.java | 4 +- .../sinkdecorator/BackOffProvider.java | 2 +- .../ExponentialBackOffProvider.java | 6 +- .../firehose/sinkdecorator/SinkDecorator.java | 8 +- .../firehose/sinkdecorator/SinkFinal.java | 12 +- .../firehose/sinkdecorator/SinkWithDlq.java | 24 +- .../sinkdecorator/SinkWithFailHandler.java | 14 +- .../firehose/sinkdecorator/SinkWithRetry.java | 24 +- .../raystack}/firehose/tracer/SinkTracer.java | 4 +- .../raystack}/firehose/tracer/Traceable.java | 4 +- .../firehose/utils/ConsumerRebalancer.java | 4 +- .../raystack}/firehose/utils/KafkaUtils.java | 14 +- .../firehose/utils/StencilUtils.java | 8 +- .../test/categories/IntegrationTest.java | 4 - .../firehose/config/GCSConfigTest.java | 2 +- .../HttpSinkDataFormatTypeConverterTest.java | 6 +- .../ProtoIndexToFieldMapConverterTest.java | 4 +- .../config/RangeToHashMapConverterTest.java | 4 +- .../firehose/config/S3ConfigTest.java | 2 +- .../InputSchemaTypeConverterTest.java | 4 +- .../SchemaRegistryHeadersConverterTest.java | 5 +- .../ConsumerAndOffsetManagerTest.java | 16 +- .../consumer/FirehoseAsyncConsumerTest.java | 24 +- .../firehose/consumer/FirehoseFilterTest.java | 12 +- .../consumer/FirehoseSyncConsumerTest.java | 24 +- .../kafka/FirehoseKafkaConsumerTest.java | 16 +- .../consumer/kafka/OffsetManagerTest.java | 4 +- .../converter/ProtoTimeConverterTest.java | 2 +- .../firehose/error/ErrorHandlerTest.java | 10 +- .../firehose/filter/NoOpFilterTest.java | 10 +- .../firehose/filter/jexl/JexlFilterTest.java | 24 +- .../firehose/filter/json/JsonFilterTest.java | 26 +- .../filter/json/JsonFilterUtilTest.java | 14 +- .../raystack}/firehose/launch/TaskTest.java | 4 +- .../message/FirehoseMessageUtilsTest.java | 19 +- .../firehose/message/MessageTest.java | 10 +- .../metrics/FirehoseInstrumentationTest.java | 10 +- .../parser/KafkaEnvironmentVariablesTest.java | 2 +- .../firehose/proto/ProtoMessageTest.java | 12 +- .../firehose/proto/ProtoUtilTest.java | 6 +- .../serializer/JsonWrappedProtoByteTest.java | 6 +- .../serializer/MessageToJsonTest.java | 16 +- .../MessageToTemplatizedJsonTest.java | 18 +- .../firehose/sink/AbstractSinkTest.java | 16 +- .../firehose/sink/GenericSinkTest.java} | 29 +- .../firehose/sink/SinkFactoryUtilsTest.java | 2 +- .../raystack}/firehose/sink/SinkPoolTest.java | 6 +- .../sink/bigquery/BigquerySinkUtilsTest.java | 2 +- .../firehose/sink/blob/BlobSinkTest.java | 26 +- .../firehose/sink/blob/TestProtoMessage.java | 8 +- .../firehose/sink/blob/TestUtils.java | 8 +- .../blob/message/KafkaMetadataUtilsTest.java | 8 +- .../blob/message/MessageDeSerializerTest.java | 16 +- .../sink/blob/message/RecordTest.java | 4 +- ...MetadataProtoFirehoseMessageUtilsTest.java | 4 +- .../proto/KafkaMetadataProtoMessageTest.java | 6 +- .../NestedKafkaMetadataProtoMessageTest.java | 6 +- .../TimestampMetadataProtoMessageTest.java | 6 +- .../writer/WriterOrchestratorStatusTest.java | 2 +- .../blob/writer/WriterOrchestratorTest.java | 29 +- .../writer/local/LocalFileCheckerTest.java | 20 +- .../blob/writer/local/LocalStorageTest.java | 10 +- .../local/TimePartitionedPathUtilsTest.java | 14 +- .../policy/SizeBasedRotatingPolicyTest.java | 4 +- .../policy/TimeBasedRotatingPolicyTest.java | 4 +- .../writer/remote/BlobStorageCheckerTest.java | 30 +- .../BlobStorageWriterFutureHandlerTest.java | 10 +- .../sink/common/KeyOrMessageParserTest.java | 10 +- .../common/gcs/GoogleCloudStorageTest.java | 8 +- .../firehose/sink/common/s3/S3Test.java | 8 +- .../firehose/sink/dlq/KafkaDlqWriterTest.java | 16 +- .../firehose/sink/dlq/LogDlqWriterTest.java | 12 +- .../blobstorage/BlobStorageDlqWriterTest.java | 16 +- .../sink/elasticsearch/EsSinkFactoryTest.java | 10 +- .../sink/elasticsearch/EsSinkTest.java | 12 +- .../request/ESUpdateRequestHandlerTest.java | 18 +- .../request/ESUpsertRequestHandlerTest.java | 18 +- .../request/EsRequestHandlerFactoryTest.java | 12 +- .../firehose/sink/grpc/GrpcClientTest.java | 26 +- .../sink/grpc/GrpcSinkFactoryTest.java | 14 +- .../firehose/sink/grpc/GrpcSinkTest.java | 20 +- .../sink/http/HttpSinkFactoryTest.java | 10 +- .../firehose/sink/http/HttpSinkTest.java | 16 +- .../sink/http/auth/OAuth2AccessTokenTest.java | 2 +- .../sink/http/auth/OAuth2ClientTest.java | 4 +- .../sink/http/auth/OAuth2CredentialTest.java | 6 +- .../sink/http/request/RequestFactoryTest.java | 24 +- .../sink/http/request/body/JsonBodyTest.java | 8 +- .../create/BatchRequestCreatorTest.java | 22 +- .../create/IndividualRequestCreatorTest.java | 22 +- .../entity/RequestEntityBuilderTest.java | 2 +- .../request/header/HeaderBuilderTest.java | 10 +- .../request/types/DynamicUrlRequestTest.java | 24 +- .../types/ParameterizedHeaderRequestTest.java | 28 +- .../types/ParameterizedUriRequestTest.java | 28 +- .../http/request/types/SimpleRequestTest.java | 24 +- .../sink/http/request/uri/UriBuilderTest.java | 8 +- .../sink/http/request/uri/UriParserTest.java | 19 +- .../sink/influxdb/InfluxSinkTest.java | 24 +- .../influxdb/builder/PointBuilderTest.java | 10 +- .../jdbc/HikariJdbcConnectionPoolTest.java | 2 +- .../firehose/sink/jdbc/JdbcSinkTest.java | 14 +- .../sink/jdbc/ProtoToFieldMapperTest.java | 22 +- .../firehose/sink/jdbc/QueryTemplateTest.java | 8 +- .../sink/jdbc/field/JdbcMapFieldTest.java | 16 +- .../field/message/DBCollectionFieldTest.java | 16 +- .../message/DBDefaultMessageFieldTest.java | 16 +- .../field/message/DBTimestampFieldTest.java | 14 +- .../sink/mongodb/MongoSinkFactoryTest.java | 6 +- .../firehose/sink/mongodb/MongoSinkTest.java | 12 +- .../mongodb/client/MongoSinkClientTest.java | 22 +- .../client/MongoSinkClientUtilTest.java | 4 +- .../MongoRequestHandlerFactoryTest.java | 12 +- .../MongoUpdateRequestHandlerTest.java | 18 +- .../MongoUpsertRequestHandlerTest.java | 20 +- .../util/MongoSinkFactoryUtilTest.java | 4 +- .../sink/prometheus/PromSinkFactoryTest.java | 10 +- .../sink/prometheus/PromSinkTest.java | 19 +- .../prometheus/builder/HeaderBuilderTest.java | 13 +- .../builder/RequestEntityBuilderTest.java | 7 +- .../builder/TimeSeriesBuilderTest.java | 18 +- .../builder/TimeSeriesBuilderUtilsTest.java | 6 +- .../builder/WriteRequestBuilderTest.java | 17 +- .../request/PromRequestCreatorTest.java | 8 +- .../prometheus/request/PromRequestTest.java | 26 +- .../ExponentialBackOffProviderTest.java | 4 +- .../firehose/sinkdecorator/SinkFinalTest.java | 10 +- .../sinkdecorator/SinkWithDlqTest.java | 44 ++- .../SinkWithFailHandlerTest.java | 16 +- .../sinkdecorator/SinkWithRetryTest.java | 42 +-- .../test/categories/IntegrationTest.java | 4 + .../firehose/tracer/SinkTracerTest.java | 4 +- src/test/proto/TestGrpc.proto | 4 +- src/test/proto/TestLogMessage.proto | 4 +- src/test/proto/TestMessage.proto | 4 +- src/test/proto/TestMessageBQ.proto | 6 +- 383 files changed, 2223 insertions(+), 2220 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/config.yml delete mode 100644 src/main/java/io/odpf/firehose/sink/blob/writer/local/policy/WriterPolicy.java delete mode 100644 src/main/java/io/odpf/firehose/sink/common/blobstorage/BlobStorageType.java rename src/main/java/{io/odpf => org/raystack}/firehose/config/AppConfig.java (84%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/BlobSinkConfig.java (83%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/DlqConfig.java (68%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/DlqKafkaProducerConfig.java (96%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/ErrorConfig.java (83%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/EsSinkConfig.java (87%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/FilterConfig.java (61%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/GCSConfig.java (98%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/GrpcSinkConfig.java (92%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/HttpSinkConfig.java (74%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/InfluxSinkConfig.java (90%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/JdbcSinkConfig.java (95%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/KafkaConsumerConfig.java (91%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/MongoSinkConfig.java (89%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/PromSinkConfig.java (88%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/S3Config.java (95%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/SinkPoolConfig.java (89%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/BlobSinkFilePartitionTypeConverter.java (77%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/BlobSinkLocalFileWriterTypeConverter.java (76%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/BlobStorageTypeConverter.java (71%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/ConsumerModeConverter.java (73%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/DlqWriterTypeConverter.java (74%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/EsSinkMessageTypeConverter.java (73%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/FilterDataSourceTypeConverter.java (81%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/FilterEngineTypeConverter.java (81%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/FilterMessageFormatTypeConverter.java (81%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/HttpSinkDataFormatTypeConverter.java (74%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/HttpSinkParameterDataFormatConverter.java (74%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/HttpSinkParameterPlacementTypeConverter.java (74%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/HttpSinkParameterSourceTypeConverter.java (74%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/HttpSinkRequestMethodConverter.java (74%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/InputSchemaTypeConverter.java (74%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/LabelMapConverter.java (95%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/MongoSinkMessageTypeConverter.java (81%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/ProtoIndexToFieldMapConverter.java (98%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/RangeToHashMapConverter.java (94%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/SchemaRegistryHeadersConverter.java (96%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/SchemaRegistryRefreshConverter.java (81%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/SetErrorTypeConverter.java (74%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/converter/SinkTypeConverter.java (73%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/enums/EsSinkMessageType.java (55%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/enums/EsSinkRequestType.java (62%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/enums/FilterDataSourceType.java (55%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/enums/FilterEngineType.java (56%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/enums/FilterMessageFormatType.java (57%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/enums/HttpSinkDataFormatType.java (57%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/enums/HttpSinkParameterPlacementType.java (61%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/enums/HttpSinkParameterSourceType.java (64%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/enums/HttpSinkRequestMethodType.java (65%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/enums/InputSchemaType.java (56%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/enums/KafkaConsumerMode.java (55%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/enums/MongoSinkMessageType.java (56%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/enums/MongoSinkRequestType.java (60%) rename src/main/java/{io/odpf => org/raystack}/firehose/config/enums/SinkType.java (81%) rename src/main/java/{io/odpf => org/raystack}/firehose/consumer/FirehoseAsyncConsumer.java (81%) rename src/main/java/{io/odpf => org/raystack}/firehose/consumer/FirehoseConsumer.java (79%) rename src/main/java/{io/odpf => org/raystack}/firehose/consumer/FirehoseConsumerFactory.java (81%) rename src/main/java/{io/odpf => org/raystack}/firehose/consumer/FirehoseFilter.java (68%) rename src/main/java/{io/odpf => org/raystack}/firehose/consumer/FirehoseSyncConsumer.java (77%) rename src/main/java/{io/odpf => org/raystack}/firehose/consumer/kafka/ConsumerAndOffsetManager.java (93%) rename src/main/java/{io/odpf => org/raystack}/firehose/consumer/kafka/FirehoseKafkaConsumer.java (91%) rename src/main/java/{io/odpf => org/raystack}/firehose/consumer/kafka/OffsetManager.java (98%) rename src/main/java/{io/odpf => org/raystack}/firehose/consumer/kafka/OffsetNode.java (92%) rename src/main/java/{io/odpf => org/raystack}/firehose/converter/ProtoTimeConverter.java (91%) rename src/main/java/{io/odpf => org/raystack}/firehose/error/ErrorHandler.java (87%) rename src/main/java/{io/odpf => org/raystack}/firehose/error/ErrorScope.java (60%) rename src/main/java/{io/odpf => org/raystack}/firehose/exception/ConfigurationException.java (89%) rename src/main/java/{io/odpf => org/raystack}/firehose/exception/DefaultException.java (87%) rename src/main/java/{io/odpf => org/raystack}/firehose/exception/DeserializerException.java (89%) rename src/main/java/{io/odpf => org/raystack}/firehose/exception/EmptyMessageException.java (84%) rename src/main/java/{io/odpf => org/raystack}/firehose/exception/FirehoseConsumerFailedException.java (79%) rename src/main/java/{io/odpf => org/raystack}/firehose/exception/JsonParseException.java (80%) rename src/main/java/{io/odpf => org/raystack}/firehose/exception/NeedToRetry.java (81%) rename src/main/java/{io/odpf => org/raystack}/firehose/exception/OAuth2Exception.java (79%) rename src/main/java/{io/odpf => org/raystack}/firehose/exception/SinkException.java (78%) rename src/main/java/{io/odpf => org/raystack}/firehose/exception/SinkTaskFailedException.java (79%) rename src/main/java/{io/odpf => org/raystack}/firehose/exception/UnknownFieldsException.java (96%) rename src/main/java/{io/odpf => org/raystack}/firehose/filter/Filter.java (83%) rename src/main/java/{io/odpf => org/raystack}/firehose/filter/FilterException.java (85%) rename src/main/java/{io/odpf => org/raystack}/firehose/filter/FilteredMessages.java (88%) rename src/main/java/{io/odpf => org/raystack}/firehose/filter/NoOpFilter.java (83%) rename src/main/java/{io/odpf => org/raystack}/firehose/filter/jexl/JexlFilter.java (90%) rename src/main/java/{io/odpf => org/raystack}/firehose/filter/json/JsonFilter.java (87%) rename src/main/java/{io/odpf => org/raystack}/firehose/filter/json/JsonFilterUtil.java (89%) rename src/main/java/{io/odpf => org/raystack}/firehose/launch/Main.java (83%) rename src/main/java/{io/odpf => org/raystack}/firehose/launch/Task.java (96%) rename src/main/java/{io/odpf => org/raystack}/firehose/message/FirehoseMessageUtils.java (76%) rename src/main/java/{io/odpf => org/raystack}/firehose/message/Message.java (94%) rename src/main/java/{io/odpf => org/raystack}/firehose/metrics/BigQueryMetrics.java (96%) rename src/main/java/{io/odpf => org/raystack}/firehose/metrics/BlobStorageMetrics.java (97%) rename src/main/java/{io/odpf => org/raystack}/firehose/metrics/FirehoseInstrumentation.java (79%) rename src/main/java/{io/odpf => org/raystack}/firehose/metrics/Metrics.java (99%) rename src/main/java/{io/odpf => org/raystack}/firehose/parser/KafkaEnvironmentVariables.java (96%) rename src/main/java/{io/odpf => org/raystack}/firehose/proto/ProtoMessage.java (90%) rename src/main/java/{io/odpf => org/raystack}/firehose/proto/ProtoToFieldMapper.java (95%) rename src/main/java/{io/odpf => org/raystack}/firehose/proto/ProtoUtils.java (97%) rename src/main/java/{io/odpf => org/raystack}/firehose/serializer/JsonWrappedProtoByte.java (86%) rename src/main/java/{io/odpf => org/raystack}/firehose/serializer/MessageJsonSerializer.java (91%) rename src/main/java/{io/odpf => org/raystack}/firehose/serializer/MessageSerializer.java (70%) rename src/main/java/{io/odpf => org/raystack}/firehose/serializer/MessageToJson.java (96%) rename src/main/java/{io/odpf => org/raystack}/firehose/serializer/MessageToTemplatizedJson.java (92%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/AbstractSink.java (88%) rename src/main/java/{io/odpf/firehose/sink/GenericOdpfSink.java => org/raystack/firehose/sink/GenericSink.java} (55%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/Sink.java (87%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/SinkFactory.java (66%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/SinkFactoryUtils.java (91%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/SinkPool.java (83%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/bigquery/BigquerySinkUtils.java (92%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/BlobSink.java (80%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/BlobSinkFactory.java (73%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/Constants.java (79%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/message/KafkaMetadataUtils.java (86%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/message/MessageDeSerializer.java (75%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/message/Record.java (92%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/proto/KafkaMetadataProtoMessage.java (99%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/proto/KafkaMetadataProtoMessageUtils.java (98%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/proto/NestedKafkaMetadataProtoMessage.java (98%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/proto/TimestampMetadataProtoMessage.java (96%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/WriterOrchestrator.java (86%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/WriterOrchestratorStatus.java (97%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/local/LocalFileChecker.java (67%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/local/LocalFileMetadata.java (86%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/local/LocalFileWriter.java (76%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/local/LocalFileWriterFailedException.java (77%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/local/LocalParquetFileWriter.java (91%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/local/LocalStorage.java (90%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/local/path/TimePartitionedPathUtils.java (91%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/local/policy/SizeBasedRotatingPolicy.java (77%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/local/policy/TimeBasedRotatingPolicy.java (82%) create mode 100644 src/main/java/org/raystack/firehose/sink/blob/writer/local/policy/WriterPolicy.java rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/remote/BlobStorageChecker.java (87%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/remote/BlobStorageFailedException.java (73%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/remote/BlobStorageWorker.java (74%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/remote/BlobStorageWriterFutureHandler.java (61%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/common/AbstractHttpSink.java (93%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/common/KeyOrMessageParser.java (85%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/common/blobstorage/BlobStorage.java (82%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/common/blobstorage/BlobStorageException.java (90%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/common/blobstorage/BlobStorageFactory.java (79%) create mode 100644 src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorageType.java rename src/main/java/{io/odpf => org/raystack}/firehose/sink/common/blobstorage/gcs/GoogleCloudStorage.java (93%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/common/blobstorage/gcs/error/GCSErrorType.java (95%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/common/blobstorage/s3/S3.java (94%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/dlq/DLQWriterType.java (62%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/dlq/DlqWriter.java (84%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/dlq/DlqWriterFactory.java (76%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/dlq/blobstorage/BlobStorageDlqWriter.java (91%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/dlq/blobstorage/DlqMessage.java (91%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/dlq/kafka/KafkaDlqWriter.java (91%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/dlq/log/LogDlqWriter.java (82%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/elasticsearch/EsSink.java (89%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/elasticsearch/EsSinkFactory.java (88%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/elasticsearch/request/EsRequestHandler.java (84%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/elasticsearch/request/EsRequestHandlerFactory.java (71%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/elasticsearch/request/EsUpdateRequestHandler.java (84%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/elasticsearch/request/EsUpsertRequestHandler.java (84%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/grpc/GrpcSink.java (82%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/grpc/GrpcSinkFactory.java (82%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/grpc/client/GrpcClient.java (93%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/HttpSink.java (82%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/HttpSinkFactory.java (86%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/auth/OAuth2AccessToken.java (94%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/auth/OAuth2Client.java (96%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/auth/OAuth2Credential.java (96%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/factory/SerializerFactory.java (73%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/HttpRequestMethodFactory.java (82%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/RequestFactory.java (70%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/body/JsonBody.java (80%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/create/BatchRequestCreator.java (77%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/create/IndividualRequestCreator.java (79%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/create/RequestCreator.java (65%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/entity/RequestEntityBuilder.java (90%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/header/HeaderBuilder.java (89%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/method/HttpDeleteWithBody.java (87%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/types/DynamicUrlRequest.java (71%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/types/ParameterizedHeaderRequest.java (72%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/types/ParameterizedUriRequest.java (71%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/types/Request.java (75%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/types/SimpleRequest.java (72%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/uri/UriBuilder.java (87%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/http/request/uri/UriParser.java (96%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/influxdb/InfluxSink.java (85%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/influxdb/InfluxSinkFactory.java (81%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/influxdb/builder/PointBuilder.java (97%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/jdbc/HikariJdbcConnectionPool.java (98%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/jdbc/JdbcConnectionPool.java (94%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/jdbc/JdbcMapper.java (94%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/jdbc/JdbcSink.java (91%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/jdbc/JdbcSinkFactory.java (86%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/jdbc/QueryTemplate.java (96%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/jdbc/field/JdbcDefaultField.java (88%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/jdbc/field/JdbcField.java (88%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/jdbc/field/JdbcFieldFactory.java (77%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/jdbc/field/JdbcMapField.java (96%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/jdbc/field/message/JdbcCollectionField.java (92%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/jdbc/field/message/JdbcDefaultMessageField.java (89%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/jdbc/field/message/JdbcTimestampField.java (91%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/mongodb/MongoSink.java (90%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/mongodb/MongoSinkFactory.java (90%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/mongodb/client/MongoSinkClient.java (90%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/mongodb/client/MongoSinkClientUtil.java (96%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/mongodb/request/MongoRequestHandler.java (94%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/mongodb/request/MongoRequestHandlerFactory.java (80%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/mongodb/request/MongoUpdateRequestHandler.java (86%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/mongodb/request/MongoUpsertRequestHandler.java (89%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/mongodb/util/MongoSinkFactoryUtil.java (95%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/prometheus/PromSink.java (83%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/prometheus/PromSinkConstants.java (94%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/prometheus/PromSinkFactory.java (86%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/prometheus/builder/HeaderBuilder.java (89%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/prometheus/builder/PrometheusLabel.java (79%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/prometheus/builder/PrometheusMetric.java (79%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/prometheus/builder/RequestEntityBuilder.java (93%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/prometheus/builder/TimeSeriesBuilder.java (94%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/prometheus/builder/TimeSeriesBuilderUtils.java (91%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/prometheus/builder/WriteRequestBuilder.java (93%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/prometheus/request/PromRequest.java (84%) rename src/main/java/{io/odpf => org/raystack}/firehose/sink/prometheus/request/PromRequestCreator.java (74%) rename src/main/java/{io/odpf => org/raystack}/firehose/sinkdecorator/BackOff.java (83%) rename src/main/java/{io/odpf => org/raystack}/firehose/sinkdecorator/BackOffProvider.java (89%) rename src/main/java/{io/odpf => org/raystack}/firehose/sinkdecorator/ExponentialBackOffProvider.java (91%) rename src/main/java/{io/odpf => org/raystack}/firehose/sinkdecorator/SinkDecorator.java (83%) rename src/main/java/{io/odpf => org/raystack}/firehose/sinkdecorator/SinkFinal.java (75%) rename src/main/java/{io/odpf => org/raystack}/firehose/sinkdecorator/SinkWithDlq.java (86%) rename src/main/java/{io/odpf => org/raystack}/firehose/sinkdecorator/SinkWithFailHandler.java (78%) rename src/main/java/{io/odpf => org/raystack}/firehose/sinkdecorator/SinkWithRetry.java (88%) rename src/main/java/{io/odpf => org/raystack}/firehose/tracer/SinkTracer.java (94%) rename src/main/java/{io/odpf => org/raystack}/firehose/tracer/Traceable.java (85%) rename src/main/java/{io/odpf => org/raystack}/firehose/utils/ConsumerRebalancer.java (91%) rename src/main/java/{io/odpf => org/raystack}/firehose/utils/KafkaUtils.java (93%) rename src/main/java/{io/odpf => org/raystack}/firehose/utils/StencilUtils.java (86%) delete mode 100644 src/test/java/io/odpf/firehose/test/categories/IntegrationTest.java rename src/test/java/{io/odpf => org/raystack}/firehose/config/GCSConfigTest.java (97%) rename src/test/java/{io/odpf => org/raystack}/firehose/config/HttpSinkDataFormatTypeConverterTest.java (90%) rename src/test/java/{io/odpf => org/raystack}/firehose/config/ProtoIndexToFieldMapConverterTest.java (97%) rename src/test/java/{io/odpf => org/raystack}/firehose/config/RangeToHashMapConverterTest.java (86%) rename src/test/java/{io/odpf => org/raystack}/firehose/config/S3ConfigTest.java (98%) rename src/test/java/{io/odpf => org/raystack}/firehose/config/converter/InputSchemaTypeConverterTest.java (87%) rename src/test/java/{io/odpf => org/raystack}/firehose/config/converter/SchemaRegistryHeadersConverterTest.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/consumer/ConsumerAndOffsetManagerTest.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/consumer/FirehoseAsyncConsumerTest.java (91%) rename src/test/java/{io/odpf => org/raystack}/firehose/consumer/FirehoseFilterTest.java (93%) rename src/test/java/{io/odpf => org/raystack}/firehose/consumer/FirehoseSyncConsumerTest.java (90%) rename src/test/java/{io/odpf => org/raystack}/firehose/consumer/kafka/FirehoseKafkaConsumerTest.java (97%) rename src/test/java/{io/odpf => org/raystack}/firehose/consumer/kafka/OffsetManagerTest.java (99%) rename src/test/java/{io/odpf => org/raystack}/firehose/converter/ProtoTimeConverterTest.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/error/ErrorHandlerTest.java (95%) rename src/test/java/{io/odpf => org/raystack}/firehose/filter/NoOpFilterTest.java (92%) rename src/test/java/{io/odpf => org/raystack}/firehose/filter/jexl/JexlFilterTest.java (88%) rename src/test/java/{io/odpf => org/raystack}/firehose/filter/json/JsonFilterTest.java (95%) rename src/test/java/{io/odpf => org/raystack}/firehose/filter/json/JsonFilterUtilTest.java (93%) rename src/test/java/{io/odpf => org/raystack}/firehose/launch/TaskTest.java (95%) rename src/test/java/{io/odpf => org/raystack}/firehose/message/FirehoseMessageUtilsTest.java (75%) rename src/test/java/{io/odpf => org/raystack}/firehose/message/MessageTest.java (88%) rename src/test/java/{io/odpf => org/raystack}/firehose/metrics/FirehoseInstrumentationTest.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/parser/KafkaEnvironmentVariablesTest.java (98%) rename src/test/java/{io/odpf => org/raystack}/firehose/proto/ProtoMessageTest.java (89%) rename src/test/java/{io/odpf => org/raystack}/firehose/proto/ProtoUtilTest.java (95%) rename src/test/java/{io/odpf => org/raystack}/firehose/serializer/JsonWrappedProtoByteTest.java (85%) rename src/test/java/{io/odpf => org/raystack}/firehose/serializer/MessageToJsonTest.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/serializer/MessageToTemplatizedJsonTest.java (91%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/AbstractSinkTest.java (96%) rename src/test/java/{io/odpf/firehose/sink/GenericOdpfSinkTest.java => org/raystack/firehose/sink/GenericSinkTest.java} (78%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/SinkFactoryUtilsTest.java (96%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/SinkPoolTest.java (95%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/bigquery/BigquerySinkUtilsTest.java (95%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/BlobSinkTest.java (92%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/TestProtoMessage.java (92%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/TestUtils.java (81%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/message/KafkaMetadataUtilsTest.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/message/MessageDeSerializerTest.java (92%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/message/RecordTest.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/proto/KafkaMetadataProtoFirehoseMessageUtilsTest.java (95%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/proto/KafkaMetadataProtoMessageTest.java (93%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/proto/NestedKafkaMetadataProtoMessageTest.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/proto/TimestampMetadataProtoMessageTest.java (84%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/WriterOrchestratorStatusTest.java (98%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/WriterOrchestratorTest.java (86%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/local/LocalFileCheckerTest.java (92%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/local/LocalStorageTest.java (82%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/local/TimePartitionedPathUtilsTest.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/local/policy/SizeBasedRotatingPolicyTest.java (91%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/local/policy/TimeBasedRotatingPolicyTest.java (92%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/remote/BlobStorageCheckerTest.java (88%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/blob/writer/remote/BlobStorageWriterFutureHandlerTest.java (91%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/common/KeyOrMessageParserTest.java (87%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/common/gcs/GoogleCloudStorageTest.java (90%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/common/s3/S3Test.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/dlq/KafkaDlqWriterTest.java (95%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/dlq/LogDlqWriterTest.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/dlq/blobstorage/BlobStorageDlqWriterTest.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/elasticsearch/EsSinkFactoryTest.java (95%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/elasticsearch/EsSinkTest.java (97%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/elasticsearch/request/ESUpdateRequestHandlerTest.java (96%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/elasticsearch/request/ESUpsertRequestHandlerTest.java (96%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/elasticsearch/request/EsRequestHandlerFactoryTest.java (84%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/grpc/GrpcClientTest.java (90%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/grpc/GrpcSinkFactoryTest.java (77%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/grpc/GrpcSinkTest.java (88%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/HttpSinkFactoryTest.java (92%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/HttpSinkTest.java (98%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/auth/OAuth2AccessTokenTest.java (97%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/auth/OAuth2ClientTest.java (97%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/auth/OAuth2CredentialTest.java (98%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/request/RequestFactoryTest.java (81%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/request/body/JsonBodyTest.java (87%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/request/create/BatchRequestCreatorTest.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/request/create/IndividualRequestCreatorTest.java (96%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/request/entity/RequestEntityBuilderTest.java (96%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/request/header/HeaderBuilderTest.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/request/types/DynamicUrlRequestTest.java (88%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/request/types/ParameterizedHeaderRequestTest.java (89%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/request/types/ParameterizedUriRequestTest.java (88%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/request/types/SimpleRequestTest.java (89%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/request/uri/UriBuilderTest.java (95%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/http/request/uri/UriParserTest.java (93%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/influxdb/InfluxSinkTest.java (93%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/influxdb/builder/PointBuilderTest.java (96%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/jdbc/HikariJdbcConnectionPoolTest.java (96%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/jdbc/JdbcSinkTest.java (95%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/jdbc/ProtoToFieldMapperTest.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/jdbc/QueryTemplateTest.java (97%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/jdbc/field/JdbcMapFieldTest.java (86%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/jdbc/field/message/DBCollectionFieldTest.java (87%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/jdbc/field/message/DBDefaultMessageFieldTest.java (83%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/jdbc/field/message/DBTimestampFieldTest.java (84%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/mongodb/MongoSinkFactoryTest.java (97%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/mongodb/MongoSinkTest.java (96%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/mongodb/client/MongoSinkClientTest.java (97%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/mongodb/client/MongoSinkClientUtilTest.java (96%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/mongodb/request/MongoRequestHandlerFactoryTest.java (93%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/mongodb/request/MongoUpdateRequestHandlerTest.java (94%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/mongodb/request/MongoUpsertRequestHandlerTest.java (95%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/mongodb/util/MongoSinkFactoryUtilTest.java (97%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/prometheus/PromSinkFactoryTest.java (75%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/prometheus/PromSinkTest.java (95%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/prometheus/builder/HeaderBuilderTest.java (74%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/prometheus/builder/RequestEntityBuilderTest.java (86%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/prometheus/builder/TimeSeriesBuilderTest.java (98%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/prometheus/builder/TimeSeriesBuilderUtilsTest.java (96%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/prometheus/builder/WriteRequestBuilderTest.java (89%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/prometheus/request/PromRequestCreatorTest.java (82%) rename src/test/java/{io/odpf => org/raystack}/firehose/sink/prometheus/request/PromRequestTest.java (75%) rename src/test/java/{io/odpf => org/raystack}/firehose/sinkdecorator/ExponentialBackOffProviderTest.java (96%) rename src/test/java/{io/odpf => org/raystack}/firehose/sinkdecorator/SinkFinalTest.java (81%) rename src/test/java/{io/odpf => org/raystack}/firehose/sinkdecorator/SinkWithDlqTest.java (90%) rename src/test/java/{io/odpf => org/raystack}/firehose/sinkdecorator/SinkWithFailHandlerTest.java (85%) rename src/test/java/{io/odpf => org/raystack}/firehose/sinkdecorator/SinkWithRetryTest.java (92%) create mode 100644 src/test/java/org/raystack/firehose/test/categories/IntegrationTest.java rename src/test/java/{io/odpf => org/raystack}/firehose/tracer/SinkTracerTest.java (95%) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index 20f91b411..000000000 --- a/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1,8 +0,0 @@ -blank_issues_enabled: true -contact_links: - - name: Help - url: https://odpf-community.slack.com/archives/C01LMME51QC - about: "If you have a question or need help, ask a question on the discussion forums." - - name: Show and tell - url: https://odpf-community.slack.com/archives/C01LCE0CYQM - about: "Have something nice to show and tell? We'd love to hear it!" diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index f729f4819..a2c8ad79c 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -45,5 +45,5 @@ jobs: context: . push: true tags: | - odpf/firehose:latest - odpf/firehose:${{ steps.get_version.outputs.version-without-v }} + raystack/firehose:latest + raystack/firehose:${{ steps.get_version.outputs.version-without-v }} diff --git a/Dockerfile b/Dockerfile index 0c3598cdc..78dd2f5ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,4 +10,4 @@ COPY --from=GRADLE_BUILD ./jolokia-jvm-agent.jar /opt/firehose COPY --from=GRADLE_BUILD ./src/main/resources/log4j.xml /opt/firehose/etc/log4j.xml COPY --from=GRADLE_BUILD ./src/main/resources/logback.xml /opt/firehose/etc/logback.xml WORKDIR /opt/firehose -CMD ["java", "-cp", "bin/*:/work-dir/*", "io.odpf.firehose.launch.Main", "-server", "-Dlogback.configurationFile=etc/firehose/logback.xml", "-Xloggc:/var/log/firehose"] +CMD ["java", "-cp", "bin/*:/work-dir/*", "org.raystack.firehose.launch.Main", "-server", "-Dlogback.configurationFile=etc/firehose/logback.xml", "-Xloggc:/var/log/firehose"] diff --git a/README.md b/README.md index 7ec88c015..52c7e2dd1 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # Firehose -![build workflow](https://github.com/odpf/firehose/actions/workflows/build.yml/badge.svg) -![package workflow](https://github.com/odpf/firehose/actions/workflows/package.yml/badge.svg) +![build workflow](https://github.com/raystack/firehose/actions/workflows/build.yml/badge.svg) +![package workflow](https://github.com/raystack/firehose/actions/workflows/package.yml/badge.svg) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg?logo=apache)](LICENSE) -[![Version](https://img.shields.io/github/v/release/odpf/firehose?logo=semantic-release)](Version) +[![Version](https://img.shields.io/github/v/release/raystack/firehose?logo=semantic-release)](Version) Firehose is a cloud native service for delivering real-time streaming data to destinations such as service endpoints (HTTP or GRPC) & managed databases (Postgres, InfluxDB, Redis, Elasticsearch, Prometheus and MongoDB). With Firehose, you don't need to write applications or manage resources. It can be scaled up to match the throughput of your data. If your data is present in Kafka, Firehose delivers it to the destination(SINK) that you specified. @@ -25,7 +25,7 @@ Discover why users choose Firehose as their main Kafka Consumer - Elasticsearch - Redis - Bigquery - - BigTable + - BigTable - Blob Storage/Object Storage : - Google Cloud Storage @@ -47,28 +47,28 @@ Explore the following resources to get started with Firehose: ## Run with Docker -Use the docker hub to download firehose [docker image](https://hub.docker.com/r/odpf/firehose/). You need to have docker installed in your system. +Use the docker hub to download firehose [docker image](https://hub.docker.com/r/raystack/firehose/). You need to have docker installed in your system. ``` # Download docker image from docker hub -$ docker pull odpf/firehose +$ docker pull raystack/firehose # Run the following docker command for a simple log sink. -$ docker run -e SOURCE_KAFKA_BROKERS=127.0.0.1:6667 -e SOURCE_KAFKA_CONSUMER_GROUP_ID=kafka-consumer-group-id -e SOURCE_KAFKA_TOPIC=sample-topic -e SINK_TYPE=log -e SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET=latest -e INPUT_SCHEMA_PROTO_CLASS=com.github.firehose.sampleLogProto.SampleLogMessage -e SCHEMA_REGISTRY_STENCIL_ENABLE=true -e SCHEMA_REGISTRY_STENCIL_URLS=http://localhost:9000/artifactory/proto-descriptors/latest odpf/firehose:latest +$ docker run -e SOURCE_KAFKA_BROKERS=127.0.0.1:6667 -e SOURCE_KAFKA_CONSUMER_GROUP_ID=kafka-consumer-group-id -e SOURCE_KAFKA_TOPIC=sample-topic -e SINK_TYPE=log -e SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET=latest -e INPUT_SCHEMA_PROTO_CLASS=com.github.firehose.sampleLogProto.SampleLogMessage -e SCHEMA_REGISTRY_STENCIL_ENABLE=true -e SCHEMA_REGISTRY_STENCIL_URLS=http://localhost:9000/artifactory/proto-descriptors/latest/raystack/firehose:latest ``` **Note:** Make sure your protos (.jar file) are located in `work-dir`, this is required for Filter functionality to work. ## Run with Kubernetes -- Create a firehose deployment using the helm chart available [here](https://github.com/odpf/charts/tree/main/stable/firehose) +- Create a firehose deployment using the helm chart available [here](https://github.com/raystack/charts/tree/main/stable/firehose) - Deployment also includes telegraf container which pushes stats metrics ## Running locally ```sh # Clone the repo -$ git clone https://github.com/odpf/firehose.git +$ git clone https://github.com/raystack/firehose.git # Build the jar $ ./gradlew clean build @@ -101,11 +101,11 @@ Development of Firehose happens in the open on GitHub, and we are grateful to th Read our [contributing guide](docs/docs/contribute/contribution.md) to learn about our development process, how to propose bugfixes and improvements, and how to build and test your changes to Firehose. -To help you get your feet wet and get you familiar with our contribution process, we have a list of [good first issues](https://github.com/odpf/firehose/labels/good%20first%20issue) that contain bugs which have a relatively limited scope. This is a great place to get started. +To help you get your feet wet and get you familiar with our contribution process, we have a list of [good first issues](https://github.com/raystack/firehose/labels/good%20first%20issue) that contain bugs which have a relatively limited scope. This is a great place to get started. ## Credits -This project exists thanks to all the [contributors](https://github.com/odpf/firehose/graphs/contributors). +This project exists thanks to all the [contributors](https://github.com/raystack/firehose/graphs/contributors). ## License diff --git a/build.gradle b/build.gradle index 834105c58..ff49b9fff 100644 --- a/build.gradle +++ b/build.gradle @@ -32,8 +32,8 @@ lombok { sha256 = "" } -group 'io.odpf' -version '0.7.4' +group 'org.raystack' +version '0.8.0' def projName = "firehose" @@ -41,6 +41,7 @@ sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 repositories { + mavenLocal() mavenCentral() jcenter() maven { @@ -54,7 +55,7 @@ private Properties loadEnv() { properties } -def mainClassName = "io.odpf.firehose.launch.Main" +def mainClassName = "org.raystack.firehose.launch.Main" dependencies { implementation group: 'com.google.protobuf', name: 'protobuf-java', version: '3.1.0' @@ -71,7 +72,7 @@ dependencies { implementation group: 'org.apache.commons', name: 'commons-jexl', version: '2.1' implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.5' implementation group: 'com.google.code.gson', name: 'gson', version: '2.7' - implementation group: 'io.odpf', name: 'stencil', version: '0.2.1' exclude group: 'org.slf4j' + implementation group: 'org.raystack', name: 'stencil', version: '0.4.0' exclude group: 'org.slf4j' implementation group: 'software.amazon.awssdk', name: 's3', version: '2.17.129' implementation group: 'org.influxdb', name: 'influxdb-java', version: '2.5' implementation group: 'com.jayway.jsonpath', name: 'json-path', version: '2.4.0' @@ -85,7 +86,7 @@ dependencies { exclude group: "log4j", module: "log4j" } implementation 'io.confluent:monitoring-interceptors:3.0.0' - implementation "io.grpc:grpc-all:1.38.0" + implementation 'io.grpc:grpc-all:1.53.0' implementation group: 'org.jfrog.buildinfo', name: 'build-info-extractor', version: '2.6.3' implementation group: 'com.google.gradle', name: 'osdetector-gradle-plugin', version: '1.2.1' implementation group: 'org.apache.ivy', name: 'ivy', version: '2.2.0' @@ -98,10 +99,9 @@ dependencies { implementation 'com.gojek.parquet:parquet-hadoop:1.11.9' implementation group: 'com.github.os72', name: 'protobuf-dynamic', version: '1.0.1' implementation platform('com.google.cloud:libraries-bom:20.5.0') - implementation 'com.google.cloud:google-cloud-storage:1.114.0' - implementation 'com.google.cloud:google-cloud-bigquery:1.115.0' - implementation 'org.apache.logging.log4j:log4j-core:2.17.1' - implementation group: 'io.odpf', name: 'depot', version: '0.3.8' + implementation 'com.google.cloud:google-cloud-storage:2.20.1' + implementation 'org.apache.logging.log4j:log4j-core:2.20.0' + implementation group: 'org.raystack', name: 'depot', version: '0.4.0' implementation group: 'com.networknt', name: 'json-schema-validator', version: '1.0.59' exclude group: 'org.slf4j' testImplementation group: 'junit', name: 'junit', version: '4.11' @@ -146,7 +146,7 @@ test { events "passed", "skipped", "failed" } useJUnit { - excludeCategories 'io.odpf.firehose.test.categories.IntegrationTest' + excludeCategories 'org.raystack.firehose.test.categories.IntegrationTest' } doLast { delete "$projectDir/src/test/resources/__files" @@ -158,7 +158,7 @@ clean { } jar { manifest { - attributes 'Main-Class': 'io.odpf.firehose.launch.Main' + attributes 'Main-Class': 'org.raystack.firehose.launch.Main' duplicatesStrategy = 'exclude' zip64 = true } @@ -181,7 +181,7 @@ publishing { repositories { maven { name = "GitHubPackages" - url = "https://maven.pkg.github.com/odpf/firehose" + url = "https://maven.pkg.github.com/raystack/firehose" credentials { username = System.getenv("GITHUB_ACTOR") password = System.getenv("GITHUB_TOKEN") diff --git a/docs/docs/advance/filters.md b/docs/docs/advance/filters.md index c8a8d4a85..49e9dc3fe 100644 --- a/docs/docs/advance/filters.md +++ b/docs/docs/advance/filters.md @@ -6,43 +6,42 @@ Following variables need to be set to enable JSON/JEXL filters. Defines whether to use `JSON` Schema-based filters or `JEXL`-based filters or `NO_OP` \(i.e. no filtering\) -* Example value: `JSON` -* Type: `optional` -* Default value`: NO_OP` +- Example value: `JSON` +- Type: `optional` +- Default value`: NO_OP` ## `FILTER_JSON_ESB_MESSAGE_TYPE` Defines the format type of the input ESB messages, i.e. JSON/Protobuf. This field is required only for JSON filters. -* Example value: `JSON` -* Type: `optional` +- Example value: `JSON` +- Type: `optional` ## `FILTER_SCHEMA_PROTO_CLASS` The fully qualified name of the proto schema so that the key/message in Kafka could be parsed. -* Example value: `com.gojek.esb.driverlocation.DriverLocationLogKey` -* Type: `optional` +- Example value: `com.raystack.esb.driverlocation.DriverLocationLogKey` +- Type: `optional` ## `FILTER_DATA_SOURCE` `key`/`message`/`none`depending on where to apply filter -* Example value: `key` -* Type: `optional` -* Default value`: none` +- Example value: `key` +- Type: `optional` +- Default value`: none` ## `FILTER_JEXL_EXPRESSION` JEXL filter expression -* Example value: `driverLocationLogKey.getVehicleType()=="BIKE"` -* Type: `optional` +- Example value: `driverLocationLogKey.getVehicleType()=="BIKE"` +- Type: `optional` ## `FILTER_JSON_SCHEMA` JSON Schema string containing the filter rules to be applied. -* Example value: `{"properties":{"order_number":{"const":"1253"}}}` -* Type: `optional` - +- Example value: `{"properties":{"order_number":{"const":"1253"}}}` +- Type: `optional` diff --git a/docs/docs/concepts/architecture.md b/docs/docs/concepts/architecture.md index 8b42bda82..f76da644d 100644 --- a/docs/docs/concepts/architecture.md +++ b/docs/docs/concepts/architecture.md @@ -40,7 +40,7 @@ _**Sink**_ - All the existing sink types follow the same contract/lifecycle defined in `AbstractSink.java`. It consists of two stages: - Prepare: Transformation over-filtered messages’ list to prepare the sink-specific insert/update client requests. - Execute: Requests created in the Prepare stage are executed at this step and a list of failed messages is returned \(if any\) for retry. -- Underlying implementation of AbstractSink can use implementation present in [depot](https://github.com/odpf/depot). +- Underlying implementation of AbstractSink can use implementation present in [depot](https://github.com/raystack/depot). - If the batch has any failures, Firehose will retry to push the failed messages to the sink _**SinkPool**_ @@ -71,7 +71,9 @@ The final state of message can be any one of the followings after it is consumed One can monitor via plotting the metrics related to messages. ### Schema Handling + - Incase when `INPUT_SCHEMA_DATA_TYPE is set to protobuf` + - Protocol buffers are Google's language-neutral, platform-neutral, extensible mechanism for serializing structured data. Data streams on Kafka topics are bound to a protobuf schema. - Firehose deserializes the data consumed from the topics using the Protobuf descriptors generated out of the artifacts. The artifactory is an HTTP interface that Firehose uses to deserialize. - The schema handling ie., find the mapped schema for the topic, downloading the descriptors, and dynamically being notified of/updating with the latest schema is abstracted through the Stencil library. @@ -81,10 +83,8 @@ One can monitor via plotting the metrics related to messages. Schema Caching, dynamic schema updates, etc. are features of the stencil client library. - Incase when `INPUT_SCHEMA_DATA_TYPE is set to json` - - Currently this config is only supported in Bigquery sink, - - For json, in bigquery sink the schema is dynamically inferred from incoming data, in future we plan to provide json schema support via stencil. - - + - Currently this config is only supported in Bigquery sink, + - For json, in bigquery sink the schema is dynamically inferred from incoming data, in future we plan to provide json schema support via stencil. ## Firehose Integration diff --git a/docs/docs/concepts/monitoring.md b/docs/docs/concepts/monitoring.md index 5322c23e3..46a0421a4 100644 --- a/docs/docs/concepts/monitoring.md +++ b/docs/docs/concepts/monitoring.md @@ -71,11 +71,11 @@ Lastly, set up Telegraf to send metrics to InfluxDB, following the corresponding #### Firehose deployed on Kubernetes _\*\*_ -1. Follow[ this guide](https://github.com/odpf/charts/tree/main/stable/firehose#readme) for deploying Firehose on a Kubernetes cluster using a Helm chart. -2. Configure the following parameters in the default [values.yaml](https://github.com/odpf/charts/blob/main/stable/firehose/values.yaml) file and run - +1. Follow[ this guide](https://github.com/raystack/charts/tree/main/stable/firehose#readme) for deploying Firehose on a Kubernetes cluster using a Helm chart. +2. Configure the following parameters in the default [values.yaml](https://github.com/raystack/charts/blob/main/stable/firehose/values.yaml) file and run - ```text -$ helm install my-release -f values.yaml odpf/firehose +$ helm install my-release -f values.yaml raystack/firehose ``` | Key | Type | Default | Description | diff --git a/docs/docs/concepts/overview.md b/docs/docs/concepts/overview.md index f862a3262..d5002b6e5 100644 --- a/docs/docs/concepts/overview.md +++ b/docs/docs/concepts/overview.md @@ -10,7 +10,7 @@ scale. This section explains the overall architecture of Firehose and describes ## [Monitoring Firehose with exposed metrics](monitoring.md) Always know what’s going on with your deployment with -built-in [monitoring](https://github.com/odpf/firehose/blob/main/docs/assets/firehose-grafana-dashboard.json) of +built-in [monitoring](https://github.com/raystack/firehose/blob/main/docs/assets/firehose-grafana-dashboard.json) of throughput, response times, errors and more. This section contains guides, best practices and advises related to managing Firehose in production. @@ -27,12 +27,13 @@ Firehose provides various templating features Decorators are used for chained processing of messages. -* SinkWithFailHandler -* SinkWithRetry -* SinkWithDlq -* SinkFinal +- SinkWithFailHandler +- SinkWithRetry +- SinkWithDlq +- SinkFinal ## [FirehoseConsumer](consumer.md) + A firehose consumer read messages from kafka, pushes those messages to sink and commits offsets back to kafka based on certain strategies. ## [Offsets](offsets.md) @@ -40,5 +41,3 @@ A firehose consumer read messages from kafka, pushes those messages to sink and Offset manager is a data structure used to manage offsets asynchronously. An offset should only be committed when a message is processed fully. Offset manager maintains a state of all the offsets of all topic-partitions, that can be committed. It can also be used by sinks to manage its own offsets. - - diff --git a/docs/docs/contribute/contribution.md b/docs/docs/contribute/contribution.md index 5d33f3a52..7fb1671dd 100644 --- a/docs/docs/contribute/contribution.md +++ b/docs/docs/contribute/contribution.md @@ -3,8 +3,8 @@ The following is a set of guidelines for contributing to Firehose. These are mostly guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request. Here are some important resources: - The [Concepts](../guides/create_firehose.md) section will explain to you about Firehose architecture, -- Our [roadmap](https://github.com/odpf/firehose/blob/main/docs/roadmap.md) is the 10k foot view of where we're going, and -- Github [issues](https://github.com/odpf/firehose/issues) track the ongoing and reported issues. +- Our [roadmap](https://github.com/raystack/firehose/blob/main/docs/roadmap.md) is the 10k foot view of where we're going, and +- Github [issues](https://github.com/raystack/firehose/issues) track the ongoing and reported issues. Development of Firehose happens in the open on GitHub, and we are grateful to the community for contributing bug fixes and improvements. Read below to learn how you can take part in improving Firehose. @@ -23,14 +23,14 @@ The following parts are open for contribution: - Provide suggestions to make the user experience better - Provide suggestions to Improve the documentation -To help you get your feet wet and get you familiar with our contribution process, we have a list of [good first issues](https://github.com/odpf/firehose/labels/good%20first%20issue) that contain bugs that have a relatively limited scope. This is a great place to get started. +To help you get your feet wet and get you familiar with our contribution process, we have a list of [good first issues](https://github.com/raystack/firehose/labels/good%20first%20issue) that contain bugs that have a relatively limited scope. This is a great place to get started. ## How can I contribute? We use RFCs and GitHub issues to communicate ideas. - You can report a bug or suggest a feature enhancement or can just ask questions. Reach out on Github discussions for this purpose. -- You are also welcome to add a new common sink in [depot](https://github.com/odpf/depot), improve monitoring and logging and improve code quality. +- You are also welcome to add a new common sink in [depot](https://github.com/raystack/depot), improve monitoring and logging and improve code quality. - You can help with documenting new features or improve existing documentation. - You can also review and accept other contributions if you are a maintainer. @@ -53,4 +53,4 @@ Please follow these practices for your change to get merged fast and smoothly: - If you are introducing a completely new feature or making any major changes to an existing one, we recommend starting with an RFC and get consensus on the basic design first. - Make sure your local build is running with all the tests and checkstyle passing. - If your change is related to user-facing protocols/configurations, you need to make the corresponding change in the documentation as well. -- Docs live in the code repo under [`docs`](https://github.com/odpf/firehose/tree/7d0df99962507e6ad2147837c4536f36d52d5a48/docs/docs/README.md) so that changes to that can be done in the same PR as changes to the code. +- Docs live in the code repo under [`docs`](https://github.com/raystack/firehose/tree/main/docs/docs/README.md) so that changes to that can be done in the same PR as changes to the code. diff --git a/docs/docs/contribute/development.md b/docs/docs/contribute/development.md index 9f561c47c..1ae462a90 100644 --- a/docs/docs/contribute/development.md +++ b/docs/docs/contribute/development.md @@ -39,7 +39,7 @@ Configuration parameter variables of each sink can be found in the [Configuratio When `INPUT_SCHEMA_DATA_TYPE is set to protobuf`, firehose uses Stencil Server as its Schema Registry for hosting Protobuf descriptors. The environment variable `SCHEMA_REGISTRY_STENCIL_ENABLE` must be set to `true` . Stencil server URL must be specified in the variable `SCHEMA_REGISTRY_STENCIL_URLS` . The Proto Descriptor Set file of the Kafka messages must be uploaded to the Stencil server. -Refer [this guide](https://github.com/odpf/stencil/tree/master/server#readme) on how to set up and configure the Stencil server, and how to generate and upload Proto descriptor set file to the server. +Refer [this guide](https://github.com/raystack/stencil/tree/master/server#readme) on how to set up and configure the Stencil server, and how to generate and upload Proto descriptor set file to the server. ### Monitoring @@ -56,7 +56,7 @@ Firehose sends critical metrics via StatsD client. Refer the[ Monitoring](../con ```bash # Clone the repo -$ git clone https://github.com/odpf/firehose.git +$ git clone https://github.com/raystack/firehose.git # Build the jar $ ./gradlew clean build @@ -64,6 +64,7 @@ $ ./gradlew clean build # Configure env variables $ cat env/local.properties ``` + ### Configure env/local.properties Set the generic variables in the local.properties file. @@ -72,9 +73,11 @@ Set the generic variables in the local.properties file. KAFKA_RECORD_PARSER_MODE = message SINK_TYPE = log INPUT_SCHEMA_DATA_TYPE=protobuf -INPUT_SCHEMA_PROTO_CLASS = io.odpf.firehose.consumer.TestMessage +INPUT_SCHEMA_PROTO_CLASS = org.raystack.firehose.consumer.TestMessage ``` + Set the variables which specify the kafka server, topic name, and group-id of the kafka consumer - the standard values are used here. + ```text SOURCE_KAFKA_BROKERS = localhost:9092 SOURCE_KAFKA_TOPIC = test-topic @@ -82,22 +85,27 @@ SOURCE_KAFKA_CONSUMER_GROUP_ID = sample-group-id ``` ### Stencil Workaround -Firehose uses [Stencil](https://github.com/odpf/stencil) as the schema-registry which enables dynamic proto schemas. For the sake of this -quick-setup guide, we can work our way around Stencil setup by setting up a simple local HTTP server which can provide the static descriptor for TestMessage schema. +Firehose uses [Stencil](https://github.com/raystack/stencil) as the schema-registry which enables dynamic proto schemas. For the sake of this +quick-setup guide, we can work our way around Stencil setup by setting up a simple local HTTP server which can provide the static descriptor for TestMessage schema. - Install a server service - like [this](https://github.com/http-party/http-server) one. - Generate the descriptor for TestMessage by running the command on terminal - + ```shell ./gradlew generateTestProto ``` -- The above should generate a file (src/test/resources/__files/descriptors.bin), move this to a new folder at a separate location, and start the HTTP-server there so that this file can be fetched at the runtime. + +- The above should generate a file (src/test/resources/\_\_files/descriptors.bin), move this to a new folder at a separate location, and start the HTTP-server there so that this file can be fetched at the runtime. - If you are using [this](https://github.com/http-party/http-server), use this command after moving the file to start server at the default port number 8080. + ```shell http-server ``` + - Because we are not using the schema-registry in the default mode, the following lines should also be added in env/local.properties to specify the new location to fetch descriptor from. + ```text SCHEMA_REGISTRY_STENCIL_ENABLE = true SCHEMA_REGISTRY_STENCIL_URLS = http://localhost:8080/descriptors.bin @@ -109,11 +117,11 @@ SCHEMA_REGISTRY_STENCIL_REFRESH_STRATEGY = LONG_POLLING - Make sure that your kafka server and local HTTP server containing the descriptor is up and running. - Run the firehose consumer through the gradlew task: + ```shell ./gradlew runConsumer ``` - **Note:** Sample configuration for other sinks along with some advanced configurations can be found [here](../advance/generic/) ### Running tests diff --git a/docs/docs/guides/create_firehose.md b/docs/docs/guides/create_firehose.md index 37cec7e7b..531672ca9 100644 --- a/docs/docs/guides/create_firehose.md +++ b/docs/docs/guides/create_firehose.md @@ -21,9 +21,9 @@ INPUT_SCHEMA_PROTO_CLASS=com.tests.TestMessage Sample output of a Firehose log sink: ```text -2021-03-29T08:43:05,998Z [pool-2-thread-1] INFO i.o.firehose.Consumer- Execution successful for 1 records -2021-03-29T08:43:06,246Z [pool-2-thread-1] INFO i.o.firehose.Consumer - Pulled 1 messages -2021-03-29T08:43:06,246Z [pool-2-thread-1] INFO io.odpf.firehose.sink.log.LogSink - +2021-03-29T08:43:05,998Z [pool-2-thread-1] INFO org.raystack.firehose.Consumer- Execution successful for 1 records +2021-03-29T08:43:06,246Z [pool-2-thread-1] INFO org.raystack.firehose.Consumer - Pulled 1 messages +2021-03-29T08:43:06,246Z [pool-2-thread-1] INFO org.raystack.firehose.sink.log.LogSink - ================= DATA ======================= sample_field: 81179979 sample_field_2: 9897987987 @@ -131,13 +131,13 @@ _**Note:**_ [_**DATABASE**_](../sinks/influxdb-sink.md#sink_influx_db_name) _**a - it requires the following [variables](../sinks/bigquery-sink.md) to be set. - For INPUT_SCHEMA_DATA_TYPE = protobuf, this sink will generate bigquery schema from protobuf message schema and update bigquery table with the latest generated schema. - - The protobuf message of a `google.protobuf.Timestamp` field might be needed when table partitioning is enabled. + - The protobuf message of a `google.protobuf.Timestamp` field might be needed when table partitioning is enabled. - For INPUT_SCHEMA_DATA_TYPE = json, this sink will generate bigquery schema by infering incoming json. In future we will add support for json schema as well coming from stencil. - - The timestamp column is needed incase of partition table. It can be generated at the time of ingestion by setting the config. Please refer to config `SINK_BIGQUERY_ADD_EVENT_TIMESTAMP_ENABLE` in [depot bigquery sink config section](https://github.com/odpf/depot/blob/main/docs/reference/configuration/bigquery-sink.md#sink_bigquery_add_event_timestamp_enable) + - The timestamp column is needed incase of partition table. It can be generated at the time of ingestion by setting the config. Please refer to config `SINK_BIGQUERY_ADD_EVENT_TIMESTAMP_ENABLE` in [depot bigquery sink config section](https://github.com/raystack/depot/blob/main/docs/reference/configuration/bigquery-sink.md#sink_bigquery_add_event_timestamp_enable) - Google cloud credential with some bigquery permission is required to run this sink. ## Create a Bigtable sink -- it requires the following environment [variables](https://github.com/odpf/depot/blob/main/docs/reference/configuration/bigtable.md) ,which are required by ODPF Depot library, to be set along with the generic firehose variables. +- it requires the following environment [variables](https://github.com/raystack/depot/blob/main/docs/reference/configuration/bigtable.md) ,which are required by Depot library, to be set along with the generic firehose variables. If you'd like to connect to a sink which is not yet supported, you can create a new sink by following the [contribution guidelines](../contribute/contribution.md) diff --git a/docs/docs/guides/deployment.md b/docs/docs/guides/deployment.md index fa418f500..e0ecc5041 100644 --- a/docs/docs/guides/deployment.md +++ b/docs/docs/guides/deployment.md @@ -13,14 +13,14 @@ Refer the [Development Guide](../contribute/development.md) section on how to se ## Deploy on Docker -Use the Docker hub to download Firehose [docker image](https://hub.docker.com/r/odpf/firehose/). You need to have Docker installed in your system. Follow[ this guide](https://www.docker.com/products/docker-desktop) on how to install and set up Docker in your system. +Use the Docker hub to download Firehose [docker image](https://hub.docker.com/r/raystack/firehose/). You need to have Docker installed in your system. Follow[ this guide](https://www.docker.com/products/docker-desktop) on how to install and set up Docker in your system. ```text # Download docker image from docker hub -$ docker pull odpf/firehose +$ docker pull raystack/firehose # Run the following docker command for a simple log sink. -$ docker run -e SOURCE_KAFKA_BROKERS=127.0.0.1:6667 -e SOURCE_KAFKA_CONSUMER_GROUP_ID=kafka-consumer-group-id -e SOURCE_KAFKA_TOPIC=sample-topic -e SINK_TYPE=log -e SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET=latest -e INPUT_SCHEMA_PROTO_CLASS=com.github.firehose.sampleLogProto.SampleLogMessage -e SCHEMA_REGISTRY_STENCIL_ENABLE=true -e SCHEMA_REGISTRY_STENCIL_URLS=http://localhost:9000/artifactory/proto-descriptors/latest odpf/firehose:latest +$ docker run -e SOURCE_KAFKA_BROKERS=127.0.0.1:6667 -e SOURCE_KAFKA_CONSUMER_GROUP_ID=kafka-consumer-group-id -e SOURCE_KAFKA_TOPIC=sample-topic -e SINK_TYPE=log -e SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET=latest -e INPUT_SCHEMA_PROTO_CLASS=com.github.firehose.sampleLogProto.SampleLogMessage -e SCHEMA_REGISTRY_STENCIL_ENABLE=true -e SCHEMA_REGISTRY_STENCIL_URLS=http://localhost:9000/artifactory/proto-descriptors/latest/raystack/firehose:latest ``` **Note:** Make sure your protos \(.jar file\) are located in `work-dir`, this is required for Filter functionality to work. @@ -29,9 +29,9 @@ $ docker run -e SOURCE_KAFKA_BROKERS=127.0.0.1:6667 -e SOURCE_KAFKA_CONSUMER_GRO Kubernetes is an open-source container-orchestration system for automating computer application deployment, scaling, and management. Follow [this guide](https://kubernetes.io/docs/setup/) on how to set up and configure a Kubernetes cluster. -Then create a Firehose deployment using the Helm chart available [here](https://github.com/odpf/charts/tree/main/stable/firehose). The Helm chart Deployment also includes Telegraf container which works as a metrics aggregator and pushes StatsD metrics to InfluxDB. Make sure to configure all Kafka, sink and filter parameters in `values.yaml` file before deploying the Helm chart. +Then create a Firehose deployment using the Helm chart available [here](https://github.com/raystack/charts/tree/main/stable/firehose). The Helm chart Deployment also includes Telegraf container which works as a metrics aggregator and pushes StatsD metrics to InfluxDB. Make sure to configure all Kafka, sink and filter parameters in `values.yaml` file before deploying the Helm chart. -Description and default values for each parameter can be found[ here](https://github.com/odpf/charts/tree/main/stable/firehose#values). +Description and default values for each parameter can be found[ here](https://github.com/raystack/charts/tree/main/stable/firehose#values). ## Deploy locally @@ -39,7 +39,7 @@ Firehose needs Java SE Development Kit 8 to be installed and configured in `JAVA ```text # Clone the repo -$ git clone https://github.com/odpf/firehose.git +$ git clone https://github.com/raystack/firehose.git # Build the jar $ ./gradlew clean build @@ -51,4 +51,4 @@ $ cat env/local.properties $ ./gradlew runConsumer ``` -**Note:** Sample configuration for other sinks along with some advanced configurations can be found [here](https://github.com/odpf/firehose/blob/main/docs/reference/configuration.md) +**Note:** Sample configuration for other sinks along with some advanced configurations can be found [here](https://github.com/raystack/firehose/blob/main/docs/reference/configuration.md) diff --git a/docs/docs/reference/core-faqs.md b/docs/docs/reference/core-faqs.md index 96c044560..2de8ab13d 100644 --- a/docs/docs/reference/core-faqs.md +++ b/docs/docs/reference/core-faqs.md @@ -126,17 +126,17 @@ Firehose will send all the fields of the incoming messages to the specified sink ## What is Stencil in context of Firehose ? -ODPF Stencil API is a dynamic schema registry for hosting and managing versions of Protobuf descriptors. The schema handling i.e., find the mapped schema for the topic, downloading the descriptors, and dynamically being notified of/updating with the latest schema is abstracted through the Stencil library. +Stencil API is a dynamic schema registry for hosting and managing versions of Protobuf descriptors. The schema handling i.e., find the mapped schema for the topic, downloading the descriptors, and dynamically being notified of/updating with the latest schema is abstracted through the Stencil library. The Stencil Client is a proprietary library that provides an abstraction layer, for schema handling. Schema Caching, dynamic schema updates are features of the stencil client library. -Refer [this article](https://odpf.gitbook.io/stencil/) for further information of the features, configuration and deployment instructions of the Stencil API. Source code of Stencil Server and Client API can be found in its [Github repository](https://github.com/odpf/stencil). +Refer [this article](https://raystack.gitbook.io/stencil/) for further information of the features, configuration and deployment instructions of the Stencil API. Source code of Stencil Server and Client API can be found in its [Github repository](https://github.com/raystack/stencil). ## How do I configure Protobuf needed to consume ? Generated Protobuf Descriptors are hosted behind an Stencil server artifactory/HTTP endpoint. This endpoint URL and the ProtoDescriptor class that the Firehose deployment should use to deserialize raw data with is configured in Firehose in the environment variables`SCHEMA_REGISTRY_STENCIL_URLS`and`INPUT_SCHEMA_PROTO_CLASS` respectively . -The Proto Descriptor Set of the Kafka messages must be uploaded to the Stencil server. Refer [this guide](https://github.com/odpf/stencil/tree/master/server#readme) on how to setup and configure the Stencil server. +The Proto Descriptor Set of the Kafka messages must be uploaded to the Stencil server. Refer [this guide](https://github.com/raystack/stencil/tree/master/server#readme) on how to setup and configure the Stencil server. ## Can we select particular fields from the input message ? diff --git a/docs/docs/reference/faq.md b/docs/docs/reference/faq.md index d0bd5631e..cf72b4758 100644 --- a/docs/docs/reference/faq.md +++ b/docs/docs/reference/faq.md @@ -6,103 +6,103 @@ answers. # Table of contents - [Frequently Asked Questions](#frequently-asked-questions) - - [Firehose Sinks](#firehose-sinks) - - [Blob Sink](#blob-sink) - - [What file formats are the records written into ?](#what-file-formats-are-the-records-written-into-) - - [Does the sink support json/Avro/Parquet/CSV ?](#does-the-sink-support-jsonavroparquetcsv-) - - [Does the sink support file compression GZIP/Snappy/lz4 ?](#does-the-sink-support-file-compression-gzipsnappylz4-) - - [Does the sink support partitioning ?](#does-the-sink-support-partitioning-) - - [How to configure partitioning ?](#how-to-configure-partitioning-) - - [Does the sink support S3 or hdfs ?](#does-the-sink-support-s3-or-hdfs-) - - [How the schema being generated ? How the data types being converted to Parquet types ?](#how-the-schema-being-generated--how-the-data-types-being-converted-to-parquet-types-) - - [How is the folder path for the file that being created ?](#how-is-the-folder-path-for-the-file-that-being-created-) - - [How to configure object storage bucket ?](#how-to-configure-object-storage-bucket-) - - [What will happen when file upload failed ?](#what-will-happen-when-file-upload-failed-) - - [What will happen when disk is full ?](#what-will-happen-when-disk-is-full-) - - [How to implement new Object Storage provider, for example S3 ?](#how-to-implement-new-object-storage-provider-for-example-s3-) - - [How to implement new file format ?](#how-to-implement-new-file-format-) - - [How to commit being handled ?](#how-to-commit-being-handled-) - - [How much Disk size recommended for the sink ?](#how-much-disk-size-recommended-for-the-sink-) - - [BigQuery Sink](#bigquery-sink) - - [What is the method that is being used to insert the bq rows ?](#what-is-the-method-that-is-being-used-to-insert-the-bq-rows-) - - [Does the sink support partitioned table ?](#does-the-sink-support-partitioned-table-) - - [How to configure partitioning ?](#how-to-configure-partitioning-) - - [How the schema being generated ? How the data types being converted to BQ ?](#how-the-schema-being-generated--how-the-data-types-being-converted-to-bq-) - - [Does the sink support ingestion time/ integer range partitioning?](#does-the-sink-support-ingestion-time-integer-range-partitioning) - - [How to configure table destination ?](#how-to-configure-table-destination-) - - [What will happen when on insertion of a record the timestamp is out of range more than 5 year in the past or 1 year in the future ?](#what-will-happen-when-on-insertion-of-a-record-the-timestamp-is-out-of-range-more-than-5-year-in-the-past-or-1-year-in-the-future-) - - [How many records are inserted/ batched each time ?](#how-many-records-are-inserted-batched-each-time-) - - [When is the BigQuery table schema updated ?](#when-is-the-bigquery-table-schema-updated-) - - [Is the table automatically created ?](#is-the-table-automatically-created-) - - [Does this sink support BigQuery table clustering configuration ?](#does-this-sink-support-bigquery-table-clustering-configuration-) - - [Does this sink support BigQuery table labeling ?](#does-this-sink-support-bigquery-table-labeling-) - - [Redis Sink](#redis-sink) - - [What is the Redis version supported ?](#what-is-the-redis-version-supported-) - - [What Data types are supported in Redis sink?](#what-data-types-are-supported-in-redis-sink) - - [What all deployments types of Redis is supported ?](#what-all-deployments-types-of-redis-is-supported-) - - [How to use Redis cluster for Redis sink?](#how-to-use-redis-cluster-for-redis-sink) - - [How to specify a template for the keys ?](#how-to-specify-a-template-for-the-keys-) - - [How to select nested fields?](#how-to-select-nested-fields) - - [What is the behaviour on connection failures?](#what-is-the-behaviour-on-connection-failures) - - [How can TTL be configured for the Redis keys?](#how-can-ttl-be-configured-for-the-redis-keys) - - [Does it support deleting the keys?](#does-it-support-deleting-the-keys) - - [What are some of the use cases of this sink?](#what-are-some-of-the-use-cases-of-this-sink) - - [What happens if the Redis goes down?](#what-happens-if-the-redis-goes-down) - - [JDBC Sink](#jdbc-sink) - - [What are some of the use cases of this sink?](#what-are-some-of-the-use-cases-of-this-sink) - - [What monitoring metrics are available for this sink?](#what-monitoring-metrics-are-available-for-this-sink) - - [Do we need to create table/schema before pushing data via JDBC sink ?](#do-we-need-to-create-tableschema-before-pushing-data-via-jdbc-sink-) - - [Any restriction of version of supported database?](#any-restriction-of-version-of-supported-database) - - [How messages get mapped to the queries?](#how-messages-get-mapped-to-the-queries) - - [How data types are handled?](#how-data-types-are-handled) - - [How are the database connections are formed?](#how-are-the-database-connections-are-formed) - - [How does JDBC sink handles connection pooling?](#how-does-jdbc-sink-handles-connection-pooling) - - [When and how do the DB connections gets closed?](#when-and-how-do-the-db-connections-gets-closed) - - [How to support a new data base which supports JDBC, e:g MySQL ?](#how-to-support-a-new-data-base-which-supports-jdbc-eg-mysql-) - - [Any transaction, locking provision?](#any-transaction-locking-provision) - - [Are there any chances of race conditions?](#are-there-any-chances-of-race-conditions) - - [HTTP Sink](#http-sink) - - [How does the payload look like?](#how-does-the-payload-look-like) - - [Does it support DELETE calls?](#does-it-support-delete-calls) - - [How many messages are pushed in one call?](#how-many-messages-are-pushed-in-one-call) - - [How can I configure the number of connections?](#how-can-i-configure-the-number-of-connections) - - [What data types of request body are supported?](#what-data-types-of-request-body-are-supported) - - [Does it support client side load balancing?](#does-it-support-client-side-load-balancing) - - [Which authentication methods are supported?](#which-authentication-methods-are-supported) - - [How can I pass a particular input field as a header in request?](#how-can-i-pass-a-particular-input-field-as-a-header-in-request) - - [How can I pass a particular input field as a query param in request?](#how-can-i-pass-a-particular-input-field-as-a-query-param-in-request) - - [What happens if my services fails ?](#what-happens-if-my-services-fails-) - - [How are HTTP connections handled, long lived?](#how-are-http-connections-handled-long-lived) - - [What is logRequest config?](#what-is-logrequest-config) - - [What is shouldRetry config?](#what-is-shouldretry-config) - - [What happens in case of null response?](#what-happens-in-case-of-null-response) - - [What happens in case of null status code in a non-null response?](#what-happens-in-case-of-null-status-code-in-a-non-null-response) - - [When is a message dropped?](#when-is-a-message-dropped) - - [What is the difference between Parameterised vs Dynamic Url ?](#what-is-the-difference-between-parameterised-vs-dynamic-url-) - - [For parameterised header, how is the data added?](#for-parameterised-header-how-is-the-data-added) - - [General Queries](#general-queries) - - [Which Java versions firehose work with?](#which-java-versions-firehose-work-with) - - [How does the execution work? Is the entire process sync or async ?](#how-does-the-execution-work-is-the-entire-process-sync-or-async-) - - [Is there any code snippet which shows how I can produce sample message in supported data format?](#is-there-any-code-snippet-which-shows-how-i-can-produce-sample-message-in-supported-data-format) - - [Why Protobuf ? Can it support other formats like JSON, AVRO, Thrift ?](#why-protobuf--can-it-support-other-formats-like-json-avro-thrift-) - - [Does firehose have support for SSL on Kafka?](#does-firehose-have-support-for-ssl-on-kafka) - - [How does Firehose create the Kafka consumer?](#how-does--firehose-create-the-kafka-consumer) - - [Can I change the consumer group name or reset it ? Overall, how do I handle my consumer group operations?](#can-i-change-the-consumer-group-name-or-reset-it--overall-how-do-i-handle-my-consumer-group-operations) - - [What are the advantages of Firehose over Kafka connect ?](#what-are-the-advantages-of-firehose-over-kafka-connect-) - - [What problems does Firehose solve?](#what-problems-does-firehose-solve) - - [Can I do any transformations before sending data to sink, for example filtering ?](#can-i-do-any-transformations-before-sending-data-to-sink-for-example-filtering-) - - [How to optimise parallelism based on input rate of Kafka messages? Does it depend on sink ?](#how-to-optimise-parallelism-based-on-input-rate-of-kafka-messages-does-it-depend-on-sink-) - - [What are the retry mechanisms in firehose?](#what-are-the-retry-mechanisms-in-firehose) - - [Which Kafka client configs are available ?](#which-kafka-client-configs-are-available-) - - [What all data formats are supported?](#what-all-data-formats-are-supported) - - [Can we select particular fields from the input message?](#can-we-select-particular-fields-from-the-input-message) - - [How do I configure the Protobuf schema for the topic Firehose needs to consume?](#how-do-i-configure-the-protobuf-schema-for-the-topic-firehose-needs-to-consume) - - [What is Stencil in context of firehose ?](#what-is-stencil-in-context-of-firehose-) - - [Will I have any data loss if my firehose is failed ?](#will-i-have-any-data-loss-if-my-firehose-is-failed-) - - [How does firehose handle failed messages?](#how-does-firehose-handle-failed-messages) - - [What all metrics are produced for me to monitor ?](#what-all-metrics-are-produced-for-me-to-monitor-) - - [What kind of delivery guarantees does Firehose provide? Is it different from what Kafka is tuned for?](#what-kind-of-delivery-guarantees-does-firehose-provide-is-it-different-from-what-kafka-is-tuned-for) - - [What happens if my firehose is stopped and Kafka retention is for few days?](#what-happens-if-my-firehose-is-stopped-and-kafka-retention-is-for-few-days) + - [Firehose Sinks](#firehose-sinks) + - [Blob Sink](#blob-sink) + - [What file formats are the records written into ?](#what-file-formats-are-the-records-written-into-) + - [Does the sink support json/Avro/Parquet/CSV ?](#does-the-sink-support-jsonavroparquetcsv-) + - [Does the sink support file compression GZIP/Snappy/lz4 ?](#does-the-sink-support-file-compression-gzipsnappylz4-) + - [Does the sink support partitioning ?](#does-the-sink-support-partitioning-) + - [How to configure partitioning ?](#how-to-configure-partitioning-) + - [Does the sink support S3 or hdfs ?](#does-the-sink-support-s3-or-hdfs-) + - [How the schema being generated ? How the data types being converted to Parquet types ?](#how-the-schema-being-generated--how-the-data-types-being-converted-to-parquet-types-) + - [How is the folder path for the file that being created ?](#how-is-the-folder-path-for-the-file-that-being-created-) + - [How to configure object storage bucket ?](#how-to-configure-object-storage-bucket-) + - [What will happen when file upload failed ?](#what-will-happen-when-file-upload-failed-) + - [What will happen when disk is full ?](#what-will-happen-when-disk-is-full-) + - [How to implement new Object Storage provider, for example S3 ?](#how-to-implement-new-object-storage-provider-for-example-s3-) + - [How to implement new file format ?](#how-to-implement-new-file-format-) + - [How to commit being handled ?](#how-to-commit-being-handled-) + - [How much Disk size recommended for the sink ?](#how-much-disk-size-recommended-for-the-sink-) + - [BigQuery Sink](#bigquery-sink) + - [What is the method that is being used to insert the bq rows ?](#what-is-the-method-that-is-being-used-to-insert-the-bq-rows-) + - [Does the sink support partitioned table ?](#does-the-sink-support-partitioned-table-) + - [How to configure partitioning ?](#how-to-configure-partitioning-) + - [How the schema being generated ? How the data types being converted to BQ ?](#how-the-schema-being-generated--how-the-data-types-being-converted-to-bq-) + - [Does the sink support ingestion time/ integer range partitioning?](#does-the-sink-support-ingestion-time-integer-range-partitioning) + - [How to configure table destination ?](#how-to-configure-table-destination-) + - [What will happen when on insertion of a record the timestamp is out of range more than 5 year in the past or 1 year in the future ?](#what-will-happen-when-on-insertion-of-a-record-the-timestamp-is-out-of-range-more-than-5-year-in-the-past-or-1-year-in-the-future-) + - [How many records are inserted/ batched each time ?](#how-many-records-are-inserted-batched-each-time-) + - [When is the BigQuery table schema updated ?](#when-is-the-bigquery-table-schema-updated-) + - [Is the table automatically created ?](#is-the-table-automatically-created-) + - [Does this sink support BigQuery table clustering configuration ?](#does-this-sink-support-bigquery-table-clustering-configuration-) + - [Does this sink support BigQuery table labeling ?](#does-this-sink-support-bigquery-table-labeling-) + - [Redis Sink](#redis-sink) + - [What is the Redis version supported ?](#what-is-the-redis-version-supported-) + - [What Data types are supported in Redis sink?](#what-data-types-are-supported-in-redis-sink) + - [What all deployments types of Redis is supported ?](#what-all-deployments-types-of-redis-is-supported-) + - [How to use Redis cluster for Redis sink?](#how-to-use-redis-cluster-for-redis-sink) + - [How to specify a template for the keys ?](#how-to-specify-a-template-for-the-keys-) + - [How to select nested fields?](#how-to-select-nested-fields) + - [What is the behaviour on connection failures?](#what-is-the-behaviour-on-connection-failures) + - [How can TTL be configured for the Redis keys?](#how-can-ttl-be-configured-for-the-redis-keys) + - [Does it support deleting the keys?](#does-it-support-deleting-the-keys) + - [What are some of the use cases of this sink?](#what-are-some-of-the-use-cases-of-this-sink) + - [What happens if the Redis goes down?](#what-happens-if-the-redis-goes-down) + - [JDBC Sink](#jdbc-sink) + - [What are some of the use cases of this sink?](#what-are-some-of-the-use-cases-of-this-sink) + - [What monitoring metrics are available for this sink?](#what-monitoring-metrics-are-available-for-this-sink) + - [Do we need to create table/schema before pushing data via JDBC sink ?](#do-we-need-to-create-tableschema-before-pushing-data-via-jdbc-sink-) + - [Any restriction of version of supported database?](#any-restriction-of-version-of-supported-database) + - [How messages get mapped to the queries?](#how-messages-get-mapped-to-the-queries) + - [How data types are handled?](#how-data-types-are-handled) + - [How are the database connections are formed?](#how-are-the-database-connections-are-formed) + - [How does JDBC sink handles connection pooling?](#how-does-jdbc-sink-handles-connection-pooling) + - [When and how do the DB connections gets closed?](#when-and-how-do-the-db-connections-gets-closed) + - [How to support a new data base which supports JDBC, e:g MySQL ?](#how-to-support-a-new-data-base-which-supports-jdbc-eg-mysql-) + - [Any transaction, locking provision?](#any-transaction-locking-provision) + - [Are there any chances of race conditions?](#are-there-any-chances-of-race-conditions) + - [HTTP Sink](#http-sink) + - [How does the payload look like?](#how-does-the-payload-look-like) + - [Does it support DELETE calls?](#does-it-support-delete-calls) + - [How many messages are pushed in one call?](#how-many-messages-are-pushed-in-one-call) + - [How can I configure the number of connections?](#how-can-i-configure-the-number-of-connections) + - [What data types of request body are supported?](#what-data-types-of-request-body-are-supported) + - [Does it support client side load balancing?](#does-it-support-client-side-load-balancing) + - [Which authentication methods are supported?](#which-authentication-methods-are-supported) + - [How can I pass a particular input field as a header in request?](#how-can-i-pass-a-particular-input-field-as-a-header-in-request) + - [How can I pass a particular input field as a query param in request?](#how-can-i-pass-a-particular-input-field-as-a-query-param-in-request) + - [What happens if my services fails ?](#what-happens-if-my-services-fails-) + - [How are HTTP connections handled, long lived?](#how-are-http-connections-handled-long-lived) + - [What is logRequest config?](#what-is-logrequest-config) + - [What is shouldRetry config?](#what-is-shouldretry-config) + - [What happens in case of null response?](#what-happens-in-case-of-null-response) + - [What happens in case of null status code in a non-null response?](#what-happens-in-case-of-null-status-code-in-a-non-null-response) + - [When is a message dropped?](#when-is-a-message-dropped) + - [What is the difference between Parameterised vs Dynamic Url ?](#what-is-the-difference-between-parameterised-vs-dynamic-url-) + - [For parameterised header, how is the data added?](#for-parameterised-header-how-is-the-data-added) + - [General Queries](#general-queries) + - [Which Java versions firehose work with?](#which-java-versions-firehose-work-with) + - [How does the execution work? Is the entire process sync or async ?](#how-does-the-execution-work-is-the-entire-process-sync-or-async-) + - [Is there any code snippet which shows how I can produce sample message in supported data format?](#is-there-any-code-snippet-which-shows-how-i-can-produce-sample-message-in-supported-data-format) + - [Why Protobuf ? Can it support other formats like JSON, AVRO, Thrift ?](#why-protobuf--can-it-support-other-formats-like-json-avro-thrift-) + - [Does firehose have support for SSL on Kafka?](#does-firehose-have-support-for-ssl-on-kafka) + - [How does Firehose create the Kafka consumer?](#how-does--firehose-create-the-kafka-consumer) + - [Can I change the consumer group name or reset it ? Overall, how do I handle my consumer group operations?](#can-i-change-the-consumer-group-name-or-reset-it--overall-how-do-i-handle-my-consumer-group-operations) + - [What are the advantages of Firehose over Kafka connect ?](#what-are-the-advantages-of-firehose-over-kafka-connect-) + - [What problems does Firehose solve?](#what-problems-does-firehose-solve) + - [Can I do any transformations before sending data to sink, for example filtering ?](#can-i-do-any-transformations-before-sending-data-to-sink-for-example-filtering-) + - [How to optimise parallelism based on input rate of Kafka messages? Does it depend on sink ?](#how-to-optimise-parallelism-based-on-input-rate-of-kafka-messages-does-it-depend-on-sink-) + - [What are the retry mechanisms in firehose?](#what-are-the-retry-mechanisms-in-firehose) + - [Which Kafka client configs are available ?](#which-kafka-client-configs-are-available-) + - [What all data formats are supported?](#what-all-data-formats-are-supported) + - [Can we select particular fields from the input message?](#can-we-select-particular-fields-from-the-input-message) + - [How do I configure the Protobuf schema for the topic Firehose needs to consume?](#how-do-i-configure-the-protobuf-schema-for-the-topic-firehose-needs-to-consume) + - [What is Stencil in context of firehose ?](#what-is-stencil-in-context-of-firehose-) + - [Will I have any data loss if my firehose is failed ?](#will-i-have-any-data-loss-if-my-firehose-is-failed-) + - [How does firehose handle failed messages?](#how-does-firehose-handle-failed-messages) + - [What all metrics are produced for me to monitor ?](#what-all-metrics-are-produced-for-me-to-monitor-) + - [What kind of delivery guarantees does Firehose provide? Is it different from what Kafka is tuned for?](#what-kind-of-delivery-guarantees-does-firehose-provide-is-it-different-from-what-kafka-is-tuned-for) + - [What happens if my firehose is stopped and Kafka retention is for few days?](#what-happens-if-my-firehose-is-stopped-and-kafka-retention-is-for-few-days) ## Firehose Sinks @@ -187,15 +187,15 @@ terminated with an abnormal status code. #### How to implement new Object Storage provider, for example S3 ? -Firehose exposes an interface class called *BlobStorage.java* which can be used to create new blob storage -implementations. Subsequently, the *BlobStorageFactory.java* can be modified to produce instance of this new blob +Firehose exposes an interface class called _BlobStorage.java_ which can be used to create new blob storage +implementations. Subsequently, the _BlobStorageFactory.java_ can be modified to produce instance of this new blob storage provider, while injecting the necessary configs. Firehose also exposes a config called as ** SINK_BLOB_STORAGE_TYPE** which is used to control which storage provider to use, with its default currently set to GCS. #### How to implement new file format ? -Firehose exposes an interface called *LocalFileWriter.java* which can be used to create new file writer implementations. -Subsequently, the *LocalStorage.java* class can be modified to produce instance of this new file writer, while injecting +Firehose exposes an interface called _LocalFileWriter.java_ which can be used to create new file writer implementations. +Subsequently, the _LocalStorage.java_ class can be modified to produce instance of this new file writer, while injecting the necessary configs. Firehose also exposes a config called as **SINK_BLOB_LOCAL_FILE_WRITER_TYPE** which is used to control which file writer type to use, with its default currently set to parquet. @@ -289,7 +289,7 @@ to `label1=value1,label2=value2`. #### What is the Redis version supported ? -Firehose uses Jedis v3.0.1 (as of the latest release) as the redis client. At the time of writing this documentation, +Firehose uses Jedis v3.0.1 (as of the latest release) as the redis client. At the time of writing this documentation, Jedis is [fully compatible](https://github.com/redis/jedis#jedis) with redis 2.8.x, 3.x.x and above. #### What Data types are supported in Redis sink? @@ -327,7 +327,7 @@ redis hash set entry. For example, consider a proto as follows:- int32 driver_id = 1; PersonName driver_name = 2; } - + message PersonName { string fname = 1; string lname = 2; @@ -409,7 +409,7 @@ SINK_JDBC_UNIQUE_KEYS**. For example, consider a proto as follows:- int32 driver_id = 1; PersonName driver_name = 2; } - + message PersonName { string fname = 1; string lname = 2; @@ -423,7 +423,7 @@ can set the **INPUT_SCHEMA_PROTO_TO_COLUMN_MAPPING** config as follows:- This will output a row in the Driver table as follows:- | id | first_name | last_name | -|------|------------|-----------| +| ---- | ---------- | --------- | | 1234 | Chuck | Norris | Unique keys as per the table schema can be configured by setting the config **SINK_JDBC_UNIQUE_KEYS**. For @@ -475,7 +475,7 @@ can override the data from other partition. #### How does the payload look like? -The payload format differs based on the configs **SINK_HTTP_DATA_FORMAT**, **SINK_HTTP_JSON_BODY_TEMPLATE**, +The payload format differs based on the configs **SINK_HTTP_DATA_FORMAT**, **SINK_HTTP_JSON_BODY_TEMPLATE**, **SINK_HTTP_PARAMETER_SOURCE**, **SINK_HTTP_SERVICE_URL** and **SINK_HTTP_PARAMETER_PLACEMENT**. For details on what these configs mean, please have a look at the config section. @@ -507,7 +507,7 @@ respective encoded serialised string from the Kafka message: #### Does it support client side load balancing? -Yes. The IP/hostname of the Load Balancer fronting the HTTP Sink Servers can be provided to Firehose via +Yes. The IP/hostname of the Load Balancer fronting the HTTP Sink Servers can be provided to Firehose via **SINK_HTTP_SERVICE_URL**. Firehose will call this endpoint when pushing messages and then the LB can take care of distributing the load among multiple sink servers. @@ -517,26 +517,26 @@ Firehose supports OAuth authentication for the HTTP Sink. #### How can I pass a particular input field as a header in request? -1. set **SINK_HTTP_PARAMETER_SOURCE** to either `key` or `message` -(based on whether the field one requires in the http request is part of the key or the message in the Kafka record) +1. set **SINK_HTTP_PARAMETER_SOURCE** to either `key` or `message` + (based on whether the field one requires in the http request is part of the key or the message in the Kafka record) 2. set **SINK_HTTP_PARAMETER_PLACEMENT** to `header`, and -3. set **SINK_HTTP_PARAMETER_SCHEMA_PROTO_CLASS** to the proto class for the input Kafka message and set -**INPUT_SCHEMA_PROTO_TO_COLUMN_MAPPING** to a json value indicating the proto field number from the input message to be -mapped to the header name. +3. set **SINK_HTTP_PARAMETER_SCHEMA_PROTO_CLASS** to the proto class for the input Kafka message and set + **INPUT_SCHEMA_PROTO_TO_COLUMN_MAPPING** to a json value indicating the proto field number from the input message to be + mapped to the header name. #### How can I pass a particular input field as a query param in request? -1. set **SINK_HTTP_PARAMETER_SOURCE** to either `key` or `message` -(based on whether the field one requires in the http request is part of the key or the message in the Kafka record), +1. set **SINK_HTTP_PARAMETER_SOURCE** to either `key` or `message` + (based on whether the field one requires in the http request is part of the key or the message in the Kafka record), 2. set **SINK_HTTP_PARAMETER_PLACEMENT** to `query`, and -3. set **SINK_HTTP_PARAMETER_SCHEMA_PROTO_CLASS** to the proto class for the input Kafka message and set -**INPUT_SCHEMA_PROTO_TO_COLUMN_MAPPING** to a json value indicating the proto field number from the input message to be -mapped to the query parameter name. +3. set **SINK_HTTP_PARAMETER_SCHEMA_PROTO_CLASS** to the proto class for the input Kafka message and set + **INPUT_SCHEMA_PROTO_TO_COLUMN_MAPPING** to a json value indicating the proto field number from the input message to be + mapped to the query parameter name. #### What happens if my services fails ? -If messages failed to get pushed to the HTTP endpoint with a retryable status code as specified via -**SINK_HTTP_RETRY_STATUS_CODE_RANGES** config, Firehose will retry sending the messages for a fixed number of attempts. +If messages failed to get pushed to the HTTP endpoint with a retryable status code as specified via +**SINK_HTTP_RETRY_STATUS_CODE_RANGES** config, Firehose will retry sending the messages for a fixed number of attempts. If the messages still failed to get published after retries, Firehose will push these messages to the DLQ topic if DLQ is enabled via configs. If not enabled, it will drop the messages. @@ -583,12 +583,12 @@ Kafka message is parsed into the request body. #### For parameterised header, how is the data added? -1. set **SINK_HTTP_PARAMETER_SOURCE** to either `key` or `message` (based on whether the field one requires in the http -request is part of the key or the message in the Kafka record), +1. set **SINK_HTTP_PARAMETER_SOURCE** to either `key` or `message` (based on whether the field one requires in the http + request is part of the key or the message in the Kafka record), 2. set **SINK_HTTP_PARAMETER_PLACEMENT** to `header`, and -3. set **SINK_HTTP_PARAMETER_SCHEMA_PROTO_CLASS** to the Protobuf class for the input Kafka message and -**INPUT_SCHEMA_PROTO_TO_COLUMN_MAPPING** to a json value indicating the proto field number from the input message to be -mapped to the header name. +3. set **SINK_HTTP_PARAMETER_SCHEMA_PROTO_CLASS** to the Protobuf class for the input Kafka message and + **INPUT_SCHEMA_PROTO_TO_COLUMN_MAPPING** to a json value indicating the proto field number from the input message to be + mapped to the header name. ### General Queries @@ -600,28 +600,29 @@ hence, needs to be tested prior to use. #### How does the execution work? Is the entire process sync or async ? The execution works as follows: -* Get messages from Kafka -* Filter the messages (optional) -* Push these messages to sink: All the existing sink types follow the same contract/lifecycle defined + +- Get messages from Kafka +- Filter the messages (optional) +- Push these messages to sink: All the existing sink types follow the same contract/lifecycle defined in `AbstractSink.java`. It consists of two stages: - * **Prepare**: Transformation over-filtered messages’ list to prepare the sink-specific insert/update client - requests. - * **Execute**: Requests created in the prepare stage are executed at this step and a list of failed - messages is returned (if any) for retry. -* If the push to the sink fails with a retryable exception, Firehose will attempt to retry pushing the messages for a + - **Prepare**: Transformation over-filtered messages’ list to prepare the sink-specific insert/update client + requests. + - **Execute**: Requests created in the prepare stage are executed at this step and a list of failed + messages is returned (if any) for retry. +- If the push to the sink fails with a retryable exception, Firehose will attempt to retry pushing the messages for a configured number of attempts with backoff. After that, if DLQ is enabled, the messages are pushed to DLQ queue with backoff. If DLQ is disabled, messages are dropped. -* Captures telemetry and success/failure events and send them to Telegraf -* Repeat the process above again +- Captures telemetry and success/failure events and send them to Telegraf +- Repeat the process above again -Firehose can configure its Kafka consumer to work in either sync or async mode. For more details, you can look +Firehose can configure its Kafka consumer to work in either sync or async mode. For more details, you can look [here](/docs/concepts/consumer.md) #### Is there any code snippet which shows how I can produce sample message in supported data format? Following is an example to demonstrate how to create a Protobuf message and then produce it to a Kafka cluster. Firstly, -create a `.proto` file containing all the required field names and their corresponding integer tags. Save it in a new -file named `person.proto` +create a `.proto` file containing all the required field names and their corresponding integer tags. Save it in a new +file named `person.proto` ``` syntax = "proto2"; @@ -653,10 +654,10 @@ message Person { ``` -Next, compile your `.proto` file using Protobuf compiler i.e. `protoc`.This will generate Person, PersonOrBuilder and +Next, compile your `.proto` file using Protobuf compiler i.e. `protoc`.This will generate Person, PersonOrBuilder and PersonProtos Java source files. Specify the source directory (where your application's source code lives – the current directory is used if you don't provide a value), the destination directory (where you want the generated code to go; -often the same as `$SRC_DIR`), and the path to your `.proto` +often the same as `$SRC_DIR`), and the path to your `.proto` ``` protoc -I=$SRC_DIR --java_out=$DST_DIR $SRC_DIR/person.proto @@ -664,12 +665,12 @@ protoc -I=$SRC_DIR --java_out=$DST_DIR $SRC_DIR/person.proto ``` Lastly, add the following lines in your Java code to generate a POJO (Plain Old Java Object) of the Person proto class -and serialize it to a byte array, using the `toByteArray()` method of -the [com.google.protobuf.GeneratedMessageV3](https://www.javadoc.io/static/com.google.protobuf/protobuf-java/3.5.1/com/google/protobuf/GeneratedMessageV3.html) +and serialize it to a byte array, using the `toByteArray()` method of +the [com.google.protobuf.GeneratedMessageV3](https://www.javadoc.io/static/com.google.protobuf/protobuf-java/3.5.1/com/google/protobuf/GeneratedMessageV3.html) class. The byte array is then sent to the Kafka cluster by the producer. KafkaProducer producer = new KafkaProducer<>(properties); - + Person john = Person.newBuilder() .setId(87182872) .setName("John Doe") @@ -679,10 +680,10 @@ class. The byte array is then sent to the Kafka cluster by the producer. .setNumber("555-4321") .setType(Person.PhoneType.HOME)) .build(); - + producer.send(new ProducerRecord(topicName, john.toByteArray())); -Refer [https://developers.google.com/protocol-buffers](https://developers.google.com/protocol-buffers) for more info +Refer [https://developers.google.com/protocol-buffers](https://developers.google.com/protocol-buffers) for more info on how to create protobuf files. #### Why Protobuf ? Can it support other formats like JSON, AVRO, Thrift ? @@ -698,11 +699,11 @@ supports only Protobuf. Support for JSON and Avro is planned and incorporated in No, the Kafka consumer used in Firehose doesn't support SSL. -#### How does Firehose create the Kafka consumer? +#### How does Firehose create the Kafka consumer? When Firehose starts, it creates as many Kafka consumers as specified by the config **APPLICATION_THREAD_COUNT** ( default set to 1). Each consumer runs on a separate thread. Please -look [here](https://github.com/odpf/firehose/blob/main/docs/reference/configuration.md#kafka-consumer) for more details +look [here](https://github.com/raystack/firehose/blob/main/docs/reference/configuration.md#kafka-consumer) for more details on how to configure the Kafka consumer. #### Can I change the consumer group name or reset it ? Overall, how do I handle my consumer group operations? @@ -710,22 +711,22 @@ on how to configure the Kafka consumer. Yes, the Kafka consumer group name can be changed by specifying the following configuration **SOURCE_KAFKA_CONSUMER_GROUP_ID**. The Kafka consumer in Firehose can also be reset. However, the default behaviour is currently set to read from the latest offset. To know further details on how to tune the Kafka consumer group, you can have a look at the Kafka consumer configurations already exposed by -Firehose [here](https://github.com/odpf/firehose/blob/main/docs/reference/configuration/kafka-consumer-1.md#kafka-consumer). +Firehose [here](https://github.com/raystack/firehose/blob/main/docs/reference/configuration/kafka-consumer-1.md#kafka-consumer). #### What are the advantages of Firehose over Kafka connect ? -- **Ease of use:** Firehose is easier to install, and using different sinks only requires changing a few +- **Ease of use:** Firehose is easier to install, and using different sinks only requires changing a few configurations. However, Kafka Connect requires connectors to be installed across all the worker nodes within the cluster when used in distributed mode. -- **Filtering:** Value-based filtering is much easier to implement as compared to Kafka Connect. Requires no additional +- **Filtering:** Value-based filtering is much easier to implement as compared to Kafka Connect. Requires no additional plugins/schema-registry to be installed. -- **Extensible:** Provides a comprehensible abstract sink contract making it easier to add a new sink in Firehose. +- **Extensible:** Provides a comprehensible abstract sink contract making it easier to add a new sink in Firehose. Firehose also comes with an inbuilt serialisation/deserialisation and doesn't require any converters and serialisers when implementing a new sink. -- **Easy monitoring:** Firehose provides a detailed health dashboard (Grafana) for effortless monitoring. -- **Connectors:** Some of the Kafka connect available connectors usually have limitations. It's usually rare to find +- **Easy monitoring:** Firehose provides a detailed health dashboard (Grafana) for effortless monitoring. +- **Connectors:** Some of the Kafka connect available connectors usually have limitations. It's usually rare to find all the required features in a single connector and so is to find documentation for the same. -- **Fully open-source:** Firehose is completely open-source while separation of commercial and open-source features is +- **Fully open-source:** Firehose is completely open-source while separation of commercial and open-source features is not very structured in Kafka Connect and for monitoring and advanced features, confluent control center requires an enterprise subscription. @@ -742,14 +743,14 @@ requires no ongoing administration. #### Can I do any transformations before sending data to sink, for example filtering ? Yes, Firehose provides JEXL based filters based on the fields in key or message of the Kafka record. Read -the [Filters](https://github.com/odpf/firehose/blob/main/docs/concepts/filters.md) section for further details. +the [Filters](https://github.com/raystack/firehose/blob/main/docs/concepts/filters.md) section for further details. #### How to optimise parallelism based on input rate of Kafka messages? Does it depend on sink ? You can increase the workers in the Firehose which will effectively multiply the number of records being processed by -Firehose. Adding some sort of filter condition in the Firehose to ignore unnecessary messages in the topic would help -you bring down the volume of data being processed by the sink.Firehose can also be configured for its Kafka consumer to -work in [async mode](/docs/concepts/consumer.md), thereby allowing it to do offset management and commit asynchronously +Firehose. Adding some sort of filter condition in the Firehose to ignore unnecessary messages in the topic would help +you bring down the volume of data being processed by the sink.Firehose can also be configured for its Kafka consumer to +work in [async mode](/docs/concepts/consumer.md), thereby allowing it to do offset management and commit asynchronously improving performance. #### What are the retry mechanisms in firehose? @@ -761,12 +762,12 @@ messages are pushed to DLQ queue with backoff. If push fails and DLQ is disabled Starting with version 0.2, Firehose also provides the ability to tag different error types under a specific scope: `DLQ` , `RETRY` or `FAIL`. This enables Firehose to effectively determine at runtime as to what should be the strategy when a particular error is encountered. For more details, please look at `ErrorConfig.java` class and classes in -the `io.odpf.firehose.error` package. +the `come.raystack.firehose.error` package. #### Which Kafka client configs are available ? Firehose provides various Kafka client configurations. -Refer [Kafka Consumer Configurations](https://github.com/odpf/firehose/blob/main/docs/reference/configuration/kafka-consumer-1.md) +Refer [Kafka Consumer Configurations](https://github.com/raystack/firehose/blob/main/docs/reference/configuration/kafka-consumer-1.md) section. #### What all data formats are supported? @@ -776,10 +777,10 @@ only Protobuf. Support for JSON and Avro is planned and incorporated in the road Protocol buffers are Google's language-neutral, platform-neutral, extensible mechanism for serialising structured data. When `INPUT_SCHEMA_DATA_TYPE=protobuf` Data streams on Kafka topics are bound to a Protobuf schema. Follow the instructions -in [this article](https://developers.google.com/protocol-buffers/docs/javatutorial) on how to create, compile and +in [this article](https://developers.google.com/protocol-buffers/docs/javatutorial) on how to create, compile and serialize a Protobuf object to send it to a binary OutputStream. -Refer [this guide](https://developers.google.com/protocol-buffers/docs/proto3) for detailed Protobuf syntax and rules -to create a `.proto` file. +Refer [this guide](https://developers.google.com/protocol-buffers/docs/proto3) for detailed Protobuf syntax and rules +to create a `.proto` file. When `INPUT_SCHEMA_DATA_TYPE=json` data streams on kafka topics are bound to having a valid json message. #### Can we select particular fields from the input message? @@ -791,28 +792,28 @@ destination/ database to consume only the required fields. Generated Protobuf Descriptors are hosted behind a Stencil server artifactory/HTTP endpoint. This endpoint URL and the ProtoDescriptor class that the Firehose deployment should use to deserialise raw data with is configured in Firehose in -the environment variables **SCHEMA_REGISTRY_STENCIL_URLS** and **INPUT_SCHEMA_PROTO_CLASS** respectively. +the environment variables **SCHEMA_REGISTRY_STENCIL_URLS** and **INPUT_SCHEMA_PROTO_CLASS** respectively. The Proto Descriptor set of the Kafka messages must be uploaded to the Stencil server. -Refer [this guide](https://github.com/odpf/stencil/blob/master/docs/guides/quick_start.md) on how to setup and +Refer [this guide](https://github.com/raystack/stencil/blob/master/docs/guides/quick_start.md) on how to setup and configure the Stencil server. #### What is Stencil in context of firehose ? -ODPF Stencil API is a dynamic schema registry for hosting and managing versions of Protobuf descriptors. The schema +Stencil API is a dynamic schema registry for hosting and managing versions of Protobuf descriptors. The schema handling i.e., find the mapped schema for the topic, downloading the descriptors, and dynamically being notified of/updating with the latest schema is abstracted through the Stencil library. The Stencil Client is a proprietary library that provides an abstraction layer, for schema handling. Schema caching, dynamic schema updates are features of the stencil client library. -Refer [this article](https://odpf.gitbook.io/stencil/) for further information of the features, configuration and +Refer [this article](https://raystack.gitbook.io/stencil/) for further information of the features, configuration and deployment instructions of the Stencil API. Source code of Stencil Server and Client API can be found in -its [Github repository](https://github.com/odpf/stencil). +its [Github repository](https://github.com/raystack/stencil). #### Will I have any data loss if my firehose is failed ? -Firehose follows an *at-least* once policy. After a batch of messages is pushed to the sink successfully, Firehose +Firehose follows an _at-least_ once policy. After a batch of messages is pushed to the sink successfully, Firehose commits the offset before the consumer polls another batch from Kafka. Thus, failed messages are not committed. So, when Firehose is restarted, the Kafka Consumer automatically starts pulling messages from the last committed offset @@ -827,12 +828,12 @@ messages are pushed to DLQ queue with backoff. If push fails and DLQ is disabled Starting with version 0.2, Firehose also provides the ability to tag different error types under a specific scope: `DLQ` , `RETRY` or `FAIL`. This enables Firehose to effectively determine at runtime as to what should be the strategy when a particular error is encountered. For more details, please look at `ErrorConfig.java` class and classes in -the `io.odpf.firehose.error` package. +the `org.raystack.firehose.error` package. #### What all metrics are produced for me to monitor ? Firehose exposes critical metrics to monitor the health of your delivery streams and take any necessary actions. Refer -the [Metrics](https://github.com/odpf/firehose/blob/main/docs/reference/metrics.md) section for further details on each +the [Metrics](https://github.com/raystack/firehose/blob/main/docs/reference/metrics.md) section for further details on each metric. #### What kind of delivery guarantees does Firehose provide? Is it different from what Kafka is tuned for? @@ -847,7 +848,7 @@ messages are pushed to DLQ queue with backoff. If push fails and DLQ is disabled Starting with version 0.2, Firehose also provides the ability to tag different error types under a specific scope: `DLQ` , `RETRY` or `FAIL`. This enables Firehose to effectively determine at runtime as to what should be the strategy when a particular error is encountered. For more details, please look at `ErrorConfig.java` class and classes in -the `io.odpf.firehose.error` package. +the `org.raystack.firehose.error` package. #### What happens if my firehose is stopped and Kafka retention is for few days? diff --git a/docs/docs/reference/metrics.md b/docs/docs/reference/metrics.md index 4ca5c97b0..5f8ffbff1 100644 --- a/docs/docs/reference/metrics.md +++ b/docs/docs/reference/metrics.md @@ -4,19 +4,19 @@ Service-level Indicators \(SLIs\) are the measurements used to calculate the per ## Table of Contents -* [Type Details](metrics.md#type-details) -* [Overview](metrics.md#overview) -* [Pods Health](metrics.md#pods-health) -* [Kafka Consumer Details](metrics.md#kafka-consumer-details) -* [Error](metrics.md#error) -* [Memory](metrics.md#memory) -* [Error](metrics.md#error) -* [Garbage Collection](metrics.md#garbage-collection) -* [Retry](metrics.md#retry) -* [HTTP Sink](metrics.md#http-sink) -* [Filter](metrics.md#filter) -* [Blob Sink](metrics.md#blob-sink) -* [Bigquery Sink](https://github.com/odpf/depot/blob/main/docs/reference/metrics.md#bigquery-sink) +- [Type Details](metrics.md#type-details) +- [Overview](metrics.md#overview) +- [Pods Health](metrics.md#pods-health) +- [Kafka Consumer Details](metrics.md#kafka-consumer-details) +- [Error](metrics.md#error) +- [Memory](metrics.md#memory) +- [Error](metrics.md#error) +- [Garbage Collection](metrics.md#garbage-collection) +- [Retry](metrics.md#retry) +- [HTTP Sink](metrics.md#http-sink) +- [Filter](metrics.md#filter) +- [Blob Sink](metrics.md#blob-sink) +- [Bigquery Sink](https://github.com/raystack/depot/blob/main/docs/reference/metrics.md#bigquery-sink) ## Type Details @@ -24,19 +24,19 @@ Collection of all the generic configurations in a Firehose. ### `Sink` -* The type of sink of the Firehose. It could be 'log', 'HTTP', 'DB', 'redis', 'influx' or 'Elasticsearch' +- The type of sink of the Firehose. It could be 'log', 'HTTP', 'DB', 'redis', 'influx' or 'Elasticsearch' ### `Team` -* Team who has the ownership for the given Firehose. +- Team who has the ownership for the given Firehose. ### `Proto Schema` -* The proto class used for creating the Firehose +- The proto class used for creating the Firehose ### `Stream` -* The stream where the input topic is read from +- The stream where the input topic is read from ## Overview @@ -44,39 +44,39 @@ Some of the most important metrics related to Firehose that gives you an overvie ### `ConsumerLag: MaxLag` -* The maximum lag in terms of number of records for any partition in this window. An increasing value over time is your best indication that the consumer group is not keeping up with the producers. +- The maximum lag in terms of number of records for any partition in this window. An increasing value over time is your best indication that the consumer group is not keeping up with the producers. ### `Total Message Received` -* Sum of all messages received from Kafka per pod. +- Sum of all messages received from Kafka per pod. ### `Message Sent Successfully` -* Messages sent successfully to the sink per batch per pod. +- Messages sent successfully to the sink per batch per pod. ### `Message Sent Failed` -* Messages failed to be pushed into the sink per batch per pod. In case of HTTP sink, if status code is not in retry codes configured, the records will be dropped. +- Messages failed to be pushed into the sink per batch per pod. In case of HTTP sink, if status code is not in retry codes configured, the records will be dropped. ### `Message Dropped` -* In case of HTTP sink, when status code is not in retry codes configured, the records are dropped. This metric captures the dropped messages count. +- In case of HTTP sink, when status code is not in retry codes configured, the records are dropped. This metric captures the dropped messages count. ### `Batch size Distribution` -* 99p of batch size distribution for pulled and pushed messages per pod. +- 99p of batch size distribution for pulled and pushed messages per pod. ### `Time spent in Firehose` -* Latency introduced by Firehose \(time before sending to sink - time when reading from Kafka\). Note: It could be high if the response time of the sink is higher as subsequent batches could be delayed. +- Latency introduced by Firehose \(time before sending to sink - time when reading from Kafka\). Note: It could be high if the response time of the sink is higher as subsequent batches could be delayed. ### `Time spent in pipeline` -* Time difference between Kafka ingestion and sending to sink \(Time before sending to sink - Time of Kafka ingestion\) +- Time difference between Kafka ingestion and sending to sink \(Time before sending to sink - Time of Kafka ingestion\) ### `Sink Response Time` -* Different percentile of the response time of the sink. +- Different percentile of the response time of the sink. ## Pods Health @@ -84,15 +84,15 @@ Since Firehose runs on Kube, this gives a nice health details of each pods. ### `JVM Lifetime` -* JVM Uptime of each pod. +- JVM Uptime of each pod. ### `Cpu Load` -* Returns the "recent cpu usage" for the Java Virtual Machine process. This value is a double in the \[0.0,1.0\] interval. A value of 0.0 means that none of the CPUs were running threads from the JVM process during the recent period of time observed, while a value of 1.0 means that all CPUs were actively running threads from the JVM 100% of the time during the recent period being observed. Threads from the JVM include the application threads as well as the JVM internal threads. All values betweens 0.0 and 1.0 are possible depending of the activities going on in the JVM process and the whole system. If the Java Virtual Machine recent CPU usage is not available, the method returns a negative value. +- Returns the "recent cpu usage" for the Java Virtual Machine process. This value is a double in the \[0.0,1.0\] interval. A value of 0.0 means that none of the CPUs were running threads from the JVM process during the recent period of time observed, while a value of 1.0 means that all CPUs were actively running threads from the JVM 100% of the time during the recent period being observed. Threads from the JVM include the application threads as well as the JVM internal threads. All values betweens 0.0 and 1.0 are possible depending of the activities going on in the JVM process and the whole system. If the Java Virtual Machine recent CPU usage is not available, the method returns a negative value. ### `Cpu Time` -* Returns the CPU time used by the process on which the Java virtual machine is running. The returned value is of nanoseconds precision but not necessarily nanoseconds accuracy. +- Returns the CPU time used by the process on which the Java virtual machine is running. The returned value is of nanoseconds precision but not necessarily nanoseconds accuracy. ## Kafka Consumer Details @@ -100,95 +100,95 @@ Listing some of the Kafka consumer metrics here. ### `Assigned partitions` -* Consumer Group Metrics: The number of partitions currently assigned to this consumer \(per pod\). +- Consumer Group Metrics: The number of partitions currently assigned to this consumer \(per pod\). ### `Consumer Number of Request/second` -* Global Request Metrics: The average number of requests sent per second per pod. +- Global Request Metrics: The average number of requests sent per second per pod. ### `Records Consumed Rate/second` -* Topic-level Fetch Metrics: The average number of records consumed per second for a specific topic per pod. +- Topic-level Fetch Metrics: The average number of records consumed per second for a specific topic per pod. ### `Bytes Consumed Rate/second` -* Topic-level Fetch Metrics: The average number of bytes consumed per second per pod. +- Topic-level Fetch Metrics: The average number of bytes consumed per second per pod. ### `Fetch Rate/second` -* Fetch Metrics: The number of fetch requests per second per pod. +- Fetch Metrics: The number of fetch requests per second per pod. ### `Max Fetch Latency` -* Fetch Metrics: The max time taken for a fetch request per pod. +- Fetch Metrics: The max time taken for a fetch request per pod. ### `Average Fetch Latency` -* Fetch Metrics: The average time taken for a fetch request per pod. +- Fetch Metrics: The average time taken for a fetch request per pod. ### `Average Fetch Size` -* Fetch Metrics: The average number of bytes fetched per request per pod. +- Fetch Metrics: The average number of bytes fetched per request per pod. ### `Max Fetch Size` -* Fetch Metrics: The max number of bytes fetched per request per pod. +- Fetch Metrics: The max number of bytes fetched per request per pod. ### `Commit Rate/second` -* Consumer Group Metrics: The number of commit calls per second per pod. +- Consumer Group Metrics: The number of commit calls per second per pod. ### `Consumer Active Connections Count` -* Global Connection Metrics: The current number of active connections per pod. +- Global Connection Metrics: The current number of active connections per pod. ### `New Connections Creation Rate/second` -* Global Connection Metrics: New connections established per second in the window per pod. +- Global Connection Metrics: New connections established per second in the window per pod. ### `Connections Close Rate/second` -* Global Connection Metrics: Connections closed per second in the window per pod. +- Global Connection Metrics: Connections closed per second in the window per pod. ### `Consumer Outgoing Byte Rate/Sec` -* Global Request Metrics: The average number of outgoing bytes sent per second to all servers per pod. +- Global Request Metrics: The average number of outgoing bytes sent per second to all servers per pod. ### `Avg time between poll` -* Average time spent between poll per pod. +- Average time spent between poll per pod. ### `Max time between poll` -* Max time spent between poll per pod. +- Max time spent between poll per pod. ### `Sync rate` -* Consumer Group Metrics: The number of group syncs per second per pod. Group synchronization is the second and last phase of the rebalance protocol. Similar to join-rate, a large value indicates group instability. +- Consumer Group Metrics: The number of group syncs per second per pod. Group synchronization is the second and last phase of the rebalance protocol. Similar to join-rate, a large value indicates group instability. ### `Consumer Network IO rate /second` -* The average number of network operations \(reads or writes\) on all connections per second per pod +- The average number of network operations \(reads or writes\) on all connections per second per pod ### `Rebalance Rate /hour` -* Rate of rebalance the consumer. +- Rate of rebalance the consumer. ### `Average Commit latency` -* Consumer Group Metrics: The average time taken for a commit request per pod +- Consumer Group Metrics: The average time taken for a commit request per pod ### `Max Commit latency` -* Consumer Group Metrics: The max time taken for a commit request per pod. +- Consumer Group Metrics: The max time taken for a commit request per pod. ### `Avg Rebalance latency` -* Average Rebalance Latency for the consumer per pod. +- Average Rebalance Latency for the consumer per pod. ### `Max Rebalance latency` -* Max Rebalance Latency for the consumer per pod. +- Max Rebalance Latency for the consumer per pod. ## Error @@ -196,11 +196,11 @@ This gives you a nice insight about the critical and noncritical exceptions happ ### `Fatal Error` -* Count of all the exception raised by the pods which can restart the Firehose. +- Count of all the exception raised by the pods which can restart the Firehose. ### `Nonfatal Error` -* Count of all the exception raised by the Firehose which will not restart the Firehose and Firehose will keep retrying. +- Count of all the exception raised by the Firehose which will not restart the Firehose and Firehose will keep retrying. ## Memory @@ -208,7 +208,7 @@ Details on memory used by the Firehose for different tasks. ### `Heap Memory Usage` -* Details of heap memory usage: +- Details of heap memory usage: ```text Max: The amount of memory that can be used for memory management @@ -217,7 +217,7 @@ Details on memory used by the Firehose for different tasks. ### `Non-Heap Memory Usage` -* Details of non-heap memory usage: +- Details of non-heap memory usage: ```text Max: The amount of memory that can be used for memory management @@ -226,15 +226,15 @@ Details on memory used by the Firehose for different tasks. ### `GC: Memory Pool Collection Usage` -* For a garbage-collected memory pool, the amount of used memory includes the memory occupied by all objects in the pool including both reachable and unreachable objects. This is for all the names in the type: MemoryPool. +- For a garbage-collected memory pool, the amount of used memory includes the memory occupied by all objects in the pool including both reachable and unreachable objects. This is for all the names in the type: MemoryPool. ### `GC: Memory Pool Peak Usage` -* Peak usage of GC memory usage. +- Peak usage of GC memory usage. ### `GC: Memory Pool Usage` -* Total usage of GC memory usage. +- Total usage of GC memory usage. ## Garbage Collection @@ -242,47 +242,47 @@ All JVM Garbage Collection Details. ### `GC Collection Count` -* The total number of collections that have occurred per pod. Rather than showing the absolute value we are showing the difference to see the rate of change more easily. +- The total number of collections that have occurred per pod. Rather than showing the absolute value we are showing the difference to see the rate of change more easily. ### `GC Collection Time` -* The approximate accumulated collection elapsed time in milliseconds per pod. Rather than showing the absolute value we are showing the difference to see the rate of change more easily. +- The approximate accumulated collection elapsed time in milliseconds per pod. Rather than showing the absolute value we are showing the difference to see the rate of change more easily. ### `Thread Count` -* daemonThreadCount: Returns the current number of live daemon threads per pod peakThreadCount: Returns the peak live thread count since the Java virtual machine started or peak was reset per pod threadCount: Returns the current number of live threads including both daemon and non-daemon threads per pod. +- daemonThreadCount: Returns the current number of live daemon threads per pod peakThreadCount: Returns the peak live thread count since the Java virtual machine started or peak was reset per pod threadCount: Returns the current number of live threads including both daemon and non-daemon threads per pod. ### `Class Count` -* loadedClass: Displays number of classes that are currently loaded in the Java virtual machine per pod unloadedClass: Displays the total number of classes unloaded since the Java virtual machine has started execution. +- loadedClass: Displays number of classes that are currently loaded in the Java virtual machine per pod unloadedClass: Displays the total number of classes unloaded since the Java virtual machine has started execution. ### `Code Cache Memory after GC` -* The code cache memory usage in the memory pools at the end of a GC per pod. +- The code cache memory usage in the memory pools at the end of a GC per pod. ### `Compressed Class Space after GC` -* The compressed class space memory usage in the memory pools at the end of a GC per pod. +- The compressed class space memory usage in the memory pools at the end of a GC per pod. ### `Metaspace after GC` -* The metaspace memory usage in the memory pools at the end of a GC per pod. +- The metaspace memory usage in the memory pools at the end of a GC per pod. ### `Par Eden Space after GC` -* The eden space memory usage in the memory pools at the end of a GC per pod. +- The eden space memory usage in the memory pools at the end of a GC per pod. ### `Par Survivor Space after GC` -* The survivor space memory usage in the memory pools at the end of a GC per pod. +- The survivor space memory usage in the memory pools at the end of a GC per pod. ### `Tenured Space after GC` -* The tenured space memory usage in the memory pools at the end of a GC per pod. +- The tenured space memory usage in the memory pools at the end of a GC per pod. **`File Descriptor`** -* Number of file descriptor per pod +- Number of file descriptor per pod ```text Open: Current open file descriptors @@ -295,11 +295,11 @@ If you have configured retries this will give you some insight about the retries ### `Average Retry Requests` -* Request retries per min per pod. +- Request retries per min per pod. ### `Back Off time` -* Time spent per pod backing off. +- Time spent per pod backing off. ## HTTP Sink @@ -307,19 +307,19 @@ HTTP Sink response code details. ### `2XX Response Count` -* Total number of 2xx response received by Firehose from the HTTP service, +- Total number of 2xx response received by Firehose from the HTTP service, ### `4XX Response Count` -* Total number of 4xx response received by Firehose from the HTTP service. +- Total number of 4xx response received by Firehose from the HTTP service. ### `5XX Response Count` -* Total number of 5xx response received by Firehose from the HTTP service. +- Total number of 5xx response received by Firehose from the HTTP service. ### `No Response Count` -* Total number of No response received by Firehose from the HTTP service. +- Total number of No response received by Firehose from the HTTP service. ## Filter @@ -327,15 +327,16 @@ Since Firehose supports filtration based on some data, these metrics give some i ### `Filter Type` -* Type of filter in the Firehose. It will be one of the "none", "key", "message". +- Type of filter in the Firehose. It will be one of the "none", "key", "message". ### `Total Messages filtered` -* Sum of all the messages filtered because of the filter condition per pod. +- Sum of all the messages filtered because of the filter condition per pod. ## Blob Sink ### `Local File Open Total` + A gauge, total number of local file that is currently being opened. ### `Local File Closed Total` diff --git a/docs/docs/roadmap.md b/docs/docs/roadmap.md index e815c29e4..d57d76c08 100644 --- a/docs/docs/roadmap.md +++ b/docs/docs/roadmap.md @@ -1,8 +1,8 @@ # Roadmap -In the following section, you can learn about what features we're working on, what stage they're in, and when we expect to bring them to you. Have any questions or comments about items on the roadmap? Join the [discussions](https://github.com/orgs/odpf/discussions) on the Firehose Github forum. +In the following section, you can learn about what features we're working on, what stage they're in, and when we expect to bring them to you. Have any questions or comments about items on the roadmap? Join the [discussions](https://github.com/orgs/raystack/discussions) on the Firehose Github forum. -We’re planning to iterate on the format of the roadmap itself, and we see the potential to engage more in discussions about the future of Firehose features. If you have feedback about the roadmap section itself, such as how the issues are presented, let us know through [discussions](https://github.com/odpf/discussions). +We’re planning to iterate on the format of the roadmap itself, and we see the potential to engage more in discussions about the future of Firehose features. If you have feedback about the roadmap section itself, such as how the issues are presented, let us know through [discussions](https://github.com/raystack/discussions). ## Firehose 0.4 diff --git a/docs/docs/sinks/bigquery-sink.md b/docs/docs/sinks/bigquery-sink.md index 92db401c2..3ea8dfa70 100644 --- a/docs/docs/sinks/bigquery-sink.md +++ b/docs/docs/sinks/bigquery-sink.md @@ -1,7 +1,7 @@ # BigQuery Bigquery Sink has several responsibilities, first creation of bigquery table and dataset when they are not exist, second update the bigquery table schema based on the latest schema defined in stencil or infer from incoming data, third translate incoming messages into bigquery records and insert them to bigquery tables. -Bigquery utilise Bigquery [Streaming API](https://cloud.google.com/bigquery/streaming-data-into-bigquery) to insert record into bigquery tables. For more info on the sink refer to [Depot Bigquery sink documentation](https://github.com/odpf/depot/blob/main/docs/sinks/bigquery.md) +Bigquery utilise Bigquery [Streaming API](https://cloud.google.com/bigquery/streaming-data-into-bigquery) to insert record into bigquery tables. For more info on the sink refer to [Depot Bigquery sink documentation](https://github.com/raystack/depot/blob/main/docs/sinks/bigquery.md) ## Asynchronous consumer mode @@ -18,35 +18,40 @@ Firehose collect all the offset sort them and only commit the latest continuous This will ensure all the offset being committed after messages successfully processed even when some messages are being re processed by retry handler or when the insert operation took a long time. ## Configurations -For Bigquery sink in Firehose we need to set first \(`SINK_TYPE`=`bigquery`\). There are some generic configs which are common across different sink types which need to be set example: kafka consumer configs, the generic ones are mentioned in [generic.md](../advance/generic.md). Bigquery sink specific configs are mentioned in depot [Depot-configuration/bigquery-sink.md section](https://github.com/odpf/depot/blob/main/docs/reference/configuration/bigquery-sink.md) +For Bigquery sink in Firehose we need to set first \(`SINK_TYPE`=`bigquery`\). There are some generic configs which are common across different sink types which need to be set example: kafka consumer configs, the generic ones are mentioned in [generic.md](../advance/generic.md). Bigquery sink specific configs are mentioned in depot [Depot-configuration/bigquery-sink.md section](https://github.com/raystack/depot/blob/main/docs/reference/configuration/bigquery-sink.md) ## Bigquery table schema update -Refer to [Depot-bigquery.md#bigquery-table-schema-update section](https://github.com/odpf/depot/blob/main/docs/sinks/bigquery.md#bigquery-table-schema-update) + +Refer to [Depot-bigquery.md#bigquery-table-schema-update section](https://github.com/raystack/depot/blob/main/docs/sinks/bigquery.md#bigquery-table-schema-update) ## Protobuf and BigQuery table type mapping + For type conversion between protobuf to bigquery type. Please refer to -[Depot-bigquery.md#protobuf-bigquery-table-type-mapping section](https://github.com/odpf/depot/blob/main/docs/sinks/bigquery.md#protobuf---bigquery-table-type-mapping) +[Depot-bigquery.md#protobuf-bigquery-table-type-mapping section](https://github.com/raystack/depot/blob/main/docs/sinks/bigquery.md#protobuf---bigquery-table-type-mapping) ## Partitioning + Bigquery Sink supports creation of table with partition configuration. -For more information refer to [Depot-bigquery.md#partitioning section](https://github.com/odpf/depot/blob/main/docs/sinks/bigquery.md#partitioning) +For more information refer to [Depot-bigquery.md#partitioning section](https://github.com/raystack/depot/blob/main/docs/sinks/bigquery.md#partitioning) ## Clustering + Bigquery Sink supports for creating and modifying clustered or unclustered table with clustering configuration. -For more information refer to [Depot-bigquery.md#clustering section](https://github.com/odpf/depot/blob/main/docs/sinks/bigquery.md#clustering) +For more information refer to [Depot-bigquery.md#clustering section](https://github.com/raystack/depot/blob/main/docs/sinks/bigquery.md#clustering) ## Kafka Metadata -For data quality checking purpose sometimes kafka metadata need to be added on the record. For more information refer to [Depot-bigquery.md#metadata sectionn](https://github.com/odpf/depot/blob/main/docs/sinks/bigquery.md#metadata) + +For data quality checking purpose sometimes kafka metadata need to be added on the record. For more information refer to [Depot-bigquery.md#metadata sectionn](https://github.com/raystack/depot/blob/main/docs/sinks/bigquery.md#metadata) ## Default columns for json data type -With dynamic schema for json we need to create table with some default columns, example like parition key needs to be set during creation of the table. Sample config `SINK_BIGQUERY_DEFAULT_COLUMNS =event_timestamp=timestamp`. For more information refer to [Depot-bigquery.md#default-columns-for-json-data-type section](https://github.com/odpf/depot/blob/main/docs/sinks/bigquery.md#default-columns-for-json-data-type) + +With dynamic schema for json we need to create table with some default columns, example like parition key needs to be set during creation of the table. Sample config `SINK_BIGQUERY_DEFAULT_COLUMNS =event_timestamp=timestamp`. For more information refer to [Depot-bigquery.md#default-columns-for-json-data-type section](https://github.com/raystack/depot/blob/main/docs/sinks/bigquery.md#default-columns-for-json-data-type) ## Error handling -The response can contain multiple errors which will be sent to the firehose from depot. Please refer to [Depot-bigquery.md#errors-handling section](https://github.com/odpf/depot/blob/main/docs/sinks/bigquery.md#errors-handling) +The response can contain multiple errors which will be sent to the firehose from depot. Please refer to [Depot-bigquery.md#errors-handling section](https://github.com/raystack/depot/blob/main/docs/sinks/bigquery.md#errors-handling) ## Google Cloud Bigquery IAM Permission -Several IAM permission is required for bigquery sink to run properly. For more detail refer to [Depot-bigquery.md#google-cloud-bigquery-iam-permission section](https://github.com/odpf/depot/blob/main/docs/sinks/bigquery.md#google-cloud-bigquery-iam-permission) - +Several IAM permission is required for bigquery sink to run properly. For more detail refer to [Depot-bigquery.md#google-cloud-bigquery-iam-permission section](https://github.com/raystack/depot/blob/main/docs/sinks/bigquery.md#google-cloud-bigquery-iam-permission) diff --git a/docs/docs/sinks/bigtable-sink.md b/docs/docs/sinks/bigtable-sink.md index 258789f09..a60cecf5e 100644 --- a/docs/docs/sinks/bigtable-sink.md +++ b/docs/docs/sinks/bigtable-sink.md @@ -1,7 +1,7 @@ # Bigtable Sink -Bigtable Sink is implemented in Firehose using the Bigtable sink connector implementation in ODPF Depot. You can check out ODPF Depot Github repository [here](https://github.com/odpf/depot). +Bigtable Sink is implemented in Firehose using the Bigtable sink connector implementation in Depot. You can check out Depot Github repository [here](https://github.com/raystack/depot). ### Configuration -For Bigtable sink in Firehose we need to set first (`SINK_TYPE`=`bigtable`). There are some generic configs which are common across different sink types which need to be set which are mentioned in [generic.md](../advance/generic.md). Bigtable sink specific configs are mentioned in ODPF Depot repository. You can check out the Bigtable Sink configs [here](https://github.com/odpf/depot/blob/main/docs/reference/configuration/bigtable.md) \ No newline at end of file +For Bigtable sink in Firehose we need to set first (`SINK_TYPE`=`bigtable`). There are some generic configs which are common across different sink types which need to be set which are mentioned in [generic.md](../advance/generic.md). Bigtable sink specific configs are mentioned in Depot repository. You can check out the Bigtable Sink configs [here](https://github.com/raystack/depot/blob/main/docs/reference/configuration/bigtable.md) diff --git a/docs/docs/sinks/http-sink.md b/docs/docs/sinks/http-sink.md index 3b9bdbc65..d1cb66725 100644 --- a/docs/docs/sinks/http-sink.md +++ b/docs/docs/sinks/http-sink.md @@ -140,4 +140,12 @@ This config if set to true will allow body for the HTTP DELETE method, otherwise - Example value: `false` - Type: `optional` -- Default value: `true` \ No newline at end of file +- Default value: `true` + +### `SINK_HTTP_SIMPLE_DATE_FORMAT_ENABLE` + +This config if set to `true` will enable the simple date format (Eg. - `May 3, 2023 11:59:37 AM` ) for timestamps (both at root and nested level also) in case of JSON payload. If set to `false`, it will send the timestamps (both at root and nested level also) in the ISO format (Eg. - `2023-05-03T11:59:36.965Z`) . Note: This config is only applicable when the `SINK_HTTP_JSON_BODY_TEMPLATE` config is empty or not supplied. + +- Example value: `false` +- Type: `optional` +- Default value: `true` diff --git a/docs/docs/sinks/redis-sink.md b/docs/docs/sinks/redis-sink.md index ec9d8789c..0e67cbe44 100644 --- a/docs/docs/sinks/redis-sink.md +++ b/docs/docs/sinks/redis-sink.md @@ -1,12 +1,14 @@ # Redis Sink -Redis Sink is implemented in Firehose using the Redis sink connector implementation in ODPF Depot. You can check out ODPF Depot Github repository [here](https://github.com/odpf/depot). +Redis Sink is implemented in Firehose using the Redis sink connector implementation in Depot. You can check out Depot Github repository [here](https://github.com/raystack/depot). ### Data Types -Redis sink can be created in 3 different modes based on the value of [`SINK_REDIS_DATA_TYPE`](https://github.com/odpf/depot/blob/main/docs/reference/configuration/redis.md#sink_redis_data_type): HashSet, KeyValue or List -- `Hashset`: For each message, an entry of the format `key : field : value` is generated and pushed to Redis. Field and value are generated on the basis of the config [`SINK_REDIS_HASHSET_FIELD_TO_COLUMN_MAPPING`](https://github.com/odpf/depot/blob/main/docs/reference/configuration/redis.md#sink_redis_hashset_field_to_column_mapping) -- `List`: For each message entry of the format `key : value` is generated and pushed to Redis. Value is fetched for the Proto field name provided in the config [`SINK_REDIS_LIST_DATA_FIELD_NAME`](https://github.com/odpf/depot/blob/main/docs/reference/configuration/redis.md#sink_redis_list_data_field_name) -- `KeyValue`: For each message entry of the format `key : value` is generated and pushed to Redis. Value is fetched for the proto field name provided in the config [`SINK_REDIS_KEY_VALUE_DATA_FIELD_NAME`](https://github.com/odpf/depot/blob/main/docs/reference/configuration/redis.md#sink_redis_key_value_data_field_name) + +Redis sink can be created in 3 different modes based on the value of [`SINK_REDIS_DATA_TYPE`](https://github.com/raystack/depot/blob/main/docs/reference/configuration/redis.md#sink_redis_data_type): HashSet, KeyValue or List + +- `Hashset`: For each message, an entry of the format `key : field : value` is generated and pushed to Redis. Field and value are generated on the basis of the config [`SINK_REDIS_HASHSET_FIELD_TO_COLUMN_MAPPING`](https://github.com/raystack/depot/blob/main/docs/reference/configuration/redis.md#sink_redis_hashset_field_to_column_mapping) +- `List`: For each message entry of the format `key : value` is generated and pushed to Redis. Value is fetched for the Proto field name provided in the config [`SINK_REDIS_LIST_DATA_FIELD_NAME`](https://github.com/raystack/depot/blob/main/docs/reference/configuration/redis.md#sink_redis_list_data_field_name) +- `KeyValue`: For each message entry of the format `key : value` is generated and pushed to Redis. Value is fetched for the proto field name provided in the config [`SINK_REDIS_KEY_VALUE_DATA_FIELD_NAME`](https://github.com/raystack/depot/blob/main/docs/reference/configuration/redis.md#sink_redis_key_value_data_field_name) The `key` is picked up from a field in the message itself. @@ -14,8 +16,8 @@ Limitation: Depot Redis sink only supports Key-Value, HashSet and List entries a ### Configuration -For Redis sink in Firehose we need to set first (`SINK_TYPE`=`redis`). There are some generic configs which are common across different sink types which need to be set which are mentioned in [generic.md](../advance/generic.md). Redis sink specific configs are mentioned in ODPF Depot repository. You can check out the Redis Sink configs [here](https://github.com/odpf/depot/blob/main/docs/reference/configuration/redis.md) - +For Redis sink in Firehose we need to set first (`SINK_TYPE`=`redis`). There are some generic configs which are common across different sink types which need to be set which are mentioned in [generic.md](../advance/generic.md). Redis sink specific configs are mentioned in Depot repository. You can check out the Redis Sink configs [here](https://github.com/raystack/depot/blob/main/docs/reference/configuration/redis.md) ### Deployment Types + Redis sink, as of now, supports two different Deployment Types `Standalone` and `Cluster`. This can be configured in the Depot environment variable `SINK_REDIS_DEPLOYMENT_TYPE`. diff --git a/docs/docs/support.md b/docs/docs/support.md index 1500c9460..1a9737e42 100644 --- a/docs/docs/support.md +++ b/docs/docs/support.md @@ -1,18 +1,13 @@ # Need help? -Need a bit of help? We're here for you. Check out our current issues, GitHub discussions, or get support through Slack. - -### Slack - -The Firehose team has an open source slack workspace to discuss development and support. Most of the Firehose discussions happen in #firehose channel. -[Join us on Slack](https://odpf-community.slack.com/) +Need a bit of help? We're here for you. Check out our current issues, GitHub discussions ### Issues Have a general issue or bug that you've found? We'd love to hear about it in our GitHub issues. This can be feature requests too! -[Go to issues](https://github.com/odpf/firehose/issues) +[Go to issues](https://github.com/raystack/firehose/issues) ### Discussions For help and questions about best practices, join our GitHub discussions. Browse and ask questions. -[Go to discussions](https://github.com/orgs/odpf/discussions) +[Go to discussions](https://github.com/orgs/raystack/discussions) diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index 17df70e85..cfa73dd1d 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -6,12 +6,12 @@ const darkCodeTheme = require('prism-react-renderer/themes/dracula'); (module.exports = { title: 'Firehose', tagline: 'Load streaming data from Kafka to data stores, data lakes, and analytical storage systems.', - url: 'https://odpf.github.io/', + url: 'https://raystack.github.io/', baseUrl: '/firehose/', onBrokenLinks: 'throw', onBrokenMarkdownLinks: 'warn', favicon: 'assets/favicon.ico', - organizationName: 'ODPF', + organizationName: 'raystack', projectName: 'firehose', presets: [ @@ -24,7 +24,7 @@ const darkCodeTheme = require('prism-react-renderer/themes/dracula'); docs: { routeBasePath: '/', sidebarPath: require.resolve('./sidebars.js'), - editUrl: 'https://github.com/odpf/firehose/edit/master/docs/', + editUrl: 'https://github.com/raystack/firehose/edit/master/docs/', sidebarCollapsed: true, breadcrumbs: false, }, @@ -64,7 +64,7 @@ const darkCodeTheme = require('prism-react-renderer/themes/dracula'); className: 'header-slack-link', }, { - href: 'https://github.com/odpf/firehose', + href: 'https://github.com/raystack/firehose', className: 'navbar-item-github', position: 'right', }, @@ -80,7 +80,7 @@ const darkCodeTheme = require('prism-react-renderer/themes/dracula'); }, announcementBar: { id: 'star-repo', - content: '⭐️ If you like Firehose, give it a star on GitHub! ⭐', + content: '⭐️ If you like Firehose, give it a star on GitHub! ⭐', backgroundColor: '#222', textColor: '#eee', isCloseable: true, diff --git a/docs/static/assets/overview.svg b/docs/static/assets/overview.svg index ec3499d5e..a7ce67a72 100644 --- a/docs/static/assets/overview.svg +++ b/docs/static/assets/overview.svg @@ -70,7 +70,7 @@ Postgres db - + diff --git a/docs/static/overview.svg b/docs/static/overview.svg index 17e26eca1..848eb0d2f 100644 --- a/docs/static/overview.svg +++ b/docs/static/overview.svg @@ -113,10 +113,10 @@ - + - + diff --git a/env/local.properties b/env/local.properties index db738cd31..b1e4db3ad 100644 --- a/env/local.properties +++ b/env/local.properties @@ -1,6 +1,6 @@ ## This is the sample configuration file for Firehose having the ## default/sample values of all sink variables. -## Refer - https://odpf.gitbook.io/firehose/reference/configuration +## Refer - https://raystack.gitbook.io/firehose/reference/configuration ## for detailed description of each configuration variable. # # @@ -10,7 +10,7 @@ # KAFKA_RECORD_PARSER_MODE=message SINK_TYPE=log -INPUT_SCHEMA_PROTO_CLASS=io.odpf.firehose.consumer.TestMessage +INPUT_SCHEMA_PROTO_CLASS=org.raystack.firehose.consumer.TestMessage # INPUT_SCHEMA_PROTO_TO_COLUMN_MAPPING={"1":"order_number","2":"event_timestamp","3":"driver_id"} # METRIC_STATSD_HOST=localhost # METRIC_STATSD_PORT=8125 diff --git a/gradlew.bat b/gradlew.bat index e57a6e267..a6884a6db 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -36,11 +36,11 @@ for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi set DEFAULT_JVM_OPTS="" @rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome +if defined JAVA_HOME raystack findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute +if "%ERRORLEVEL%" == "0" raystack execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -48,13 +48,13 @@ echo. echo Please set the JAVA_HOME variable in your environment to match the echo location of your Java installation. -goto fail +raystack fail :findJavaFromJavaHome set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe -if exist "%JAVA_EXE%" goto execute +if exist "%JAVA_EXE%" raystack execute echo. echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% @@ -62,7 +62,7 @@ echo. echo Please set the JAVA_HOME variable in your environment to match the echo location of your Java installation. -goto fail +raystack fail :execute @rem Setup the command line @@ -75,7 +75,7 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if "%ERRORLEVEL%"=="0" raystack mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/local/policy/WriterPolicy.java b/src/main/java/io/odpf/firehose/sink/blob/writer/local/policy/WriterPolicy.java deleted file mode 100644 index 42ce3d96f..000000000 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/local/policy/WriterPolicy.java +++ /dev/null @@ -1,7 +0,0 @@ -package io.odpf.firehose.sink.blob.writer.local.policy; - -import io.odpf.firehose.sink.blob.writer.local.LocalFileMetadata; - -public interface WriterPolicy { - boolean shouldRotate(LocalFileMetadata metadata); -} diff --git a/src/main/java/io/odpf/firehose/sink/common/blobstorage/BlobStorageType.java b/src/main/java/io/odpf/firehose/sink/common/blobstorage/BlobStorageType.java deleted file mode 100644 index 33b6a9c5f..000000000 --- a/src/main/java/io/odpf/firehose/sink/common/blobstorage/BlobStorageType.java +++ /dev/null @@ -1,6 +0,0 @@ -package io.odpf.firehose.sink.common.blobstorage; - -public enum BlobStorageType { - GCS, - S3 -} diff --git a/src/main/java/io/odpf/firehose/config/AppConfig.java b/src/main/java/org/raystack/firehose/config/AppConfig.java similarity index 84% rename from src/main/java/io/odpf/firehose/config/AppConfig.java rename to src/main/java/org/raystack/firehose/config/AppConfig.java index 612dbb2a1..d71cdfba4 100644 --- a/src/main/java/io/odpf/firehose/config/AppConfig.java +++ b/src/main/java/org/raystack/firehose/config/AppConfig.java @@ -1,13 +1,13 @@ -package io.odpf.firehose.config; - -import io.odpf.firehose.config.converter.InputSchemaTypeConverter; -import io.odpf.firehose.config.converter.ProtoIndexToFieldMapConverter; -import io.odpf.firehose.config.converter.SchemaRegistryHeadersConverter; -import io.odpf.firehose.config.converter.SchemaRegistryRefreshConverter; -import io.odpf.firehose.config.converter.SinkTypeConverter; -import io.odpf.firehose.config.enums.InputSchemaType; -import io.odpf.firehose.config.enums.SinkType; -import io.odpf.stencil.cache.SchemaRefreshStrategy; +package org.raystack.firehose.config; + +import org.raystack.firehose.config.enums.InputSchemaType; +import org.raystack.firehose.config.enums.SinkType; +import org.raystack.firehose.config.converter.InputSchemaTypeConverter; +import org.raystack.firehose.config.converter.ProtoIndexToFieldMapConverter; +import org.raystack.firehose.config.converter.SchemaRegistryHeadersConverter; +import org.raystack.firehose.config.converter.SchemaRegistryRefreshConverter; +import org.raystack.firehose.config.converter.SinkTypeConverter; +import org.raystack.stencil.cache.SchemaRefreshStrategy; import org.aeonbits.owner.Config; import org.apache.http.Header; diff --git a/src/main/java/io/odpf/firehose/config/BlobSinkConfig.java b/src/main/java/org/raystack/firehose/config/BlobSinkConfig.java similarity index 83% rename from src/main/java/io/odpf/firehose/config/BlobSinkConfig.java rename to src/main/java/org/raystack/firehose/config/BlobSinkConfig.java index 8134d5be6..1e6b174c7 100644 --- a/src/main/java/io/odpf/firehose/config/BlobSinkConfig.java +++ b/src/main/java/org/raystack/firehose/config/BlobSinkConfig.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; -import io.odpf.firehose.config.converter.BlobSinkLocalFileWriterTypeConverter; -import io.odpf.firehose.config.converter.BlobSinkFilePartitionTypeConverter; -import io.odpf.firehose.config.converter.BlobStorageTypeConverter; -import io.odpf.firehose.sink.common.blobstorage.BlobStorageType; -import io.odpf.firehose.sink.blob.Constants; +import org.raystack.firehose.config.converter.BlobSinkFilePartitionTypeConverter; +import org.raystack.firehose.config.converter.BlobSinkLocalFileWriterTypeConverter; +import org.raystack.firehose.config.converter.BlobStorageTypeConverter; +import org.raystack.firehose.sink.common.blobstorage.BlobStorageType; +import org.raystack.firehose.sink.blob.Constants; public interface BlobSinkConfig extends AppConfig { diff --git a/src/main/java/io/odpf/firehose/config/DlqConfig.java b/src/main/java/org/raystack/firehose/config/DlqConfig.java similarity index 68% rename from src/main/java/io/odpf/firehose/config/DlqConfig.java rename to src/main/java/org/raystack/firehose/config/DlqConfig.java index 947c94b0d..b2974aa05 100644 --- a/src/main/java/io/odpf/firehose/config/DlqConfig.java +++ b/src/main/java/org/raystack/firehose/config/DlqConfig.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; -import io.odpf.firehose.config.converter.DlqWriterTypeConverter; -import io.odpf.firehose.config.converter.BlobStorageTypeConverter; -import io.odpf.firehose.sink.common.blobstorage.BlobStorageType; -import io.odpf.firehose.sink.dlq.DLQWriterType; +import org.raystack.firehose.config.converter.BlobStorageTypeConverter; +import org.raystack.firehose.config.converter.DlqWriterTypeConverter; +import org.raystack.firehose.sink.common.blobstorage.BlobStorageType; +import org.raystack.firehose.sink.dlq.DLQWriterType; public interface DlqConfig extends AppConfig { diff --git a/src/main/java/io/odpf/firehose/config/DlqKafkaProducerConfig.java b/src/main/java/org/raystack/firehose/config/DlqKafkaProducerConfig.java similarity index 96% rename from src/main/java/io/odpf/firehose/config/DlqKafkaProducerConfig.java rename to src/main/java/org/raystack/firehose/config/DlqKafkaProducerConfig.java index c7182ff43..7698d8158 100644 --- a/src/main/java/io/odpf/firehose/config/DlqKafkaProducerConfig.java +++ b/src/main/java/org/raystack/firehose/config/DlqKafkaProducerConfig.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; public interface DlqKafkaProducerConfig extends DlqConfig { diff --git a/src/main/java/io/odpf/firehose/config/ErrorConfig.java b/src/main/java/org/raystack/firehose/config/ErrorConfig.java similarity index 83% rename from src/main/java/io/odpf/firehose/config/ErrorConfig.java rename to src/main/java/org/raystack/firehose/config/ErrorConfig.java index 4568af826..298657b62 100644 --- a/src/main/java/io/odpf/firehose/config/ErrorConfig.java +++ b/src/main/java/org/raystack/firehose/config/ErrorConfig.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; -import io.odpf.depot.error.ErrorType; -import io.odpf.firehose.config.converter.SetErrorTypeConverter; +import org.raystack.firehose.config.converter.SetErrorTypeConverter; +import org.raystack.depot.error.ErrorType; import org.aeonbits.owner.Config; import org.aeonbits.owner.Mutable; diff --git a/src/main/java/io/odpf/firehose/config/EsSinkConfig.java b/src/main/java/org/raystack/firehose/config/EsSinkConfig.java similarity index 87% rename from src/main/java/io/odpf/firehose/config/EsSinkConfig.java rename to src/main/java/org/raystack/firehose/config/EsSinkConfig.java index a5e5953fd..464611b22 100644 --- a/src/main/java/io/odpf/firehose/config/EsSinkConfig.java +++ b/src/main/java/org/raystack/firehose/config/EsSinkConfig.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; -import io.odpf.firehose.config.converter.EsSinkMessageTypeConverter; -import io.odpf.firehose.config.enums.EsSinkMessageType; +import org.raystack.firehose.config.converter.EsSinkMessageTypeConverter; +import org.raystack.firehose.config.enums.EsSinkMessageType; public interface EsSinkConfig extends AppConfig { diff --git a/src/main/java/io/odpf/firehose/config/FilterConfig.java b/src/main/java/org/raystack/firehose/config/FilterConfig.java similarity index 61% rename from src/main/java/io/odpf/firehose/config/FilterConfig.java rename to src/main/java/org/raystack/firehose/config/FilterConfig.java index 6c117ff0c..80f3d9cf9 100644 --- a/src/main/java/io/odpf/firehose/config/FilterConfig.java +++ b/src/main/java/org/raystack/firehose/config/FilterConfig.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.config; - -import io.odpf.firehose.config.converter.FilterDataSourceTypeConverter; -import io.odpf.firehose.config.converter.FilterEngineTypeConverter; -import io.odpf.firehose.config.converter.FilterMessageFormatTypeConverter; -import io.odpf.firehose.config.enums.FilterDataSourceType; -import io.odpf.firehose.config.enums.FilterEngineType; -import io.odpf.firehose.config.enums.FilterMessageFormatType; +package org.raystack.firehose.config; + +import org.raystack.firehose.config.converter.FilterDataSourceTypeConverter; +import org.raystack.firehose.config.converter.FilterEngineTypeConverter; +import org.raystack.firehose.config.converter.FilterMessageFormatTypeConverter; +import org.raystack.firehose.config.enums.FilterDataSourceType; +import org.raystack.firehose.config.enums.FilterEngineType; +import org.raystack.firehose.config.enums.FilterMessageFormatType; import org.aeonbits.owner.Config; public interface FilterConfig extends Config { diff --git a/src/main/java/io/odpf/firehose/config/GCSConfig.java b/src/main/java/org/raystack/firehose/config/GCSConfig.java similarity index 98% rename from src/main/java/io/odpf/firehose/config/GCSConfig.java rename to src/main/java/org/raystack/firehose/config/GCSConfig.java index c1c7e8afb..f60c1a1d9 100644 --- a/src/main/java/io/odpf/firehose/config/GCSConfig.java +++ b/src/main/java/org/raystack/firehose/config/GCSConfig.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; import org.aeonbits.owner.Config; diff --git a/src/main/java/io/odpf/firehose/config/GrpcSinkConfig.java b/src/main/java/org/raystack/firehose/config/GrpcSinkConfig.java similarity index 92% rename from src/main/java/io/odpf/firehose/config/GrpcSinkConfig.java rename to src/main/java/org/raystack/firehose/config/GrpcSinkConfig.java index bb0c7d75d..82828b424 100644 --- a/src/main/java/io/odpf/firehose/config/GrpcSinkConfig.java +++ b/src/main/java/org/raystack/firehose/config/GrpcSinkConfig.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; import org.aeonbits.owner.Config; diff --git a/src/main/java/io/odpf/firehose/config/HttpSinkConfig.java b/src/main/java/org/raystack/firehose/config/HttpSinkConfig.java similarity index 74% rename from src/main/java/io/odpf/firehose/config/HttpSinkConfig.java rename to src/main/java/org/raystack/firehose/config/HttpSinkConfig.java index 38382ee4c..c53dc6e48 100644 --- a/src/main/java/io/odpf/firehose/config/HttpSinkConfig.java +++ b/src/main/java/org/raystack/firehose/config/HttpSinkConfig.java @@ -1,14 +1,14 @@ -package io.odpf.firehose.config; - -import io.odpf.firehose.config.converter.HttpSinkRequestMethodConverter; -import io.odpf.firehose.config.converter.HttpSinkParameterDataFormatConverter; -import io.odpf.firehose.config.converter.HttpSinkParameterPlacementTypeConverter; -import io.odpf.firehose.config.converter.HttpSinkParameterSourceTypeConverter; -import io.odpf.firehose.config.converter.RangeToHashMapConverter; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; -import io.odpf.firehose.config.enums.HttpSinkDataFormatType; -import io.odpf.firehose.config.enums.HttpSinkParameterPlacementType; -import io.odpf.firehose.config.enums.HttpSinkParameterSourceType; +package org.raystack.firehose.config; + +import org.raystack.firehose.config.enums.HttpSinkDataFormatType; +import org.raystack.firehose.config.enums.HttpSinkParameterPlacementType; +import org.raystack.firehose.config.enums.HttpSinkParameterSourceType; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.config.converter.HttpSinkRequestMethodConverter; +import org.raystack.firehose.config.converter.HttpSinkParameterDataFormatConverter; +import org.raystack.firehose.config.converter.HttpSinkParameterPlacementTypeConverter; +import org.raystack.firehose.config.converter.HttpSinkParameterSourceTypeConverter; +import org.raystack.firehose.config.converter.RangeToHashMapConverter; import java.util.Map; @@ -90,4 +90,8 @@ public interface HttpSinkConfig extends AppConfig { @DefaultValue("true") Boolean getSinkHttpDeleteBodyEnable(); + @Key("SINK_HTTP_SIMPLE_DATE_FORMAT_ENABLE") + @DefaultValue("true") + Boolean getSinkHttpSimpleDateFormatEnable(); + } diff --git a/src/main/java/io/odpf/firehose/config/InfluxSinkConfig.java b/src/main/java/org/raystack/firehose/config/InfluxSinkConfig.java similarity index 90% rename from src/main/java/io/odpf/firehose/config/InfluxSinkConfig.java rename to src/main/java/org/raystack/firehose/config/InfluxSinkConfig.java index 8a0faf4e6..182eb1683 100644 --- a/src/main/java/io/odpf/firehose/config/InfluxSinkConfig.java +++ b/src/main/java/org/raystack/firehose/config/InfluxSinkConfig.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; -import io.odpf.firehose.config.converter.ProtoIndexToFieldMapConverter; +import org.raystack.firehose.config.converter.ProtoIndexToFieldMapConverter; import org.aeonbits.owner.Config; import java.util.Properties; diff --git a/src/main/java/io/odpf/firehose/config/JdbcSinkConfig.java b/src/main/java/org/raystack/firehose/config/JdbcSinkConfig.java similarity index 95% rename from src/main/java/io/odpf/firehose/config/JdbcSinkConfig.java rename to src/main/java/org/raystack/firehose/config/JdbcSinkConfig.java index 7ac45271b..763aa4448 100644 --- a/src/main/java/io/odpf/firehose/config/JdbcSinkConfig.java +++ b/src/main/java/org/raystack/firehose/config/JdbcSinkConfig.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; public interface JdbcSinkConfig extends AppConfig { diff --git a/src/main/java/io/odpf/firehose/config/KafkaConsumerConfig.java b/src/main/java/org/raystack/firehose/config/KafkaConsumerConfig.java similarity index 91% rename from src/main/java/io/odpf/firehose/config/KafkaConsumerConfig.java rename to src/main/java/org/raystack/firehose/config/KafkaConsumerConfig.java index 819d96c88..5faaf0501 100644 --- a/src/main/java/io/odpf/firehose/config/KafkaConsumerConfig.java +++ b/src/main/java/org/raystack/firehose/config/KafkaConsumerConfig.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; -import io.odpf.firehose.config.converter.ConsumerModeConverter; -import io.odpf.firehose.config.enums.KafkaConsumerMode; +import org.raystack.firehose.config.converter.ConsumerModeConverter; +import org.raystack.firehose.config.enums.KafkaConsumerMode; /** * The interface for configurations required to instantiate a consumer. diff --git a/src/main/java/io/odpf/firehose/config/MongoSinkConfig.java b/src/main/java/org/raystack/firehose/config/MongoSinkConfig.java similarity index 89% rename from src/main/java/io/odpf/firehose/config/MongoSinkConfig.java rename to src/main/java/org/raystack/firehose/config/MongoSinkConfig.java index b531ffb17..a2aa001cc 100644 --- a/src/main/java/io/odpf/firehose/config/MongoSinkConfig.java +++ b/src/main/java/org/raystack/firehose/config/MongoSinkConfig.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; -import io.odpf.firehose.config.converter.MongoSinkMessageTypeConverter; -import io.odpf.firehose.config.enums.MongoSinkMessageType; +import org.raystack.firehose.config.converter.MongoSinkMessageTypeConverter; +import org.raystack.firehose.config.enums.MongoSinkMessageType; public interface MongoSinkConfig extends AppConfig { diff --git a/src/main/java/io/odpf/firehose/config/PromSinkConfig.java b/src/main/java/org/raystack/firehose/config/PromSinkConfig.java similarity index 88% rename from src/main/java/io/odpf/firehose/config/PromSinkConfig.java rename to src/main/java/org/raystack/firehose/config/PromSinkConfig.java index 8f4bb4bbf..36b741936 100644 --- a/src/main/java/io/odpf/firehose/config/PromSinkConfig.java +++ b/src/main/java/org/raystack/firehose/config/PromSinkConfig.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; -import io.odpf.firehose.config.converter.ProtoIndexToFieldMapConverter; -import io.odpf.firehose.config.converter.RangeToHashMapConverter; +import org.raystack.firehose.config.converter.ProtoIndexToFieldMapConverter; +import org.raystack.firehose.config.converter.RangeToHashMapConverter; import java.util.Map; import java.util.Properties; diff --git a/src/main/java/io/odpf/firehose/config/S3Config.java b/src/main/java/org/raystack/firehose/config/S3Config.java similarity index 95% rename from src/main/java/io/odpf/firehose/config/S3Config.java rename to src/main/java/org/raystack/firehose/config/S3Config.java index 827edbbb4..25cff477f 100644 --- a/src/main/java/io/odpf/firehose/config/S3Config.java +++ b/src/main/java/org/raystack/firehose/config/S3Config.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; import org.aeonbits.owner.Config; diff --git a/src/main/java/io/odpf/firehose/config/SinkPoolConfig.java b/src/main/java/org/raystack/firehose/config/SinkPoolConfig.java similarity index 89% rename from src/main/java/io/odpf/firehose/config/SinkPoolConfig.java rename to src/main/java/org/raystack/firehose/config/SinkPoolConfig.java index 46ce0fa08..bed74d9cb 100644 --- a/src/main/java/io/odpf/firehose/config/SinkPoolConfig.java +++ b/src/main/java/org/raystack/firehose/config/SinkPoolConfig.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; import org.aeonbits.owner.Config; diff --git a/src/main/java/io/odpf/firehose/config/converter/BlobSinkFilePartitionTypeConverter.java b/src/main/java/org/raystack/firehose/config/converter/BlobSinkFilePartitionTypeConverter.java similarity index 77% rename from src/main/java/io/odpf/firehose/config/converter/BlobSinkFilePartitionTypeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/BlobSinkFilePartitionTypeConverter.java index 2c7de5d31..93aaab23e 100644 --- a/src/main/java/io/odpf/firehose/config/converter/BlobSinkFilePartitionTypeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/BlobSinkFilePartitionTypeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.sink.blob.Constants; +import org.raystack.firehose.sink.blob.Constants; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/BlobSinkLocalFileWriterTypeConverter.java b/src/main/java/org/raystack/firehose/config/converter/BlobSinkLocalFileWriterTypeConverter.java similarity index 76% rename from src/main/java/io/odpf/firehose/config/converter/BlobSinkLocalFileWriterTypeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/BlobSinkLocalFileWriterTypeConverter.java index 8406d8de8..19fd64e63 100644 --- a/src/main/java/io/odpf/firehose/config/converter/BlobSinkLocalFileWriterTypeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/BlobSinkLocalFileWriterTypeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.sink.blob.Constants; +import org.raystack.firehose.sink.blob.Constants; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/BlobStorageTypeConverter.java b/src/main/java/org/raystack/firehose/config/converter/BlobStorageTypeConverter.java similarity index 71% rename from src/main/java/io/odpf/firehose/config/converter/BlobStorageTypeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/BlobStorageTypeConverter.java index 180f59eb9..fcb96da1b 100644 --- a/src/main/java/io/odpf/firehose/config/converter/BlobStorageTypeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/BlobStorageTypeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.sink.common.blobstorage.BlobStorageType; +import org.raystack.firehose.sink.common.blobstorage.BlobStorageType; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/ConsumerModeConverter.java b/src/main/java/org/raystack/firehose/config/converter/ConsumerModeConverter.java similarity index 73% rename from src/main/java/io/odpf/firehose/config/converter/ConsumerModeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/ConsumerModeConverter.java index 2cfcfc21d..f77dbf1a4 100644 --- a/src/main/java/io/odpf/firehose/config/converter/ConsumerModeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/ConsumerModeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.config.enums.KafkaConsumerMode; +import org.raystack.firehose.config.enums.KafkaConsumerMode; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/DlqWriterTypeConverter.java b/src/main/java/org/raystack/firehose/config/converter/DlqWriterTypeConverter.java similarity index 74% rename from src/main/java/io/odpf/firehose/config/converter/DlqWriterTypeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/DlqWriterTypeConverter.java index b40424675..ec24065f5 100644 --- a/src/main/java/io/odpf/firehose/config/converter/DlqWriterTypeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/DlqWriterTypeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.sink.dlq.DLQWriterType; +import org.raystack.firehose.sink.dlq.DLQWriterType; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/EsSinkMessageTypeConverter.java b/src/main/java/org/raystack/firehose/config/converter/EsSinkMessageTypeConverter.java similarity index 73% rename from src/main/java/io/odpf/firehose/config/converter/EsSinkMessageTypeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/EsSinkMessageTypeConverter.java index 303b2a8bc..185da9b39 100644 --- a/src/main/java/io/odpf/firehose/config/converter/EsSinkMessageTypeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/EsSinkMessageTypeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.config.enums.EsSinkMessageType; +import org.raystack.firehose.config.enums.EsSinkMessageType; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/FilterDataSourceTypeConverter.java b/src/main/java/org/raystack/firehose/config/converter/FilterDataSourceTypeConverter.java similarity index 81% rename from src/main/java/io/odpf/firehose/config/converter/FilterDataSourceTypeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/FilterDataSourceTypeConverter.java index 62a9cee37..54ca4b26f 100644 --- a/src/main/java/io/odpf/firehose/config/converter/FilterDataSourceTypeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/FilterDataSourceTypeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.config.enums.FilterDataSourceType; +import org.raystack.firehose.config.enums.FilterDataSourceType; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/FilterEngineTypeConverter.java b/src/main/java/org/raystack/firehose/config/converter/FilterEngineTypeConverter.java similarity index 81% rename from src/main/java/io/odpf/firehose/config/converter/FilterEngineTypeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/FilterEngineTypeConverter.java index 091434013..638ba30f5 100644 --- a/src/main/java/io/odpf/firehose/config/converter/FilterEngineTypeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/FilterEngineTypeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.config.enums.FilterEngineType; +import org.raystack.firehose.config.enums.FilterEngineType; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/FilterMessageFormatTypeConverter.java b/src/main/java/org/raystack/firehose/config/converter/FilterMessageFormatTypeConverter.java similarity index 81% rename from src/main/java/io/odpf/firehose/config/converter/FilterMessageFormatTypeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/FilterMessageFormatTypeConverter.java index 57d395459..ffb2844ee 100644 --- a/src/main/java/io/odpf/firehose/config/converter/FilterMessageFormatTypeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/FilterMessageFormatTypeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.config.enums.FilterMessageFormatType; +import org.raystack.firehose.config.enums.FilterMessageFormatType; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/HttpSinkDataFormatTypeConverter.java b/src/main/java/org/raystack/firehose/config/converter/HttpSinkDataFormatTypeConverter.java similarity index 74% rename from src/main/java/io/odpf/firehose/config/converter/HttpSinkDataFormatTypeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/HttpSinkDataFormatTypeConverter.java index a44f1d2e1..e8b6816ee 100644 --- a/src/main/java/io/odpf/firehose/config/converter/HttpSinkDataFormatTypeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/HttpSinkDataFormatTypeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.config.enums.HttpSinkDataFormatType; +import org.raystack.firehose.config.enums.HttpSinkDataFormatType; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/HttpSinkParameterDataFormatConverter.java b/src/main/java/org/raystack/firehose/config/converter/HttpSinkParameterDataFormatConverter.java similarity index 74% rename from src/main/java/io/odpf/firehose/config/converter/HttpSinkParameterDataFormatConverter.java rename to src/main/java/org/raystack/firehose/config/converter/HttpSinkParameterDataFormatConverter.java index 3dc7ef3d5..db99eff4b 100644 --- a/src/main/java/io/odpf/firehose/config/converter/HttpSinkParameterDataFormatConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/HttpSinkParameterDataFormatConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.config.enums.HttpSinkDataFormatType; +import org.raystack.firehose.config.enums.HttpSinkDataFormatType; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/HttpSinkParameterPlacementTypeConverter.java b/src/main/java/org/raystack/firehose/config/converter/HttpSinkParameterPlacementTypeConverter.java similarity index 74% rename from src/main/java/io/odpf/firehose/config/converter/HttpSinkParameterPlacementTypeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/HttpSinkParameterPlacementTypeConverter.java index a478f2ac1..dc1187553 100644 --- a/src/main/java/io/odpf/firehose/config/converter/HttpSinkParameterPlacementTypeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/HttpSinkParameterPlacementTypeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.config.enums.HttpSinkParameterPlacementType; +import org.raystack.firehose.config.enums.HttpSinkParameterPlacementType; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/HttpSinkParameterSourceTypeConverter.java b/src/main/java/org/raystack/firehose/config/converter/HttpSinkParameterSourceTypeConverter.java similarity index 74% rename from src/main/java/io/odpf/firehose/config/converter/HttpSinkParameterSourceTypeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/HttpSinkParameterSourceTypeConverter.java index 545bf7e5c..84f97de1b 100644 --- a/src/main/java/io/odpf/firehose/config/converter/HttpSinkParameterSourceTypeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/HttpSinkParameterSourceTypeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.config.enums.HttpSinkParameterSourceType; +import org.raystack.firehose.config.enums.HttpSinkParameterSourceType; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/HttpSinkRequestMethodConverter.java b/src/main/java/org/raystack/firehose/config/converter/HttpSinkRequestMethodConverter.java similarity index 74% rename from src/main/java/io/odpf/firehose/config/converter/HttpSinkRequestMethodConverter.java rename to src/main/java/org/raystack/firehose/config/converter/HttpSinkRequestMethodConverter.java index 3976c921c..33356debc 100644 --- a/src/main/java/io/odpf/firehose/config/converter/HttpSinkRequestMethodConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/HttpSinkRequestMethodConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/InputSchemaTypeConverter.java b/src/main/java/org/raystack/firehose/config/converter/InputSchemaTypeConverter.java similarity index 74% rename from src/main/java/io/odpf/firehose/config/converter/InputSchemaTypeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/InputSchemaTypeConverter.java index 8341a6f3f..83a07a617 100644 --- a/src/main/java/io/odpf/firehose/config/converter/InputSchemaTypeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/InputSchemaTypeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.config.enums.InputSchemaType; +import org.raystack.firehose.config.enums.InputSchemaType; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/LabelMapConverter.java b/src/main/java/org/raystack/firehose/config/converter/LabelMapConverter.java similarity index 95% rename from src/main/java/io/odpf/firehose/config/converter/LabelMapConverter.java rename to src/main/java/org/raystack/firehose/config/converter/LabelMapConverter.java index 0dfb2a5ad..e395c1a16 100644 --- a/src/main/java/io/odpf/firehose/config/converter/LabelMapConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/LabelMapConverter.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/MongoSinkMessageTypeConverter.java b/src/main/java/org/raystack/firehose/config/converter/MongoSinkMessageTypeConverter.java similarity index 81% rename from src/main/java/io/odpf/firehose/config/converter/MongoSinkMessageTypeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/MongoSinkMessageTypeConverter.java index 2d8849c42..92b3639ac 100644 --- a/src/main/java/io/odpf/firehose/config/converter/MongoSinkMessageTypeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/MongoSinkMessageTypeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.config.enums.MongoSinkMessageType; +import org.raystack.firehose.config.enums.MongoSinkMessageType; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/ProtoIndexToFieldMapConverter.java b/src/main/java/org/raystack/firehose/config/converter/ProtoIndexToFieldMapConverter.java similarity index 98% rename from src/main/java/io/odpf/firehose/config/converter/ProtoIndexToFieldMapConverter.java rename to src/main/java/org/raystack/firehose/config/converter/ProtoIndexToFieldMapConverter.java index 951d249d2..4ff521631 100644 --- a/src/main/java/io/odpf/firehose/config/converter/ProtoIndexToFieldMapConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/ProtoIndexToFieldMapConverter.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; import com.google.common.base.Strings; import com.google.gson.Gson; diff --git a/src/main/java/io/odpf/firehose/config/converter/RangeToHashMapConverter.java b/src/main/java/org/raystack/firehose/config/converter/RangeToHashMapConverter.java similarity index 94% rename from src/main/java/io/odpf/firehose/config/converter/RangeToHashMapConverter.java rename to src/main/java/org/raystack/firehose/config/converter/RangeToHashMapConverter.java index cdb2d1a7e..c0d16e74d 100644 --- a/src/main/java/io/odpf/firehose/config/converter/RangeToHashMapConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/RangeToHashMapConverter.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; import org.aeonbits.owner.Converter; diff --git a/src/main/java/io/odpf/firehose/config/converter/SchemaRegistryHeadersConverter.java b/src/main/java/org/raystack/firehose/config/converter/SchemaRegistryHeadersConverter.java similarity index 96% rename from src/main/java/io/odpf/firehose/config/converter/SchemaRegistryHeadersConverter.java rename to src/main/java/org/raystack/firehose/config/converter/SchemaRegistryHeadersConverter.java index f73efb561..aec5c255d 100644 --- a/src/main/java/io/odpf/firehose/config/converter/SchemaRegistryHeadersConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/SchemaRegistryHeadersConverter.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; import java.lang.reflect.Method; import java.util.regex.Pattern; diff --git a/src/main/java/io/odpf/firehose/config/converter/SchemaRegistryRefreshConverter.java b/src/main/java/org/raystack/firehose/config/converter/SchemaRegistryRefreshConverter.java similarity index 81% rename from src/main/java/io/odpf/firehose/config/converter/SchemaRegistryRefreshConverter.java rename to src/main/java/org/raystack/firehose/config/converter/SchemaRegistryRefreshConverter.java index 295ca9e57..cf2f0cb4d 100644 --- a/src/main/java/io/odpf/firehose/config/converter/SchemaRegistryRefreshConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/SchemaRegistryRefreshConverter.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; import java.lang.reflect.Method; import org.aeonbits.owner.Converter; -import io.odpf.stencil.cache.SchemaRefreshStrategy; +import org.raystack.stencil.cache.SchemaRefreshStrategy; public class SchemaRegistryRefreshConverter implements Converter { diff --git a/src/main/java/io/odpf/firehose/config/converter/SetErrorTypeConverter.java b/src/main/java/org/raystack/firehose/config/converter/SetErrorTypeConverter.java similarity index 74% rename from src/main/java/io/odpf/firehose/config/converter/SetErrorTypeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/SetErrorTypeConverter.java index 3b121593d..a93a7bd02 100644 --- a/src/main/java/io/odpf/firehose/config/converter/SetErrorTypeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/SetErrorTypeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.depot.error.ErrorType; +import org.raystack.depot.error.ErrorType; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/converter/SinkTypeConverter.java b/src/main/java/org/raystack/firehose/config/converter/SinkTypeConverter.java similarity index 73% rename from src/main/java/io/odpf/firehose/config/converter/SinkTypeConverter.java rename to src/main/java/org/raystack/firehose/config/converter/SinkTypeConverter.java index aa553d4c1..2c11f4e4c 100644 --- a/src/main/java/io/odpf/firehose/config/converter/SinkTypeConverter.java +++ b/src/main/java/org/raystack/firehose/config/converter/SinkTypeConverter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.config.enums.SinkType; +import org.raystack.firehose.config.enums.SinkType; import org.aeonbits.owner.Converter; import java.lang.reflect.Method; diff --git a/src/main/java/io/odpf/firehose/config/enums/EsSinkMessageType.java b/src/main/java/org/raystack/firehose/config/enums/EsSinkMessageType.java similarity index 55% rename from src/main/java/io/odpf/firehose/config/enums/EsSinkMessageType.java rename to src/main/java/org/raystack/firehose/config/enums/EsSinkMessageType.java index 503c969f9..a7b7670ec 100644 --- a/src/main/java/io/odpf/firehose/config/enums/EsSinkMessageType.java +++ b/src/main/java/org/raystack/firehose/config/enums/EsSinkMessageType.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.enums; +package org.raystack.firehose.config.enums; public enum EsSinkMessageType { JSON, PROTOBUF diff --git a/src/main/java/io/odpf/firehose/config/enums/EsSinkRequestType.java b/src/main/java/org/raystack/firehose/config/enums/EsSinkRequestType.java similarity index 62% rename from src/main/java/io/odpf/firehose/config/enums/EsSinkRequestType.java rename to src/main/java/org/raystack/firehose/config/enums/EsSinkRequestType.java index 663e0da23..3f4130f78 100644 --- a/src/main/java/io/odpf/firehose/config/enums/EsSinkRequestType.java +++ b/src/main/java/org/raystack/firehose/config/enums/EsSinkRequestType.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.enums; +package org.raystack.firehose.config.enums; public enum EsSinkRequestType { UPDATE_ONLY, diff --git a/src/main/java/io/odpf/firehose/config/enums/FilterDataSourceType.java b/src/main/java/org/raystack/firehose/config/enums/FilterDataSourceType.java similarity index 55% rename from src/main/java/io/odpf/firehose/config/enums/FilterDataSourceType.java rename to src/main/java/org/raystack/firehose/config/enums/FilterDataSourceType.java index ea0fa4a92..e9412511a 100644 --- a/src/main/java/io/odpf/firehose/config/enums/FilterDataSourceType.java +++ b/src/main/java/org/raystack/firehose/config/enums/FilterDataSourceType.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.enums; +package org.raystack.firehose.config.enums; public enum FilterDataSourceType { KEY, MESSAGE diff --git a/src/main/java/io/odpf/firehose/config/enums/FilterEngineType.java b/src/main/java/org/raystack/firehose/config/enums/FilterEngineType.java similarity index 56% rename from src/main/java/io/odpf/firehose/config/enums/FilterEngineType.java rename to src/main/java/org/raystack/firehose/config/enums/FilterEngineType.java index b2de50828..97927c596 100644 --- a/src/main/java/io/odpf/firehose/config/enums/FilterEngineType.java +++ b/src/main/java/org/raystack/firehose/config/enums/FilterEngineType.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.enums; +package org.raystack.firehose.config.enums; public enum FilterEngineType { JEXL, JSON, NO_OP diff --git a/src/main/java/io/odpf/firehose/config/enums/FilterMessageFormatType.java b/src/main/java/org/raystack/firehose/config/enums/FilterMessageFormatType.java similarity index 57% rename from src/main/java/io/odpf/firehose/config/enums/FilterMessageFormatType.java rename to src/main/java/org/raystack/firehose/config/enums/FilterMessageFormatType.java index 4b5e3e1d0..f2b6ebdf6 100644 --- a/src/main/java/io/odpf/firehose/config/enums/FilterMessageFormatType.java +++ b/src/main/java/org/raystack/firehose/config/enums/FilterMessageFormatType.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.enums; +package org.raystack.firehose.config.enums; public enum FilterMessageFormatType { JSON, PROTOBUF diff --git a/src/main/java/io/odpf/firehose/config/enums/HttpSinkDataFormatType.java b/src/main/java/org/raystack/firehose/config/enums/HttpSinkDataFormatType.java similarity index 57% rename from src/main/java/io/odpf/firehose/config/enums/HttpSinkDataFormatType.java rename to src/main/java/org/raystack/firehose/config/enums/HttpSinkDataFormatType.java index ff4586254..029c78d92 100644 --- a/src/main/java/io/odpf/firehose/config/enums/HttpSinkDataFormatType.java +++ b/src/main/java/org/raystack/firehose/config/enums/HttpSinkDataFormatType.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.enums; +package org.raystack.firehose.config.enums; public enum HttpSinkDataFormatType { PROTO, diff --git a/src/main/java/io/odpf/firehose/config/enums/HttpSinkParameterPlacementType.java b/src/main/java/org/raystack/firehose/config/enums/HttpSinkParameterPlacementType.java similarity index 61% rename from src/main/java/io/odpf/firehose/config/enums/HttpSinkParameterPlacementType.java rename to src/main/java/org/raystack/firehose/config/enums/HttpSinkParameterPlacementType.java index 7cb91b544..4b7532e3d 100644 --- a/src/main/java/io/odpf/firehose/config/enums/HttpSinkParameterPlacementType.java +++ b/src/main/java/org/raystack/firehose/config/enums/HttpSinkParameterPlacementType.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.enums; +package org.raystack.firehose.config.enums; public enum HttpSinkParameterPlacementType { QUERY, diff --git a/src/main/java/io/odpf/firehose/config/enums/HttpSinkParameterSourceType.java b/src/main/java/org/raystack/firehose/config/enums/HttpSinkParameterSourceType.java similarity index 64% rename from src/main/java/io/odpf/firehose/config/enums/HttpSinkParameterSourceType.java rename to src/main/java/org/raystack/firehose/config/enums/HttpSinkParameterSourceType.java index 92e5eb8b7..d4acb84a3 100644 --- a/src/main/java/io/odpf/firehose/config/enums/HttpSinkParameterSourceType.java +++ b/src/main/java/org/raystack/firehose/config/enums/HttpSinkParameterSourceType.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.enums; +package org.raystack.firehose.config.enums; public enum HttpSinkParameterSourceType { KEY, diff --git a/src/main/java/io/odpf/firehose/config/enums/HttpSinkRequestMethodType.java b/src/main/java/org/raystack/firehose/config/enums/HttpSinkRequestMethodType.java similarity index 65% rename from src/main/java/io/odpf/firehose/config/enums/HttpSinkRequestMethodType.java rename to src/main/java/org/raystack/firehose/config/enums/HttpSinkRequestMethodType.java index 6d708c15f..a6bdb0144 100644 --- a/src/main/java/io/odpf/firehose/config/enums/HttpSinkRequestMethodType.java +++ b/src/main/java/org/raystack/firehose/config/enums/HttpSinkRequestMethodType.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.enums; +package org.raystack.firehose.config.enums; public enum HttpSinkRequestMethodType { PUT, diff --git a/src/main/java/io/odpf/firehose/config/enums/InputSchemaType.java b/src/main/java/org/raystack/firehose/config/enums/InputSchemaType.java similarity index 56% rename from src/main/java/io/odpf/firehose/config/enums/InputSchemaType.java rename to src/main/java/org/raystack/firehose/config/enums/InputSchemaType.java index 49fadc084..9a37d9b60 100644 --- a/src/main/java/io/odpf/firehose/config/enums/InputSchemaType.java +++ b/src/main/java/org/raystack/firehose/config/enums/InputSchemaType.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.enums; +package org.raystack.firehose.config.enums; public enum InputSchemaType { PROTOBUF, diff --git a/src/main/java/io/odpf/firehose/config/enums/KafkaConsumerMode.java b/src/main/java/org/raystack/firehose/config/enums/KafkaConsumerMode.java similarity index 55% rename from src/main/java/io/odpf/firehose/config/enums/KafkaConsumerMode.java rename to src/main/java/org/raystack/firehose/config/enums/KafkaConsumerMode.java index 8e01de36d..2024645f7 100644 --- a/src/main/java/io/odpf/firehose/config/enums/KafkaConsumerMode.java +++ b/src/main/java/org/raystack/firehose/config/enums/KafkaConsumerMode.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.enums; +package org.raystack.firehose.config.enums; public enum KafkaConsumerMode { ASYNC, diff --git a/src/main/java/io/odpf/firehose/config/enums/MongoSinkMessageType.java b/src/main/java/org/raystack/firehose/config/enums/MongoSinkMessageType.java similarity index 56% rename from src/main/java/io/odpf/firehose/config/enums/MongoSinkMessageType.java rename to src/main/java/org/raystack/firehose/config/enums/MongoSinkMessageType.java index 8f760506e..48c2f7b9f 100644 --- a/src/main/java/io/odpf/firehose/config/enums/MongoSinkMessageType.java +++ b/src/main/java/org/raystack/firehose/config/enums/MongoSinkMessageType.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.enums; +package org.raystack.firehose.config.enums; public enum MongoSinkMessageType { JSON, PROTOBUF diff --git a/src/main/java/io/odpf/firehose/config/enums/MongoSinkRequestType.java b/src/main/java/org/raystack/firehose/config/enums/MongoSinkRequestType.java similarity index 60% rename from src/main/java/io/odpf/firehose/config/enums/MongoSinkRequestType.java rename to src/main/java/org/raystack/firehose/config/enums/MongoSinkRequestType.java index fb45cfd14..eacf51bb0 100644 --- a/src/main/java/io/odpf/firehose/config/enums/MongoSinkRequestType.java +++ b/src/main/java/org/raystack/firehose/config/enums/MongoSinkRequestType.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.enums; +package org.raystack.firehose.config.enums; public enum MongoSinkRequestType { UPDATE_ONLY, diff --git a/src/main/java/io/odpf/firehose/config/enums/SinkType.java b/src/main/java/org/raystack/firehose/config/enums/SinkType.java similarity index 81% rename from src/main/java/io/odpf/firehose/config/enums/SinkType.java rename to src/main/java/org/raystack/firehose/config/enums/SinkType.java index eaa73a157..80e84fc47 100644 --- a/src/main/java/io/odpf/firehose/config/enums/SinkType.java +++ b/src/main/java/org/raystack/firehose/config/enums/SinkType.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config.enums; +package org.raystack.firehose.config.enums; public enum SinkType { JDBC, diff --git a/src/main/java/io/odpf/firehose/consumer/FirehoseAsyncConsumer.java b/src/main/java/org/raystack/firehose/consumer/FirehoseAsyncConsumer.java similarity index 81% rename from src/main/java/io/odpf/firehose/consumer/FirehoseAsyncConsumer.java rename to src/main/java/org/raystack/firehose/consumer/FirehoseAsyncConsumer.java index 45f9d8190..7c0af3db7 100644 --- a/src/main/java/io/odpf/firehose/consumer/FirehoseAsyncConsumer.java +++ b/src/main/java/org/raystack/firehose/consumer/FirehoseAsyncConsumer.java @@ -1,13 +1,13 @@ -package io.odpf.firehose.consumer; +package org.raystack.firehose.consumer; -import io.odpf.firehose.consumer.kafka.ConsumerAndOffsetManager; -import io.odpf.firehose.exception.FirehoseConsumerFailedException; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.SinkPool; -import io.odpf.firehose.filter.FilterException; -import io.odpf.firehose.filter.FilteredMessages; -import io.odpf.firehose.tracer.SinkTracer; +import org.raystack.firehose.consumer.kafka.ConsumerAndOffsetManager; +import org.raystack.firehose.exception.FirehoseConsumerFailedException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.SinkPool; +import org.raystack.firehose.filter.FilterException; +import org.raystack.firehose.filter.FilteredMessages; +import org.raystack.firehose.tracer.SinkTracer; import io.opentracing.Span; import lombok.AllArgsConstructor; @@ -16,7 +16,7 @@ import java.util.List; import java.util.concurrent.Future; -import static io.odpf.firehose.metrics.Metrics.SOURCE_KAFKA_PARTITIONS_PROCESS_TIME_MILLISECONDS; +import static org.raystack.firehose.metrics.Metrics.SOURCE_KAFKA_PARTITIONS_PROCESS_TIME_MILLISECONDS; @AllArgsConstructor public class FirehoseAsyncConsumer implements FirehoseConsumer { @@ -66,9 +66,9 @@ private Future> scheduleTask(List messages) { @Override public void close() throws IOException { + sinkPool.close(); consumerAndOffsetManager.close(); tracer.close(); - sinkPool.close(); firehoseInstrumentation.close(); } } diff --git a/src/main/java/io/odpf/firehose/consumer/FirehoseConsumer.java b/src/main/java/org/raystack/firehose/consumer/FirehoseConsumer.java similarity index 79% rename from src/main/java/io/odpf/firehose/consumer/FirehoseConsumer.java rename to src/main/java/org/raystack/firehose/consumer/FirehoseConsumer.java index f3874d93b..408ee43dc 100644 --- a/src/main/java/io/odpf/firehose/consumer/FirehoseConsumer.java +++ b/src/main/java/org/raystack/firehose/consumer/FirehoseConsumer.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.consumer; +package org.raystack.firehose.consumer; import java.io.Closeable; import java.io.IOException; diff --git a/src/main/java/io/odpf/firehose/consumer/FirehoseConsumerFactory.java b/src/main/java/org/raystack/firehose/consumer/FirehoseConsumerFactory.java similarity index 81% rename from src/main/java/io/odpf/firehose/consumer/FirehoseConsumerFactory.java rename to src/main/java/org/raystack/firehose/consumer/FirehoseConsumerFactory.java index d17e04616..9b74094df 100644 --- a/src/main/java/io/odpf/firehose/consumer/FirehoseConsumerFactory.java +++ b/src/main/java/org/raystack/firehose/consumer/FirehoseConsumerFactory.java @@ -1,42 +1,42 @@ -package io.odpf.firehose.consumer; +package org.raystack.firehose.consumer; +import org.raystack.firehose.consumer.kafka.ConsumerAndOffsetManager; +import org.raystack.firehose.consumer.kafka.FirehoseKafkaConsumer; +import org.raystack.firehose.consumer.kafka.OffsetManager; import io.jaegertracing.Configuration; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.consumer.kafka.ConsumerAndOffsetManager; -import io.odpf.firehose.consumer.kafka.FirehoseKafkaConsumer; -import io.odpf.firehose.consumer.kafka.OffsetManager; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.SinkFactory; -import io.odpf.firehose.utils.KafkaUtils; -import io.odpf.firehose.config.AppConfig; -import io.odpf.firehose.config.DlqConfig; -import io.odpf.firehose.config.FilterConfig; -import io.odpf.firehose.config.ErrorConfig; -import io.odpf.firehose.config.KafkaConsumerConfig; -import io.odpf.firehose.config.SinkPoolConfig; -import io.odpf.firehose.config.enums.KafkaConsumerMode; -import io.odpf.firehose.sink.SinkPool; -import io.odpf.firehose.filter.Filter; -import io.odpf.firehose.filter.NoOpFilter; -import io.odpf.firehose.filter.jexl.JexlFilter; -import io.odpf.firehose.filter.json.JsonFilter; -import io.odpf.firehose.filter.json.JsonFilterUtil; -import io.odpf.firehose.sink.Sink; -import io.odpf.firehose.sink.common.KeyOrMessageParser; -import io.odpf.firehose.sinkdecorator.BackOff; -import io.odpf.firehose.sinkdecorator.BackOffProvider; -import io.odpf.firehose.error.ErrorHandler; -import io.odpf.firehose.sinkdecorator.ExponentialBackOffProvider; -import io.odpf.firehose.sinkdecorator.SinkFinal; -import io.odpf.firehose.sinkdecorator.SinkWithDlq; -import io.odpf.firehose.sinkdecorator.SinkWithFailHandler; -import io.odpf.firehose.sinkdecorator.SinkWithRetry; -import io.odpf.firehose.sink.dlq.DlqWriter; -import io.odpf.firehose.sink.dlq.DlqWriterFactory; -import io.odpf.firehose.tracer.SinkTracer; -import io.odpf.firehose.utils.StencilUtils; -import io.odpf.stencil.StencilClientFactory; -import io.odpf.stencil.client.StencilClient; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.SinkFactory; +import org.raystack.firehose.utils.KafkaUtils; +import org.raystack.firehose.config.AppConfig; +import org.raystack.firehose.config.DlqConfig; +import org.raystack.firehose.config.FilterConfig; +import org.raystack.firehose.config.ErrorConfig; +import org.raystack.firehose.config.KafkaConsumerConfig; +import org.raystack.firehose.config.SinkPoolConfig; +import org.raystack.firehose.config.enums.KafkaConsumerMode; +import org.raystack.firehose.sink.SinkPool; +import org.raystack.firehose.filter.Filter; +import org.raystack.firehose.filter.NoOpFilter; +import org.raystack.firehose.filter.jexl.JexlFilter; +import org.raystack.firehose.filter.json.JsonFilter; +import org.raystack.firehose.filter.json.JsonFilterUtil; +import org.raystack.firehose.sink.Sink; +import org.raystack.firehose.sink.common.KeyOrMessageParser; +import org.raystack.firehose.sinkdecorator.BackOff; +import org.raystack.firehose.sinkdecorator.BackOffProvider; +import org.raystack.firehose.error.ErrorHandler; +import org.raystack.firehose.sinkdecorator.ExponentialBackOffProvider; +import org.raystack.firehose.sinkdecorator.SinkFinal; +import org.raystack.firehose.sinkdecorator.SinkWithDlq; +import org.raystack.firehose.sinkdecorator.SinkWithFailHandler; +import org.raystack.firehose.sinkdecorator.SinkWithRetry; +import org.raystack.firehose.sink.dlq.DlqWriter; +import org.raystack.firehose.sink.dlq.DlqWriterFactory; +import org.raystack.firehose.tracer.SinkTracer; +import org.raystack.firehose.utils.StencilUtils; +import org.raystack.stencil.StencilClientFactory; +import org.raystack.stencil.client.StencilClient; import io.opentracing.Tracer; import io.opentracing.noop.NoopTracerFactory; import org.aeonbits.owner.ConfigFactory; @@ -145,6 +145,7 @@ public FirehoseConsumer buildConsumer() { ConsumerAndOffsetManager consumerAndOffsetManager = new ConsumerAndOffsetManager(sinks, offsetManager, firehoseKafkaConsumer, kafkaConsumerConfig, new FirehoseInstrumentation(statsDReporter, ConsumerAndOffsetManager.class)); SinkPool sinkPool = new SinkPool( new LinkedBlockingQueue<>(sinks), + sinks, Executors.newCachedThreadPool(), sinkPoolConfig.getSinkPoolQueuePollTimeoutMS()); return new FirehoseAsyncConsumer( diff --git a/src/main/java/io/odpf/firehose/consumer/FirehoseFilter.java b/src/main/java/org/raystack/firehose/consumer/FirehoseFilter.java similarity index 68% rename from src/main/java/io/odpf/firehose/consumer/FirehoseFilter.java rename to src/main/java/org/raystack/firehose/consumer/FirehoseFilter.java index 4115795d4..c38168968 100644 --- a/src/main/java/io/odpf/firehose/consumer/FirehoseFilter.java +++ b/src/main/java/org/raystack/firehose/consumer/FirehoseFilter.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.consumer; +package org.raystack.firehose.consumer; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.filter.Filter; -import io.odpf.firehose.filter.FilterException; -import io.odpf.firehose.filter.FilteredMessages; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.metrics.Metrics; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.filter.Filter; +import org.raystack.firehose.filter.FilterException; +import org.raystack.firehose.filter.FilteredMessages; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; import lombok.AllArgsConstructor; import java.util.List; diff --git a/src/main/java/io/odpf/firehose/consumer/FirehoseSyncConsumer.java b/src/main/java/org/raystack/firehose/consumer/FirehoseSyncConsumer.java similarity index 77% rename from src/main/java/io/odpf/firehose/consumer/FirehoseSyncConsumer.java rename to src/main/java/org/raystack/firehose/consumer/FirehoseSyncConsumer.java index ff96a1cfb..642247305 100644 --- a/src/main/java/io/odpf/firehose/consumer/FirehoseSyncConsumer.java +++ b/src/main/java/org/raystack/firehose/consumer/FirehoseSyncConsumer.java @@ -1,13 +1,13 @@ -package io.odpf.firehose.consumer; +package org.raystack.firehose.consumer; -import io.odpf.firehose.consumer.kafka.ConsumerAndOffsetManager; -import io.odpf.firehose.exception.FirehoseConsumerFailedException; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.filter.FilterException; -import io.odpf.firehose.filter.FilteredMessages; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.Sink; -import io.odpf.firehose.tracer.SinkTracer; +import org.raystack.firehose.consumer.kafka.ConsumerAndOffsetManager; +import org.raystack.firehose.exception.FirehoseConsumerFailedException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.filter.FilterException; +import org.raystack.firehose.filter.FilteredMessages; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.Sink; +import org.raystack.firehose.tracer.SinkTracer; import io.opentracing.Span; import lombok.AllArgsConstructor; @@ -15,7 +15,7 @@ import java.time.Instant; import java.util.List; -import static io.odpf.firehose.metrics.Metrics.SOURCE_KAFKA_PARTITIONS_PROCESS_TIME_MILLISECONDS; +import static org.raystack.firehose.metrics.Metrics.SOURCE_KAFKA_PARTITIONS_PROCESS_TIME_MILLISECONDS; /** * Firehose consumer reads messages from Generic consumer and pushes messages to the configured sink. @@ -55,9 +55,9 @@ public void process() throws IOException { @Override public void close() throws IOException { + sink.close(); tracer.close(); consumerAndOffsetManager.close(); firehoseInstrumentation.close(); - sink.close(); } } diff --git a/src/main/java/io/odpf/firehose/consumer/kafka/ConsumerAndOffsetManager.java b/src/main/java/org/raystack/firehose/consumer/kafka/ConsumerAndOffsetManager.java similarity index 93% rename from src/main/java/io/odpf/firehose/consumer/kafka/ConsumerAndOffsetManager.java rename to src/main/java/org/raystack/firehose/consumer/kafka/ConsumerAndOffsetManager.java index 161b30855..96b00a7fc 100644 --- a/src/main/java/io/odpf/firehose/consumer/kafka/ConsumerAndOffsetManager.java +++ b/src/main/java/org/raystack/firehose/consumer/kafka/ConsumerAndOffsetManager.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.consumer.kafka; +package org.raystack.firehose.consumer.kafka; -import io.odpf.firehose.config.KafkaConsumerConfig; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.Sink; +import org.raystack.firehose.config.KafkaConsumerConfig; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.Sink; import java.io.IOException; import java.util.List; diff --git a/src/main/java/io/odpf/firehose/consumer/kafka/FirehoseKafkaConsumer.java b/src/main/java/org/raystack/firehose/consumer/kafka/FirehoseKafkaConsumer.java similarity index 91% rename from src/main/java/io/odpf/firehose/consumer/kafka/FirehoseKafkaConsumer.java rename to src/main/java/org/raystack/firehose/consumer/kafka/FirehoseKafkaConsumer.java index 8dd38f4df..7cbe3244e 100644 --- a/src/main/java/io/odpf/firehose/consumer/kafka/FirehoseKafkaConsumer.java +++ b/src/main/java/org/raystack/firehose/consumer/kafka/FirehoseKafkaConsumer.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.consumer.kafka; +package org.raystack.firehose.consumer.kafka; -import io.odpf.firehose.config.KafkaConsumerConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.metrics.Metrics; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.config.KafkaConsumerConfig; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.message.Message; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; @@ -17,9 +17,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; -import static io.odpf.firehose.metrics.Metrics.FAILURE_TAG; -import static io.odpf.firehose.metrics.Metrics.SOURCE_KAFKA_MESSAGES_COMMIT_TOTAL; -import static io.odpf.firehose.metrics.Metrics.SUCCESS_TAG; +import static org.raystack.firehose.metrics.Metrics.FAILURE_TAG; +import static org.raystack.firehose.metrics.Metrics.SOURCE_KAFKA_MESSAGES_COMMIT_TOTAL; +import static org.raystack.firehose.metrics.Metrics.SUCCESS_TAG; /** * A class responsible for consuming and committing kafka records. diff --git a/src/main/java/io/odpf/firehose/consumer/kafka/OffsetManager.java b/src/main/java/org/raystack/firehose/consumer/kafka/OffsetManager.java similarity index 98% rename from src/main/java/io/odpf/firehose/consumer/kafka/OffsetManager.java rename to src/main/java/org/raystack/firehose/consumer/kafka/OffsetManager.java index 6799a4448..0fa4ccf6e 100644 --- a/src/main/java/io/odpf/firehose/consumer/kafka/OffsetManager.java +++ b/src/main/java/org/raystack/firehose/consumer/kafka/OffsetManager.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.consumer.kafka; +package org.raystack.firehose.consumer.kafka; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.message.Message; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; diff --git a/src/main/java/io/odpf/firehose/consumer/kafka/OffsetNode.java b/src/main/java/org/raystack/firehose/consumer/kafka/OffsetNode.java similarity index 92% rename from src/main/java/io/odpf/firehose/consumer/kafka/OffsetNode.java rename to src/main/java/org/raystack/firehose/consumer/kafka/OffsetNode.java index 0ac6f9327..7bf87415e 100644 --- a/src/main/java/io/odpf/firehose/consumer/kafka/OffsetNode.java +++ b/src/main/java/org/raystack/firehose/consumer/kafka/OffsetNode.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.consumer.kafka; +package org.raystack.firehose.consumer.kafka; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/src/main/java/io/odpf/firehose/converter/ProtoTimeConverter.java b/src/main/java/org/raystack/firehose/converter/ProtoTimeConverter.java similarity index 91% rename from src/main/java/io/odpf/firehose/converter/ProtoTimeConverter.java rename to src/main/java/org/raystack/firehose/converter/ProtoTimeConverter.java index 0a487d976..0e20e498f 100644 --- a/src/main/java/io/odpf/firehose/converter/ProtoTimeConverter.java +++ b/src/main/java/org/raystack/firehose/converter/ProtoTimeConverter.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.converter; +package org.raystack.firehose.converter; import com.google.protobuf.Timestamp; diff --git a/src/main/java/io/odpf/firehose/error/ErrorHandler.java b/src/main/java/org/raystack/firehose/error/ErrorHandler.java similarity index 87% rename from src/main/java/io/odpf/firehose/error/ErrorHandler.java rename to src/main/java/org/raystack/firehose/error/ErrorHandler.java index 685c85259..e7685db40 100644 --- a/src/main/java/io/odpf/firehose/error/ErrorHandler.java +++ b/src/main/java/org/raystack/firehose/error/ErrorHandler.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.error; +package org.raystack.firehose.error; -import io.odpf.depot.error.ErrorType; -import io.odpf.firehose.config.ErrorConfig; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.config.ErrorConfig; +import org.raystack.firehose.message.Message; +import org.raystack.depot.error.ErrorType; import lombok.AllArgsConstructor; import java.util.List; diff --git a/src/main/java/io/odpf/firehose/error/ErrorScope.java b/src/main/java/org/raystack/firehose/error/ErrorScope.java similarity index 60% rename from src/main/java/io/odpf/firehose/error/ErrorScope.java rename to src/main/java/org/raystack/firehose/error/ErrorScope.java index 5689fd39b..43550fc7a 100644 --- a/src/main/java/io/odpf/firehose/error/ErrorScope.java +++ b/src/main/java/org/raystack/firehose/error/ErrorScope.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.error; +package org.raystack.firehose.error; public enum ErrorScope { DLQ, diff --git a/src/main/java/io/odpf/firehose/exception/ConfigurationException.java b/src/main/java/org/raystack/firehose/exception/ConfigurationException.java similarity index 89% rename from src/main/java/io/odpf/firehose/exception/ConfigurationException.java rename to src/main/java/org/raystack/firehose/exception/ConfigurationException.java index 79b121b25..a9db364e2 100644 --- a/src/main/java/io/odpf/firehose/exception/ConfigurationException.java +++ b/src/main/java/org/raystack/firehose/exception/ConfigurationException.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.exception; +package org.raystack.firehose.exception; /** * This exception is thrown when there is invalid configuration encountered. diff --git a/src/main/java/io/odpf/firehose/exception/DefaultException.java b/src/main/java/org/raystack/firehose/exception/DefaultException.java similarity index 87% rename from src/main/java/io/odpf/firehose/exception/DefaultException.java rename to src/main/java/org/raystack/firehose/exception/DefaultException.java index dc63177a1..85f67c81f 100644 --- a/src/main/java/io/odpf/firehose/exception/DefaultException.java +++ b/src/main/java/org/raystack/firehose/exception/DefaultException.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.exception; +package org.raystack.firehose.exception; import lombok.EqualsAndHashCode; diff --git a/src/main/java/io/odpf/firehose/exception/DeserializerException.java b/src/main/java/org/raystack/firehose/exception/DeserializerException.java similarity index 89% rename from src/main/java/io/odpf/firehose/exception/DeserializerException.java rename to src/main/java/org/raystack/firehose/exception/DeserializerException.java index 2d7c5723f..7d82b9088 100644 --- a/src/main/java/io/odpf/firehose/exception/DeserializerException.java +++ b/src/main/java/org/raystack/firehose/exception/DeserializerException.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.exception; +package org.raystack.firehose.exception; /** * Deserializer exception is thrown when message from proto is not deserializable into the Java object. diff --git a/src/main/java/io/odpf/firehose/exception/EmptyMessageException.java b/src/main/java/org/raystack/firehose/exception/EmptyMessageException.java similarity index 84% rename from src/main/java/io/odpf/firehose/exception/EmptyMessageException.java rename to src/main/java/org/raystack/firehose/exception/EmptyMessageException.java index c13fe4576..d344d24a9 100644 --- a/src/main/java/io/odpf/firehose/exception/EmptyMessageException.java +++ b/src/main/java/org/raystack/firehose/exception/EmptyMessageException.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.exception; +package org.raystack.firehose.exception; /** * Empty thrown when the message is contains zero bytes. diff --git a/src/main/java/io/odpf/firehose/exception/FirehoseConsumerFailedException.java b/src/main/java/org/raystack/firehose/exception/FirehoseConsumerFailedException.java similarity index 79% rename from src/main/java/io/odpf/firehose/exception/FirehoseConsumerFailedException.java rename to src/main/java/org/raystack/firehose/exception/FirehoseConsumerFailedException.java index 9c3ea12ac..e9b987cb2 100644 --- a/src/main/java/io/odpf/firehose/exception/FirehoseConsumerFailedException.java +++ b/src/main/java/org/raystack/firehose/exception/FirehoseConsumerFailedException.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.exception; +package org.raystack.firehose.exception; public class FirehoseConsumerFailedException extends RuntimeException { public FirehoseConsumerFailedException(Throwable th) { diff --git a/src/main/java/io/odpf/firehose/exception/JsonParseException.java b/src/main/java/org/raystack/firehose/exception/JsonParseException.java similarity index 80% rename from src/main/java/io/odpf/firehose/exception/JsonParseException.java rename to src/main/java/org/raystack/firehose/exception/JsonParseException.java index ba8667558..6bd316ff1 100644 --- a/src/main/java/io/odpf/firehose/exception/JsonParseException.java +++ b/src/main/java/org/raystack/firehose/exception/JsonParseException.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.exception; +package org.raystack.firehose.exception; public class JsonParseException extends RuntimeException { public JsonParseException(String message, Throwable cause) { diff --git a/src/main/java/io/odpf/firehose/exception/NeedToRetry.java b/src/main/java/org/raystack/firehose/exception/NeedToRetry.java similarity index 81% rename from src/main/java/io/odpf/firehose/exception/NeedToRetry.java rename to src/main/java/org/raystack/firehose/exception/NeedToRetry.java index c388daba3..c6c2608df 100644 --- a/src/main/java/io/odpf/firehose/exception/NeedToRetry.java +++ b/src/main/java/org/raystack/firehose/exception/NeedToRetry.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.exception; +package org.raystack.firehose.exception; public class NeedToRetry extends Exception { public NeedToRetry(String statusCode) { diff --git a/src/main/java/io/odpf/firehose/exception/OAuth2Exception.java b/src/main/java/org/raystack/firehose/exception/OAuth2Exception.java similarity index 79% rename from src/main/java/io/odpf/firehose/exception/OAuth2Exception.java rename to src/main/java/org/raystack/firehose/exception/OAuth2Exception.java index 8352f2625..1e428fc82 100644 --- a/src/main/java/io/odpf/firehose/exception/OAuth2Exception.java +++ b/src/main/java/org/raystack/firehose/exception/OAuth2Exception.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.exception; +package org.raystack.firehose.exception; import java.io.IOException; diff --git a/src/main/java/io/odpf/firehose/exception/SinkException.java b/src/main/java/org/raystack/firehose/exception/SinkException.java similarity index 78% rename from src/main/java/io/odpf/firehose/exception/SinkException.java rename to src/main/java/org/raystack/firehose/exception/SinkException.java index f5629ae6a..b83cc1248 100644 --- a/src/main/java/io/odpf/firehose/exception/SinkException.java +++ b/src/main/java/org/raystack/firehose/exception/SinkException.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.exception; +package org.raystack.firehose.exception; public class SinkException extends RuntimeException { public SinkException(String message, Throwable cause) { diff --git a/src/main/java/io/odpf/firehose/exception/SinkTaskFailedException.java b/src/main/java/org/raystack/firehose/exception/SinkTaskFailedException.java similarity index 79% rename from src/main/java/io/odpf/firehose/exception/SinkTaskFailedException.java rename to src/main/java/org/raystack/firehose/exception/SinkTaskFailedException.java index 02ce68130..8d8902827 100644 --- a/src/main/java/io/odpf/firehose/exception/SinkTaskFailedException.java +++ b/src/main/java/org/raystack/firehose/exception/SinkTaskFailedException.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.exception; +package org.raystack.firehose.exception; public class SinkTaskFailedException extends RuntimeException { public SinkTaskFailedException(Throwable throwable) { diff --git a/src/main/java/io/odpf/firehose/exception/UnknownFieldsException.java b/src/main/java/org/raystack/firehose/exception/UnknownFieldsException.java similarity index 96% rename from src/main/java/io/odpf/firehose/exception/UnknownFieldsException.java rename to src/main/java/org/raystack/firehose/exception/UnknownFieldsException.java index 30ed0f64f..530c2f61c 100644 --- a/src/main/java/io/odpf/firehose/exception/UnknownFieldsException.java +++ b/src/main/java/org/raystack/firehose/exception/UnknownFieldsException.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.exception; +package org.raystack.firehose.exception; import com.google.protobuf.DynamicMessage; diff --git a/src/main/java/io/odpf/firehose/filter/Filter.java b/src/main/java/org/raystack/firehose/filter/Filter.java similarity index 83% rename from src/main/java/io/odpf/firehose/filter/Filter.java rename to src/main/java/org/raystack/firehose/filter/Filter.java index bcda398fe..5e1e25663 100644 --- a/src/main/java/io/odpf/firehose/filter/Filter.java +++ b/src/main/java/org/raystack/firehose/filter/Filter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.filter; +package org.raystack.firehose.filter; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.message.Message; import java.util.List; diff --git a/src/main/java/io/odpf/firehose/filter/FilterException.java b/src/main/java/org/raystack/firehose/filter/FilterException.java similarity index 85% rename from src/main/java/io/odpf/firehose/filter/FilterException.java rename to src/main/java/org/raystack/firehose/filter/FilterException.java index ed50c4431..4705d148a 100644 --- a/src/main/java/io/odpf/firehose/filter/FilterException.java +++ b/src/main/java/org/raystack/firehose/filter/FilterException.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.filter; +package org.raystack.firehose.filter; public class FilterException extends Exception { diff --git a/src/main/java/io/odpf/firehose/filter/FilteredMessages.java b/src/main/java/org/raystack/firehose/filter/FilteredMessages.java similarity index 88% rename from src/main/java/io/odpf/firehose/filter/FilteredMessages.java rename to src/main/java/org/raystack/firehose/filter/FilteredMessages.java index 1d59362d6..781eb2676 100644 --- a/src/main/java/io/odpf/firehose/filter/FilteredMessages.java +++ b/src/main/java/org/raystack/firehose/filter/FilteredMessages.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.filter; +package org.raystack.firehose.filter; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.message.Message; import lombok.EqualsAndHashCode; import lombok.Getter; diff --git a/src/main/java/io/odpf/firehose/filter/NoOpFilter.java b/src/main/java/org/raystack/firehose/filter/NoOpFilter.java similarity index 83% rename from src/main/java/io/odpf/firehose/filter/NoOpFilter.java rename to src/main/java/org/raystack/firehose/filter/NoOpFilter.java index a8f1af431..a8b630165 100644 --- a/src/main/java/io/odpf/firehose/filter/NoOpFilter.java +++ b/src/main/java/org/raystack/firehose/filter/NoOpFilter.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.filter; +package org.raystack.firehose.filter; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import java.util.List; diff --git a/src/main/java/io/odpf/firehose/filter/jexl/JexlFilter.java b/src/main/java/org/raystack/firehose/filter/jexl/JexlFilter.java similarity index 90% rename from src/main/java/io/odpf/firehose/filter/jexl/JexlFilter.java rename to src/main/java/org/raystack/firehose/filter/jexl/JexlFilter.java index acab2d856..016cfb46f 100644 --- a/src/main/java/io/odpf/firehose/filter/jexl/JexlFilter.java +++ b/src/main/java/org/raystack/firehose/filter/jexl/JexlFilter.java @@ -1,12 +1,12 @@ -package io.odpf.firehose.filter.jexl; +package org.raystack.firehose.filter.jexl; -import io.odpf.firehose.config.FilterConfig; -import io.odpf.firehose.config.enums.FilterDataSourceType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.filter.Filter; -import io.odpf.firehose.filter.FilterException; -import io.odpf.firehose.filter.FilteredMessages; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.config.FilterConfig; +import org.raystack.firehose.config.enums.FilterDataSourceType; +import org.raystack.firehose.filter.Filter; +import org.raystack.firehose.filter.FilterException; +import org.raystack.firehose.filter.FilteredMessages; import org.apache.commons.jexl2.Expression; import org.apache.commons.jexl2.JexlContext; import org.apache.commons.jexl2.JexlEngine; diff --git a/src/main/java/io/odpf/firehose/filter/json/JsonFilter.java b/src/main/java/org/raystack/firehose/filter/json/JsonFilter.java similarity index 87% rename from src/main/java/io/odpf/firehose/filter/json/JsonFilter.java rename to src/main/java/org/raystack/firehose/filter/json/JsonFilter.java index 0ba501ee0..ddc389bf4 100644 --- a/src/main/java/io/odpf/firehose/filter/json/JsonFilter.java +++ b/src/main/java/org/raystack/firehose/filter/json/JsonFilter.java @@ -1,29 +1,29 @@ -package io.odpf.firehose.filter.json; +package org.raystack.firehose.filter.json; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.protobuf.DynamicMessage; import com.google.protobuf.util.JsonFormat; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import com.networknt.schema.JsonSchema; import com.networknt.schema.JsonSchemaFactory; import com.networknt.schema.SpecVersion; import com.networknt.schema.ValidationMessage; -import io.odpf.firehose.config.FilterConfig; -import io.odpf.firehose.config.enums.FilterMessageFormatType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.filter.Filter; -import io.odpf.firehose.filter.FilterException; -import io.odpf.firehose.filter.FilteredMessages; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.stencil.client.StencilClient; -import io.odpf.stencil.Parser; +import org.raystack.firehose.config.FilterConfig; +import org.raystack.firehose.config.enums.FilterMessageFormatType; +import org.raystack.firehose.filter.Filter; +import org.raystack.firehose.filter.FilterException; +import org.raystack.firehose.filter.FilteredMessages; +import org.raystack.stencil.client.StencilClient; +import org.raystack.stencil.Parser; import java.nio.charset.Charset; import java.util.List; import java.util.Set; -import static io.odpf.firehose.config.enums.FilterDataSourceType.KEY; +import static org.raystack.firehose.config.enums.FilterDataSourceType.KEY; /** * JSON-based filter to filter protobuf/JSON messages based on rules diff --git a/src/main/java/io/odpf/firehose/filter/json/JsonFilterUtil.java b/src/main/java/org/raystack/firehose/filter/json/JsonFilterUtil.java similarity index 89% rename from src/main/java/io/odpf/firehose/filter/json/JsonFilterUtil.java rename to src/main/java/org/raystack/firehose/filter/json/JsonFilterUtil.java index 198bee9d5..21a963d75 100644 --- a/src/main/java/io/odpf/firehose/filter/json/JsonFilterUtil.java +++ b/src/main/java/org/raystack/firehose/filter/json/JsonFilterUtil.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.filter.json; +package org.raystack.firehose.filter.json; -import io.odpf.firehose.config.FilterConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.config.FilterConfig; import lombok.experimental.UtilityClass; -import static io.odpf.firehose.config.enums.FilterMessageFormatType.PROTOBUF; +import static org.raystack.firehose.config.enums.FilterMessageFormatType.PROTOBUF; /** * The type Json filter util. diff --git a/src/main/java/io/odpf/firehose/launch/Main.java b/src/main/java/org/raystack/firehose/launch/Main.java similarity index 83% rename from src/main/java/io/odpf/firehose/launch/Main.java rename to src/main/java/org/raystack/firehose/launch/Main.java index 541b5fe07..70e9b9e75 100644 --- a/src/main/java/io/odpf/firehose/launch/Main.java +++ b/src/main/java/org/raystack/firehose/launch/Main.java @@ -1,13 +1,13 @@ -package io.odpf.firehose.launch; +package org.raystack.firehose.launch; -import io.odpf.depot.config.MetricsConfig; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.depot.metrics.StatsDReporterBuilder; -import io.odpf.firehose.config.KafkaConsumerConfig; -import io.odpf.firehose.consumer.FirehoseConsumer; -import io.odpf.firehose.consumer.FirehoseConsumerFactory; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.metrics.Metrics; +import org.raystack.firehose.consumer.FirehoseConsumer; +import org.raystack.firehose.consumer.FirehoseConsumerFactory; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.depot.config.MetricsConfig; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.depot.metrics.StatsDReporterBuilder; +import org.raystack.firehose.config.KafkaConsumerConfig; import org.aeonbits.owner.ConfigFactory; import java.io.IOException; @@ -54,6 +54,7 @@ private static void multiThreadedConsumers(KafkaConsumerConfig kafkaConsumerConf firehoseConsumer.process(); } } catch (Exception | Error e) { + ensureThreadInterruptStateIsClearedAndClose(firehoseConsumer, firehoseInstrumentation); firehoseInstrumentation.captureFatalError("firehose_error_event", e, "Caught exception or error, exiting the application"); System.exit(1); } finally { @@ -73,9 +74,10 @@ private static void multiThreadedConsumers(KafkaConsumerConfig kafkaConsumerConf } private static void ensureThreadInterruptStateIsClearedAndClose(FirehoseConsumer firehoseConsumer, FirehoseInstrumentation firehoseInstrumentation) { - Thread.interrupted(); try { - firehoseConsumer.close(); + if (firehoseConsumer != null) { + firehoseConsumer.close(); + } } catch (IOException e) { firehoseInstrumentation.captureFatalError("firehose_error_event", e, "Exception on closing firehose consumer"); } diff --git a/src/main/java/io/odpf/firehose/launch/Task.java b/src/main/java/org/raystack/firehose/launch/Task.java similarity index 96% rename from src/main/java/io/odpf/firehose/launch/Task.java rename to src/main/java/org/raystack/firehose/launch/Task.java index 35e087e79..db4590fee 100644 --- a/src/main/java/io/odpf/firehose/launch/Task.java +++ b/src/main/java/org/raystack/firehose/launch/Task.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.launch; +package org.raystack.firehose.launch; import java.util.ArrayList; import java.util.List; @@ -8,7 +8,7 @@ import java.util.concurrent.Future; import java.util.function.Consumer; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.FirehoseInstrumentation; /** * The Task with parallelism. diff --git a/src/main/java/io/odpf/firehose/message/FirehoseMessageUtils.java b/src/main/java/org/raystack/firehose/message/FirehoseMessageUtils.java similarity index 76% rename from src/main/java/io/odpf/firehose/message/FirehoseMessageUtils.java rename to src/main/java/org/raystack/firehose/message/FirehoseMessageUtils.java index b23f8ebe1..8590884c4 100644 --- a/src/main/java/io/odpf/firehose/message/FirehoseMessageUtils.java +++ b/src/main/java/org/raystack/firehose/message/FirehoseMessageUtils.java @@ -1,16 +1,15 @@ -package io.odpf.firehose.message; +package org.raystack.firehose.message; -import io.odpf.depot.common.Tuple; -import io.odpf.depot.message.OdpfMessage; +import org.raystack.depot.common.Tuple; import java.util.List; import java.util.stream.Collectors; public class FirehoseMessageUtils { - public static List convertToOdpfMessage(List messages) { + public static List convertToDepotMessage(List messages) { return messages.stream().map(message -> - new OdpfMessage( + new org.raystack.depot.message.Message( message.getLogKey(), message.getLogMessage(), new Tuple<>("message_topic", message.getTopic()), diff --git a/src/main/java/io/odpf/firehose/message/Message.java b/src/main/java/org/raystack/firehose/message/Message.java similarity index 94% rename from src/main/java/io/odpf/firehose/message/Message.java rename to src/main/java/org/raystack/firehose/message/Message.java index 131d93f4e..ae0e5afc4 100644 --- a/src/main/java/io/odpf/firehose/message/Message.java +++ b/src/main/java/org/raystack/firehose/message/Message.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.message; +package org.raystack.firehose.message; -import io.odpf.depot.error.ErrorInfo; -import io.odpf.depot.error.ErrorType; -import io.odpf.firehose.exception.DefaultException; +import org.raystack.firehose.exception.DefaultException; +import org.raystack.depot.error.ErrorInfo; +import org.raystack.depot.error.ErrorType; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; diff --git a/src/main/java/io/odpf/firehose/metrics/BigQueryMetrics.java b/src/main/java/org/raystack/firehose/metrics/BigQueryMetrics.java similarity index 96% rename from src/main/java/io/odpf/firehose/metrics/BigQueryMetrics.java rename to src/main/java/org/raystack/firehose/metrics/BigQueryMetrics.java index 89e3a446c..4657016f3 100644 --- a/src/main/java/io/odpf/firehose/metrics/BigQueryMetrics.java +++ b/src/main/java/org/raystack/firehose/metrics/BigQueryMetrics.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.metrics; +package org.raystack.firehose.metrics; public class BigQueryMetrics { public enum BigQueryAPIType { diff --git a/src/main/java/io/odpf/firehose/metrics/BlobStorageMetrics.java b/src/main/java/org/raystack/firehose/metrics/BlobStorageMetrics.java similarity index 97% rename from src/main/java/io/odpf/firehose/metrics/BlobStorageMetrics.java rename to src/main/java/org/raystack/firehose/metrics/BlobStorageMetrics.java index 9bbc78187..f65249fd7 100644 --- a/src/main/java/io/odpf/firehose/metrics/BlobStorageMetrics.java +++ b/src/main/java/org/raystack/firehose/metrics/BlobStorageMetrics.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.metrics; +package org.raystack.firehose.metrics; public class BlobStorageMetrics { diff --git a/src/main/java/io/odpf/firehose/metrics/FirehoseInstrumentation.java b/src/main/java/org/raystack/firehose/metrics/FirehoseInstrumentation.java similarity index 79% rename from src/main/java/io/odpf/firehose/metrics/FirehoseInstrumentation.java rename to src/main/java/org/raystack/firehose/metrics/FirehoseInstrumentation.java index 3652269a5..597eab121 100644 --- a/src/main/java/io/odpf/firehose/metrics/FirehoseInstrumentation.java +++ b/src/main/java/org/raystack/firehose/metrics/FirehoseInstrumentation.java @@ -1,27 +1,27 @@ -package io.odpf.firehose.metrics; +package org.raystack.firehose.metrics; -import io.odpf.depot.error.ErrorType; -import io.odpf.depot.metrics.Instrumentation; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.message.Message; +import org.raystack.depot.error.ErrorType; +import org.raystack.depot.metrics.Instrumentation; +import org.raystack.depot.metrics.StatsDReporter; import org.slf4j.Logger; import java.io.IOException; import java.time.Instant; import java.util.List; -import static io.odpf.firehose.metrics.Metrics.ERROR_MESSAGES_TOTAL; -import static io.odpf.firehose.metrics.Metrics.ERROR_TYPE_TAG; -import static io.odpf.firehose.metrics.Metrics.GLOBAL_MESSAGES_TOTAL; -import static io.odpf.firehose.metrics.Metrics.MESSAGE_SCOPE_TAG; -import static io.odpf.firehose.metrics.Metrics.MESSAGE_TYPE_TAG; -import static io.odpf.firehose.metrics.Metrics.MessageType; -import static io.odpf.firehose.metrics.Metrics.PIPELINE_END_LATENCY_MILLISECONDS; -import static io.odpf.firehose.metrics.Metrics.PIPELINE_EXECUTION_LIFETIME_MILLISECONDS; -import static io.odpf.firehose.metrics.Metrics.SINK_PUSH_BATCH_SIZE_TOTAL; -import static io.odpf.firehose.metrics.Metrics.SINK_RESPONSE_TIME_MILLISECONDS; -import static io.odpf.firehose.metrics.Metrics.SOURCE_KAFKA_MESSAGES_FILTER_TOTAL; -import static io.odpf.firehose.metrics.Metrics.SOURCE_KAFKA_PULL_BATCH_SIZE_TOTAL; +import static org.raystack.firehose.metrics.Metrics.ERROR_MESSAGES_TOTAL; +import static org.raystack.firehose.metrics.Metrics.ERROR_TYPE_TAG; +import static org.raystack.firehose.metrics.Metrics.GLOBAL_MESSAGES_TOTAL; +import static org.raystack.firehose.metrics.Metrics.MESSAGE_SCOPE_TAG; +import static org.raystack.firehose.metrics.Metrics.MESSAGE_TYPE_TAG; +import static org.raystack.firehose.metrics.Metrics.MessageType; +import static org.raystack.firehose.metrics.Metrics.PIPELINE_END_LATENCY_MILLISECONDS; +import static org.raystack.firehose.metrics.Metrics.PIPELINE_EXECUTION_LIFETIME_MILLISECONDS; +import static org.raystack.firehose.metrics.Metrics.SINK_PUSH_BATCH_SIZE_TOTAL; +import static org.raystack.firehose.metrics.Metrics.SINK_RESPONSE_TIME_MILLISECONDS; +import static org.raystack.firehose.metrics.Metrics.SOURCE_KAFKA_MESSAGES_FILTER_TOTAL; +import static org.raystack.firehose.metrics.Metrics.SOURCE_KAFKA_PULL_BATCH_SIZE_TOTAL; /** * Instrumentation. diff --git a/src/main/java/io/odpf/firehose/metrics/Metrics.java b/src/main/java/org/raystack/firehose/metrics/Metrics.java similarity index 99% rename from src/main/java/io/odpf/firehose/metrics/Metrics.java rename to src/main/java/org/raystack/firehose/metrics/Metrics.java index 2fab203fd..3d837ac9e 100644 --- a/src/main/java/io/odpf/firehose/metrics/Metrics.java +++ b/src/main/java/org/raystack/firehose/metrics/Metrics.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.metrics; +package org.raystack.firehose.metrics; public class Metrics { //APPLICATION PREFIX diff --git a/src/main/java/io/odpf/firehose/parser/KafkaEnvironmentVariables.java b/src/main/java/org/raystack/firehose/parser/KafkaEnvironmentVariables.java similarity index 96% rename from src/main/java/io/odpf/firehose/parser/KafkaEnvironmentVariables.java rename to src/main/java/org/raystack/firehose/parser/KafkaEnvironmentVariables.java index 1d1dc220b..386ad4732 100644 --- a/src/main/java/io/odpf/firehose/parser/KafkaEnvironmentVariables.java +++ b/src/main/java/org/raystack/firehose/parser/KafkaEnvironmentVariables.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.parser; +package org.raystack.firehose.parser; import java.util.Collections; import java.util.Map; diff --git a/src/main/java/io/odpf/firehose/proto/ProtoMessage.java b/src/main/java/org/raystack/firehose/proto/ProtoMessage.java similarity index 90% rename from src/main/java/io/odpf/firehose/proto/ProtoMessage.java rename to src/main/java/org/raystack/firehose/proto/ProtoMessage.java index 005d870d3..122f79b8a 100644 --- a/src/main/java/io/odpf/firehose/proto/ProtoMessage.java +++ b/src/main/java/org/raystack/firehose/proto/ProtoMessage.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.proto; +package org.raystack.firehose.proto; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.exception.ConfigurationException; +import org.raystack.firehose.exception.ConfigurationException; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; import com.google.protobuf.Descriptors; import com.google.protobuf.GeneratedMessageV3; diff --git a/src/main/java/io/odpf/firehose/proto/ProtoToFieldMapper.java b/src/main/java/org/raystack/firehose/proto/ProtoToFieldMapper.java similarity index 95% rename from src/main/java/io/odpf/firehose/proto/ProtoToFieldMapper.java rename to src/main/java/org/raystack/firehose/proto/ProtoToFieldMapper.java index fdc7c8dbf..39d9da5ca 100644 --- a/src/main/java/io/odpf/firehose/proto/ProtoToFieldMapper.java +++ b/src/main/java/org/raystack/firehose/proto/ProtoToFieldMapper.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.proto; +package org.raystack.firehose.proto; -import io.odpf.firehose.sink.jdbc.JdbcMapper; +import org.raystack.firehose.sink.jdbc.JdbcMapper; import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; -import io.odpf.stencil.Parser; +import org.raystack.stencil.Parser; import org.apache.http.util.Asserts; import java.util.Enumeration; diff --git a/src/main/java/io/odpf/firehose/proto/ProtoUtils.java b/src/main/java/org/raystack/firehose/proto/ProtoUtils.java similarity index 97% rename from src/main/java/io/odpf/firehose/proto/ProtoUtils.java rename to src/main/java/org/raystack/firehose/proto/ProtoUtils.java index f70f1cfc7..d170c3c49 100644 --- a/src/main/java/io/odpf/firehose/proto/ProtoUtils.java +++ b/src/main/java/org/raystack/firehose/proto/ProtoUtils.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.proto; +package org.raystack.firehose.proto; import com.google.protobuf.DynamicMessage; diff --git a/src/main/java/io/odpf/firehose/serializer/JsonWrappedProtoByte.java b/src/main/java/org/raystack/firehose/serializer/JsonWrappedProtoByte.java similarity index 86% rename from src/main/java/io/odpf/firehose/serializer/JsonWrappedProtoByte.java rename to src/main/java/org/raystack/firehose/serializer/JsonWrappedProtoByte.java index cc6c7dd5f..5e6243fbc 100644 --- a/src/main/java/io/odpf/firehose/serializer/JsonWrappedProtoByte.java +++ b/src/main/java/org/raystack/firehose/serializer/JsonWrappedProtoByte.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.serializer; +package org.raystack.firehose.serializer; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.exception.DeserializerException; import com.google.gson.Gson; import com.google.gson.GsonBuilder; diff --git a/src/main/java/io/odpf/firehose/serializer/MessageJsonSerializer.java b/src/main/java/org/raystack/firehose/serializer/MessageJsonSerializer.java similarity index 91% rename from src/main/java/io/odpf/firehose/serializer/MessageJsonSerializer.java rename to src/main/java/org/raystack/firehose/serializer/MessageJsonSerializer.java index f645205da..1b1399b55 100644 --- a/src/main/java/io/odpf/firehose/serializer/MessageJsonSerializer.java +++ b/src/main/java/org/raystack/firehose/serializer/MessageJsonSerializer.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.serializer; +package org.raystack.firehose.serializer; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.message.Message; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonSerializationContext; diff --git a/src/main/java/io/odpf/firehose/serializer/MessageSerializer.java b/src/main/java/org/raystack/firehose/serializer/MessageSerializer.java similarity index 70% rename from src/main/java/io/odpf/firehose/serializer/MessageSerializer.java rename to src/main/java/org/raystack/firehose/serializer/MessageSerializer.java index 4edddec9b..1a59fa0d1 100644 --- a/src/main/java/io/odpf/firehose/serializer/MessageSerializer.java +++ b/src/main/java/org/raystack/firehose/serializer/MessageSerializer.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.serializer; +package org.raystack.firehose.serializer; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.exception.DeserializerException; /** * Serializer serialize Message into string format. diff --git a/src/main/java/io/odpf/firehose/serializer/MessageToJson.java b/src/main/java/org/raystack/firehose/serializer/MessageToJson.java similarity index 96% rename from src/main/java/io/odpf/firehose/serializer/MessageToJson.java rename to src/main/java/org/raystack/firehose/serializer/MessageToJson.java index d17b34ce4..66d4773d8 100644 --- a/src/main/java/io/odpf/firehose/serializer/MessageToJson.java +++ b/src/main/java/org/raystack/firehose/serializer/MessageToJson.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.serializer; +package org.raystack.firehose.serializer; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.exception.DeserializerException; import com.google.gson.ExclusionStrategy; import com.google.gson.FieldAttributes; import com.google.gson.Gson; @@ -12,7 +12,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Timestamp; import com.google.protobuf.util.JsonFormat; -import io.odpf.stencil.Parser; +import org.raystack.stencil.Parser; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; diff --git a/src/main/java/io/odpf/firehose/serializer/MessageToTemplatizedJson.java b/src/main/java/org/raystack/firehose/serializer/MessageToTemplatizedJson.java similarity index 92% rename from src/main/java/io/odpf/firehose/serializer/MessageToTemplatizedJson.java rename to src/main/java/org/raystack/firehose/serializer/MessageToTemplatizedJson.java index 189064a42..d05872b2e 100644 --- a/src/main/java/io/odpf/firehose/serializer/MessageToTemplatizedJson.java +++ b/src/main/java/org/raystack/firehose/serializer/MessageToTemplatizedJson.java @@ -1,17 +1,17 @@ -package io.odpf.firehose.serializer; +package org.raystack.firehose.serializer; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.exception.ConfigurationException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.exception.ConfigurationException; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import com.google.gson.Gson; import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.PathNotFoundException; -import io.odpf.stencil.Parser; +import org.raystack.stencil.Parser; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; diff --git a/src/main/java/io/odpf/firehose/sink/AbstractSink.java b/src/main/java/org/raystack/firehose/sink/AbstractSink.java similarity index 88% rename from src/main/java/io/odpf/firehose/sink/AbstractSink.java rename to src/main/java/org/raystack/firehose/sink/AbstractSink.java index e97bafa38..90e6c89d1 100644 --- a/src/main/java/io/odpf/firehose/sink/AbstractSink.java +++ b/src/main/java/org/raystack/firehose/sink/AbstractSink.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sink; +package org.raystack.firehose.sink; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.exception.ConfigurationException; -import io.odpf.firehose.exception.SinkException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.metrics.Metrics; +import org.raystack.firehose.exception.ConfigurationException; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.exception.SinkException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; import lombok.AllArgsConstructor; import java.io.Closeable; @@ -14,8 +14,6 @@ import java.time.Instant; import java.util.List; -import static io.odpf.firehose.metrics.Metrics.SINK_MESSAGES_TOTAL; - /** * Abstract sink. * All other type of sink will implement this. @@ -70,7 +68,7 @@ private void processFailedMessages(List failedMessages) { firehoseInstrumentation.logError("Failed to Push {} messages to sink ", failedMessages.size()); failedMessages.forEach(m -> { m.setDefaultErrorIfNotPresent(); - firehoseInstrumentation.captureMessageMetrics(SINK_MESSAGES_TOTAL, Metrics.MessageType.FAILURE, m.getErrorInfo().getErrorType(), 1); + firehoseInstrumentation.captureMessageMetrics(Metrics.SINK_MESSAGES_TOTAL, Metrics.MessageType.FAILURE, m.getErrorInfo().getErrorType(), 1); firehoseInstrumentation.captureErrorMetrics(m.getErrorInfo().getErrorType()); firehoseInstrumentation.logError("Failed to Push message. Error: {},Topic: {}, Partition: {},Offset: {}", m.getErrorInfo().getErrorType(), diff --git a/src/main/java/io/odpf/firehose/sink/GenericOdpfSink.java b/src/main/java/org/raystack/firehose/sink/GenericSink.java similarity index 55% rename from src/main/java/io/odpf/firehose/sink/GenericOdpfSink.java rename to src/main/java/org/raystack/firehose/sink/GenericSink.java index 0bc698650..0bf8e57e4 100644 --- a/src/main/java/io/odpf/firehose/sink/GenericOdpfSink.java +++ b/src/main/java/org/raystack/firehose/sink/GenericSink.java @@ -1,12 +1,11 @@ -package io.odpf.firehose.sink; +package org.raystack.firehose.sink; -import io.odpf.depot.OdpfSink; -import io.odpf.depot.OdpfSinkResponse; -import io.odpf.depot.message.OdpfMessage; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.message.FirehoseMessageUtils; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.depot.Sink; +import org.raystack.depot.SinkResponse; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.FirehoseMessageUtils; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import java.io.IOException; import java.sql.SQLException; @@ -14,19 +13,19 @@ import java.util.List; import java.util.stream.Collectors; -public class GenericOdpfSink extends AbstractSink { +public class GenericSink extends AbstractSink { private final List messageList = new ArrayList<>(); - private final OdpfSink odpfSink; + private final Sink sink; - public GenericOdpfSink(FirehoseInstrumentation firehoseInstrumentation, String sinkType, OdpfSink odpfSink) { + public GenericSink(FirehoseInstrumentation firehoseInstrumentation, String sinkType, Sink sink) { super(firehoseInstrumentation, sinkType); - this.odpfSink = odpfSink; + this.sink = sink; } @Override protected List execute() throws Exception { - List odpfMessages = FirehoseMessageUtils.convertToOdpfMessage(messageList); - OdpfSinkResponse response = odpfSink.pushToSink(odpfMessages); + List messages = FirehoseMessageUtils.convertToDepotMessage(messageList); + SinkResponse response = sink.pushToSink(messages); return response.getErrors().keySet().stream() .map(index -> { Message message = messageList.get(index.intValue()); @@ -43,6 +42,6 @@ protected void prepare(List messages) throws DeserializerException, IOE @Override public void close() throws IOException { - + sink.close(); } } diff --git a/src/main/java/io/odpf/firehose/sink/Sink.java b/src/main/java/org/raystack/firehose/sink/Sink.java similarity index 87% rename from src/main/java/io/odpf/firehose/sink/Sink.java rename to src/main/java/org/raystack/firehose/sink/Sink.java index 5a8a0941e..81bc7bb64 100644 --- a/src/main/java/io/odpf/firehose/sink/Sink.java +++ b/src/main/java/org/raystack/firehose/sink/Sink.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink; +package org.raystack.firehose.sink; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.message.Message; import java.io.Closeable; import java.io.IOException; @@ -19,7 +19,6 @@ public interface Sink extends Closeable { * @param message list of {@see EsbMessage} * @return the list of failed messages * @throws IOException in case of error conditions while persisting it to the custom sink. - * @throws io.odpf.firehose.exception.DeserializerException in case of problems with deserialising the message into a protobuf object. */ List pushMessage(List message) throws IOException; diff --git a/src/main/java/io/odpf/firehose/sink/SinkFactory.java b/src/main/java/org/raystack/firehose/sink/SinkFactory.java similarity index 66% rename from src/main/java/io/odpf/firehose/sink/SinkFactory.java rename to src/main/java/org/raystack/firehose/sink/SinkFactory.java index 579283d89..bf40630df 100644 --- a/src/main/java/io/odpf/firehose/sink/SinkFactory.java +++ b/src/main/java/org/raystack/firehose/sink/SinkFactory.java @@ -1,32 +1,32 @@ -package io.odpf.firehose.sink; +package org.raystack.firehose.sink; -import io.odpf.depot.bigquery.BigQuerySink; -import io.odpf.depot.bigquery.BigQuerySinkFactory; -import io.odpf.depot.config.BigQuerySinkConfig; -import io.odpf.depot.config.RedisSinkConfig; -import io.odpf.depot.bigtable.BigTableSinkFactory; -import io.odpf.depot.bigtable.BigTableSink; -import io.odpf.depot.config.BigTableSinkConfig; -import io.odpf.depot.log.LogSink; -import io.odpf.depot.log.LogSinkFactory; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.depot.redis.RedisSink; -import io.odpf.depot.redis.RedisSinkFactory; -import io.odpf.firehose.config.KafkaConsumerConfig; -import io.odpf.firehose.config.enums.SinkType; -import io.odpf.firehose.consumer.kafka.OffsetManager; -import io.odpf.firehose.exception.ConfigurationException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.bigquery.BigquerySinkUtils; -import io.odpf.firehose.sink.blob.BlobSinkFactory; -import io.odpf.firehose.sink.elasticsearch.EsSinkFactory; -import io.odpf.firehose.sink.grpc.GrpcSinkFactory; -import io.odpf.firehose.sink.http.HttpSinkFactory; -import io.odpf.firehose.sink.influxdb.InfluxSinkFactory; -import io.odpf.firehose.sink.jdbc.JdbcSinkFactory; -import io.odpf.firehose.sink.mongodb.MongoSinkFactory; -import io.odpf.firehose.sink.prometheus.PromSinkFactory; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.config.KafkaConsumerConfig; +import org.raystack.firehose.config.enums.SinkType; +import org.raystack.firehose.consumer.kafka.OffsetManager; +import org.raystack.firehose.exception.ConfigurationException; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.bigquery.BigquerySinkUtils; +import org.raystack.firehose.sink.blob.BlobSinkFactory; +import org.raystack.firehose.sink.elasticsearch.EsSinkFactory; +import org.raystack.firehose.sink.grpc.GrpcSinkFactory; +import org.raystack.firehose.sink.http.HttpSinkFactory; +import org.raystack.firehose.sink.influxdb.InfluxSinkFactory; +import org.raystack.firehose.sink.jdbc.JdbcSinkFactory; +import org.raystack.firehose.sink.mongodb.MongoSinkFactory; +import org.raystack.firehose.sink.prometheus.PromSinkFactory; +import org.raystack.depot.bigquery.BigQuerySink; +import org.raystack.depot.bigquery.BigQuerySinkFactory; +import org.raystack.depot.config.BigQuerySinkConfig; +import org.raystack.depot.config.RedisSinkConfig; +import org.raystack.depot.bigtable.BigTableSinkFactory; +import org.raystack.depot.bigtable.BigTableSink; +import org.raystack.depot.config.BigTableSinkConfig; +import org.raystack.depot.log.LogSink; +import org.raystack.depot.log.LogSinkFactory; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.depot.redis.RedisSink; +import org.raystack.depot.redis.RedisSinkFactory; +import org.raystack.stencil.client.StencilClient; import org.aeonbits.owner.ConfigFactory; import java.util.Map; @@ -37,11 +37,11 @@ public class SinkFactory { private final FirehoseInstrumentation firehoseInstrumentation; private final StencilClient stencilClient; private final OffsetManager offsetManager; + private final Map config; private BigQuerySinkFactory bigQuerySinkFactory; private BigTableSinkFactory bigTableSinkFactory; private LogSinkFactory logSinkFactory; private RedisSinkFactory redisSinkFactory; - private final Map config; public SinkFactory(KafkaConsumerConfig kafkaConsumerConfig, StatsDReporter statsDReporter, @@ -109,11 +109,11 @@ public Sink getSink() { case INFLUXDB: return InfluxSinkFactory.create(config, statsDReporter, stencilClient); case LOG: - return new GenericOdpfSink(new FirehoseInstrumentation(statsDReporter, LogSink.class), sinkType.name(), logSinkFactory.create()); + return new GenericSink(new FirehoseInstrumentation(statsDReporter, LogSink.class), sinkType.name(), logSinkFactory.create()); case ELASTICSEARCH: return EsSinkFactory.create(config, statsDReporter, stencilClient); case REDIS: - return new GenericOdpfSink(new FirehoseInstrumentation(statsDReporter, RedisSink.class), sinkType.name(), redisSinkFactory.create()); + return new GenericSink(new FirehoseInstrumentation(statsDReporter, RedisSink.class), sinkType.name(), redisSinkFactory.create()); case GRPC: return GrpcSinkFactory.create(config, statsDReporter, stencilClient); case PROMETHEUS: @@ -121,9 +121,9 @@ public Sink getSink() { case BLOB: return BlobSinkFactory.create(config, offsetManager, statsDReporter, stencilClient); case BIGQUERY: - return new GenericOdpfSink(new FirehoseInstrumentation(statsDReporter, BigQuerySink.class), sinkType.name(), bigQuerySinkFactory.create()); + return new GenericSink(new FirehoseInstrumentation(statsDReporter, BigQuerySink.class), sinkType.name(), bigQuerySinkFactory.create()); case BIGTABLE: - return new GenericOdpfSink(new FirehoseInstrumentation(statsDReporter, BigTableSink.class), sinkType.name(), bigTableSinkFactory.create()); + return new GenericSink(new FirehoseInstrumentation(statsDReporter, BigTableSink.class), sinkType.name(), bigTableSinkFactory.create()); case MONGODB: return MongoSinkFactory.create(config, statsDReporter, stencilClient); default: diff --git a/src/main/java/io/odpf/firehose/sink/SinkFactoryUtils.java b/src/main/java/org/raystack/firehose/sink/SinkFactoryUtils.java similarity index 91% rename from src/main/java/io/odpf/firehose/sink/SinkFactoryUtils.java rename to src/main/java/org/raystack/firehose/sink/SinkFactoryUtils.java index 2c2b80925..f9cfca4f7 100644 --- a/src/main/java/io/odpf/firehose/sink/SinkFactoryUtils.java +++ b/src/main/java/org/raystack/firehose/sink/SinkFactoryUtils.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink; +package org.raystack.firehose.sink; -import io.odpf.depot.message.SinkConnectorSchemaMessageMode; +import org.raystack.depot.message.SinkConnectorSchemaMessageMode; import java.util.HashMap; import java.util.Map; diff --git a/src/main/java/io/odpf/firehose/sink/SinkPool.java b/src/main/java/org/raystack/firehose/sink/SinkPool.java similarity index 83% rename from src/main/java/io/odpf/firehose/sink/SinkPool.java rename to src/main/java/org/raystack/firehose/sink/SinkPool.java index 4ed3ed35f..b42416803 100644 --- a/src/main/java/io/odpf/firehose/sink/SinkPool.java +++ b/src/main/java/org/raystack/firehose/sink/SinkPool.java @@ -1,11 +1,13 @@ -package io.odpf.firehose.sink; +package org.raystack.firehose.sink; -import io.odpf.firehose.exception.SinkTaskFailedException; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.exception.SinkTaskFailedException; +import org.raystack.firehose.message.Message; import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; +import java.io.IOException; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -18,9 +20,11 @@ import java.util.stream.Collectors; @AllArgsConstructor +@Slf4j public class SinkPool implements AutoCloseable { private final Set sinkFutures = new HashSet<>(); private final BlockingQueue workerSinks; + private final List allSinks; private final ExecutorService executorService; private final long pollTimeOutMillis; @@ -59,6 +63,15 @@ public Future> submitTask(List messages) { @Override public void close() { + allSinks.forEach(sink -> { + try { + log.info("Closing sink"); + sink.close(); + } catch (IOException e) { + log.error("Error happened while closing sink"); + e.printStackTrace(); + } + }); executorService.shutdown(); } diff --git a/src/main/java/io/odpf/firehose/sink/bigquery/BigquerySinkUtils.java b/src/main/java/org/raystack/firehose/sink/bigquery/BigquerySinkUtils.java similarity index 92% rename from src/main/java/io/odpf/firehose/sink/bigquery/BigquerySinkUtils.java rename to src/main/java/org/raystack/firehose/sink/bigquery/BigquerySinkUtils.java index 6e6b2f441..d5da98291 100644 --- a/src/main/java/io/odpf/firehose/sink/bigquery/BigquerySinkUtils.java +++ b/src/main/java/org/raystack/firehose/sink/bigquery/BigquerySinkUtils.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.bigquery; +package org.raystack.firehose.sink.bigquery; import java.util.Map; import java.util.function.Function; diff --git a/src/main/java/io/odpf/firehose/sink/blob/BlobSink.java b/src/main/java/org/raystack/firehose/sink/blob/BlobSink.java similarity index 80% rename from src/main/java/io/odpf/firehose/sink/blob/BlobSink.java rename to src/main/java/org/raystack/firehose/sink/blob/BlobSink.java index 9f16e686d..25276496b 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/BlobSink.java +++ b/src/main/java/org/raystack/firehose/sink/blob/BlobSink.java @@ -1,18 +1,18 @@ -package io.odpf.firehose.sink.blob; +package org.raystack.firehose.sink.blob; -import io.odpf.depot.error.ErrorInfo; -import io.odpf.depot.error.ErrorType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.kafka.OffsetManager; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.exception.SinkException; -import io.odpf.firehose.exception.UnknownFieldsException; -import io.odpf.firehose.exception.EmptyMessageException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.AbstractSink; -import io.odpf.firehose.sink.blob.message.MessageDeSerializer; -import io.odpf.firehose.sink.blob.message.Record; -import io.odpf.firehose.sink.blob.writer.WriterOrchestrator; +import org.raystack.firehose.consumer.kafka.OffsetManager; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.exception.EmptyMessageException; +import org.raystack.firehose.exception.SinkException; +import org.raystack.firehose.exception.UnknownFieldsException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.blob.message.MessageDeSerializer; +import org.raystack.firehose.sink.blob.message.Record; +import org.raystack.firehose.sink.blob.writer.WriterOrchestrator; +import org.raystack.depot.error.ErrorInfo; +import org.raystack.depot.error.ErrorType; +import org.raystack.firehose.sink.AbstractSink; import java.io.IOException; import java.sql.SQLException; diff --git a/src/main/java/io/odpf/firehose/sink/blob/BlobSinkFactory.java b/src/main/java/org/raystack/firehose/sink/blob/BlobSinkFactory.java similarity index 73% rename from src/main/java/io/odpf/firehose/sink/blob/BlobSinkFactory.java rename to src/main/java/org/raystack/firehose/sink/blob/BlobSinkFactory.java index faf08c060..fedd59552 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/BlobSinkFactory.java +++ b/src/main/java/org/raystack/firehose/sink/blob/BlobSinkFactory.java @@ -1,23 +1,23 @@ -package io.odpf.firehose.sink.blob; +package org.raystack.firehose.sink.blob; import com.google.protobuf.Descriptors; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.BlobSinkConfig; -import io.odpf.firehose.consumer.kafka.OffsetManager; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.Sink; -import io.odpf.firehose.sink.blob.message.MessageDeSerializer; -import io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessage; -import io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils; -import io.odpf.firehose.sink.blob.proto.NestedKafkaMetadataProtoMessage; -import io.odpf.firehose.sink.blob.writer.WriterOrchestrator; -import io.odpf.firehose.sink.blob.writer.local.LocalStorage; -import io.odpf.firehose.sink.blob.writer.local.policy.SizeBasedRotatingPolicy; -import io.odpf.firehose.sink.blob.writer.local.policy.TimeBasedRotatingPolicy; -import io.odpf.firehose.sink.blob.writer.local.policy.WriterPolicy; -import io.odpf.firehose.sink.common.blobstorage.BlobStorage; -import io.odpf.firehose.sink.common.blobstorage.BlobStorageFactory; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.config.BlobSinkConfig; +import org.raystack.firehose.consumer.kafka.OffsetManager; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.blob.message.MessageDeSerializer; +import org.raystack.firehose.sink.blob.writer.WriterOrchestrator; +import org.raystack.firehose.sink.blob.writer.local.LocalStorage; +import org.raystack.firehose.sink.blob.writer.local.policy.SizeBasedRotatingPolicy; +import org.raystack.firehose.sink.blob.writer.local.policy.TimeBasedRotatingPolicy; +import org.raystack.firehose.sink.blob.writer.local.policy.WriterPolicy; +import org.raystack.firehose.sink.common.blobstorage.BlobStorage; +import org.raystack.firehose.sink.common.blobstorage.BlobStorageFactory; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.sink.Sink; +import org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessage; +import org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils; +import org.raystack.firehose.sink.blob.proto.NestedKafkaMetadataProtoMessage; +import org.raystack.stencil.client.StencilClient; import org.aeonbits.owner.ConfigFactory; import java.util.ArrayList; diff --git a/src/main/java/io/odpf/firehose/sink/blob/Constants.java b/src/main/java/org/raystack/firehose/sink/blob/Constants.java similarity index 79% rename from src/main/java/io/odpf/firehose/sink/blob/Constants.java rename to src/main/java/org/raystack/firehose/sink/blob/Constants.java index f9ac21c59..a6d88251e 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/Constants.java +++ b/src/main/java/org/raystack/firehose/sink/blob/Constants.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.blob; +package org.raystack.firehose.sink.blob; public class Constants { public enum WriterType { diff --git a/src/main/java/io/odpf/firehose/sink/blob/message/KafkaMetadataUtils.java b/src/main/java/org/raystack/firehose/sink/blob/message/KafkaMetadataUtils.java similarity index 86% rename from src/main/java/io/odpf/firehose/sink/blob/message/KafkaMetadataUtils.java rename to src/main/java/org/raystack/firehose/sink/blob/message/KafkaMetadataUtils.java index 03d595925..827c09a1e 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/message/KafkaMetadataUtils.java +++ b/src/main/java/org/raystack/firehose/sink/blob/message/KafkaMetadataUtils.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.sink.blob.message; +package org.raystack.firehose.sink.blob.message; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessage; -import io.odpf.firehose.sink.blob.proto.NestedKafkaMetadataProtoMessage; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessage; +import org.raystack.firehose.sink.blob.proto.NestedKafkaMetadataProtoMessage; import java.time.Instant; diff --git a/src/main/java/io/odpf/firehose/sink/blob/message/MessageDeSerializer.java b/src/main/java/org/raystack/firehose/sink/blob/message/MessageDeSerializer.java similarity index 75% rename from src/main/java/io/odpf/firehose/sink/blob/message/MessageDeSerializer.java rename to src/main/java/org/raystack/firehose/sink/blob/message/MessageDeSerializer.java index e3d19d2c0..de987e6c5 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/message/MessageDeSerializer.java +++ b/src/main/java/org/raystack/firehose/sink/blob/message/MessageDeSerializer.java @@ -1,17 +1,17 @@ -package io.odpf.firehose.sink.blob.message; +package org.raystack.firehose.sink.blob.message; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; -import io.odpf.firehose.config.BlobSinkConfig; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.proto.ProtoUtils; -import io.odpf.firehose.exception.EmptyMessageException; -import io.odpf.firehose.exception.UnknownFieldsException; -import io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils; -import io.odpf.stencil.client.StencilClient; -import io.odpf.stencil.Parser; +import org.raystack.firehose.config.BlobSinkConfig; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.exception.EmptyMessageException; +import org.raystack.firehose.exception.UnknownFieldsException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.proto.ProtoUtils; +import org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils; +import org.raystack.stencil.client.StencilClient; +import org.raystack.stencil.Parser; import lombok.AllArgsConstructor; @AllArgsConstructor diff --git a/src/main/java/io/odpf/firehose/sink/blob/message/Record.java b/src/main/java/org/raystack/firehose/sink/blob/message/Record.java similarity index 92% rename from src/main/java/io/odpf/firehose/sink/blob/message/Record.java rename to src/main/java/org/raystack/firehose/sink/blob/message/Record.java index e285338ee..e7d00ce85 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/message/Record.java +++ b/src/main/java/org/raystack/firehose/sink/blob/message/Record.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.blob.message; +package org.raystack.firehose.sink.blob.message; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; -import io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessage; +import org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessage; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/src/main/java/io/odpf/firehose/sink/blob/proto/KafkaMetadataProtoMessage.java b/src/main/java/org/raystack/firehose/sink/blob/proto/KafkaMetadataProtoMessage.java similarity index 99% rename from src/main/java/io/odpf/firehose/sink/blob/proto/KafkaMetadataProtoMessage.java rename to src/main/java/org/raystack/firehose/sink/blob/proto/KafkaMetadataProtoMessage.java index 5dd1b07b6..652f6c10e 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/proto/KafkaMetadataProtoMessage.java +++ b/src/main/java/org/raystack/firehose/sink/blob/proto/KafkaMetadataProtoMessage.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.blob.proto; +package org.raystack.firehose.sink.blob.proto; import com.github.os72.protobuf.dynamic.MessageDefinition; import com.google.protobuf.Descriptors; diff --git a/src/main/java/io/odpf/firehose/sink/blob/proto/KafkaMetadataProtoMessageUtils.java b/src/main/java/org/raystack/firehose/sink/blob/proto/KafkaMetadataProtoMessageUtils.java similarity index 98% rename from src/main/java/io/odpf/firehose/sink/blob/proto/KafkaMetadataProtoMessageUtils.java rename to src/main/java/org/raystack/firehose/sink/blob/proto/KafkaMetadataProtoMessageUtils.java index d6cb12f92..11ec552b1 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/proto/KafkaMetadataProtoMessageUtils.java +++ b/src/main/java/org/raystack/firehose/sink/blob/proto/KafkaMetadataProtoMessageUtils.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.blob.proto; +package org.raystack.firehose.sink.blob.proto; import com.github.os72.protobuf.dynamic.DynamicSchema; import com.github.os72.protobuf.dynamic.MessageDefinition; diff --git a/src/main/java/io/odpf/firehose/sink/blob/proto/NestedKafkaMetadataProtoMessage.java b/src/main/java/org/raystack/firehose/sink/blob/proto/NestedKafkaMetadataProtoMessage.java similarity index 98% rename from src/main/java/io/odpf/firehose/sink/blob/proto/NestedKafkaMetadataProtoMessage.java rename to src/main/java/org/raystack/firehose/sink/blob/proto/NestedKafkaMetadataProtoMessage.java index 1e8fe15f0..533017481 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/proto/NestedKafkaMetadataProtoMessage.java +++ b/src/main/java/org/raystack/firehose/sink/blob/proto/NestedKafkaMetadataProtoMessage.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.blob.proto; +package org.raystack.firehose.sink.blob.proto; import com.github.os72.protobuf.dynamic.MessageDefinition; import com.google.protobuf.Descriptors; diff --git a/src/main/java/io/odpf/firehose/sink/blob/proto/TimestampMetadataProtoMessage.java b/src/main/java/org/raystack/firehose/sink/blob/proto/TimestampMetadataProtoMessage.java similarity index 96% rename from src/main/java/io/odpf/firehose/sink/blob/proto/TimestampMetadataProtoMessage.java rename to src/main/java/org/raystack/firehose/sink/blob/proto/TimestampMetadataProtoMessage.java index 3990ef553..4f2eab95f 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/proto/TimestampMetadataProtoMessage.java +++ b/src/main/java/org/raystack/firehose/sink/blob/proto/TimestampMetadataProtoMessage.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.blob.proto; +package org.raystack.firehose.sink.blob.proto; import com.github.os72.protobuf.dynamic.MessageDefinition; import com.google.protobuf.Timestamp; diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/WriterOrchestrator.java b/src/main/java/org/raystack/firehose/sink/blob/writer/WriterOrchestrator.java similarity index 86% rename from src/main/java/io/odpf/firehose/sink/blob/writer/WriterOrchestrator.java rename to src/main/java/org/raystack/firehose/sink/blob/writer/WriterOrchestrator.java index 71fac427e..344ea7186 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/WriterOrchestrator.java +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/WriterOrchestrator.java @@ -1,17 +1,17 @@ -package io.odpf.firehose.sink.blob.writer; +package org.raystack.firehose.sink.blob.writer; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.BlobSinkConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.common.blobstorage.BlobStorage; -import io.odpf.firehose.sink.blob.message.Record; -import io.odpf.firehose.sink.blob.writer.local.LocalFileMetadata; -import io.odpf.firehose.sink.blob.writer.local.LocalFileChecker; -import io.odpf.firehose.sink.blob.writer.local.LocalFileWriter; -import io.odpf.firehose.sink.blob.writer.local.LocalStorage; -import io.odpf.firehose.sink.blob.writer.local.path.TimePartitionedPathUtils; -import io.odpf.firehose.sink.blob.writer.remote.BlobStorageChecker; -import io.odpf.firehose.sink.blob.writer.remote.BlobStorageWriterFutureHandler; +import org.raystack.firehose.config.BlobSinkConfig; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.blob.message.Record; +import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata; +import org.raystack.firehose.sink.blob.writer.local.LocalFileWriter; +import org.raystack.firehose.sink.blob.writer.local.path.TimePartitionedPathUtils; +import org.raystack.firehose.sink.blob.writer.remote.BlobStorageChecker; +import org.raystack.firehose.sink.blob.writer.remote.BlobStorageWriterFutureHandler; +import org.raystack.firehose.sink.common.blobstorage.BlobStorage; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.sink.blob.writer.local.LocalFileChecker; +import org.raystack.firehose.sink.blob.writer.local.LocalStorage; import java.io.Closeable; import java.io.IOException; @@ -125,7 +125,7 @@ private String write(Record record, Path timePartitionedPath) throws IOException if (!writer.write(record)) { return write(record, timePartitionedPath); } - return writer.getMetadata().getFullPath(); + return writer.getFullPath(); } @Override @@ -139,7 +139,7 @@ public void close() throws IOException { writer.close(); } for (LocalFileWriter p : timePartitionWriterMap.values()) { - localStorage.deleteLocalFile(p.getMetadata().getFullPath()); + localStorage.deleteLocalFile(p.getFullPath()); } } } diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/WriterOrchestratorStatus.java b/src/main/java/org/raystack/firehose/sink/blob/writer/WriterOrchestratorStatus.java similarity index 97% rename from src/main/java/io/odpf/firehose/sink/blob/writer/WriterOrchestratorStatus.java rename to src/main/java/org/raystack/firehose/sink/blob/writer/WriterOrchestratorStatus.java index 0a94ce2b3..261c2b619 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/WriterOrchestratorStatus.java +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/WriterOrchestratorStatus.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.blob.writer; +package org.raystack.firehose.sink.blob.writer; import lombok.Data; diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalFileChecker.java b/src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalFileChecker.java similarity index 67% rename from src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalFileChecker.java rename to src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalFileChecker.java index 135648505..81a03c8db 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalFileChecker.java +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalFileChecker.java @@ -1,6 +1,8 @@ -package io.odpf.firehose.sink.blob.writer.local; +package org.raystack.firehose.sink.blob.writer.local; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.BlobStorageMetrics; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; import java.io.IOException; import java.nio.file.Path; @@ -9,10 +11,6 @@ import java.util.Queue; import java.util.stream.Collectors; -import static io.odpf.firehose.metrics.Metrics.FAILURE_TAG; -import static io.odpf.firehose.metrics.Metrics.SUCCESS_TAG; -import static io.odpf.firehose.metrics.BlobStorageMetrics.*; - public class LocalFileChecker implements Runnable { private final Queue toBeFlushedToRemotePaths; private final Map timePartitionWriterMap; @@ -32,7 +30,7 @@ public LocalFileChecker(Queue toBeFlushedToRemotePaths, @Override public void run() { - firehoseInstrumentation.captureValue(LOCAL_FILE_OPEN_TOTAL, timePartitionWriterMap.size()); + firehoseInstrumentation.captureValue(BlobStorageMetrics.LOCAL_FILE_OPEN_TOTAL, timePartitionWriterMap.size()); Map toBeRotated = timePartitionWriterMap.entrySet().stream().filter(kv -> localStorage.shouldRotate(kv.getValue())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -50,17 +48,17 @@ public void run() { throw new LocalFileWriterFailedException(e); } }); - firehoseInstrumentation.captureValue(LOCAL_FILE_OPEN_TOTAL, timePartitionWriterMap.size()); + firehoseInstrumentation.captureValue(BlobStorageMetrics.LOCAL_FILE_OPEN_TOTAL, timePartitionWriterMap.size()); } private void captureFileClosedSuccessMetric(Instant startTime, LocalFileMetadata localFileMetadata) { - firehoseInstrumentation.incrementCounter(LOCAL_FILE_CLOSE_TOTAL, SUCCESS_TAG); - firehoseInstrumentation.captureDurationSince(LOCAL_FILE_CLOSING_TIME_MILLISECONDS, startTime); - firehoseInstrumentation.captureCount(LOCAL_FILE_SIZE_BYTES, localFileMetadata.getSize()); - firehoseInstrumentation.captureCount(LOCAL_FILE_RECORDS_TOTAL, localFileMetadata.getRecordCount()); + firehoseInstrumentation.incrementCounter(BlobStorageMetrics.LOCAL_FILE_CLOSE_TOTAL, Metrics.SUCCESS_TAG); + firehoseInstrumentation.captureDurationSince(BlobStorageMetrics.LOCAL_FILE_CLOSING_TIME_MILLISECONDS, startTime); + firehoseInstrumentation.captureCount(BlobStorageMetrics.LOCAL_FILE_SIZE_BYTES, localFileMetadata.getSize()); + firehoseInstrumentation.captureCount(BlobStorageMetrics.LOCAL_FILE_RECORDS_TOTAL, localFileMetadata.getRecordCount()); } private void captureFileCloseFailedMetric() { - firehoseInstrumentation.incrementCounter(LOCAL_FILE_CLOSE_TOTAL, FAILURE_TAG); + firehoseInstrumentation.incrementCounter(BlobStorageMetrics.LOCAL_FILE_CLOSE_TOTAL, Metrics.FAILURE_TAG); } } diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalFileMetadata.java b/src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalFileMetadata.java similarity index 86% rename from src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalFileMetadata.java rename to src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalFileMetadata.java index facacf4f7..e916ae048 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalFileMetadata.java +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalFileMetadata.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.blob.writer.local; +package org.raystack.firehose.sink.blob.writer.local; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalFileWriter.java b/src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalFileWriter.java similarity index 76% rename from src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalFileWriter.java rename to src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalFileWriter.java index 63a0494f1..b0029054e 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalFileWriter.java +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalFileWriter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink.blob.writer.local; +package org.raystack.firehose.sink.blob.writer.local; -import io.odpf.firehose.sink.blob.message.Record; +import org.raystack.firehose.sink.blob.message.Record; import java.io.Closeable; import java.io.IOException; @@ -16,4 +16,6 @@ public interface LocalFileWriter extends Closeable { LocalFileMetadata getMetadata(); LocalFileMetadata closeAndFetchMetaData() throws IOException; + + String getFullPath(); } diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalFileWriterFailedException.java b/src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalFileWriterFailedException.java similarity index 77% rename from src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalFileWriterFailedException.java rename to src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalFileWriterFailedException.java index c0321f382..681e80994 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalFileWriterFailedException.java +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalFileWriterFailedException.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.blob.writer.local; +package org.raystack.firehose.sink.blob.writer.local; import java.io.IOException; diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalParquetFileWriter.java b/src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalParquetFileWriter.java similarity index 91% rename from src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalParquetFileWriter.java rename to src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalParquetFileWriter.java index 49219fae6..237e2e86b 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalParquetFileWriter.java +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalParquetFileWriter.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.blob.writer.local; +package org.raystack.firehose.sink.blob.writer.local; import com.google.protobuf.Descriptors; -import io.odpf.firehose.config.BlobSinkConfig; -import io.odpf.firehose.sink.blob.message.Record; +import org.raystack.firehose.config.BlobSinkConfig; +import org.raystack.firehose.sink.blob.message.Record; import org.apache.hadoop.fs.Path; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.metadata.CompressionCodecName; @@ -70,4 +70,9 @@ public synchronized LocalFileMetadata closeAndFetchMetaData() throws IOException this.close(); return metadata; } + + @Override + public String getFullPath() { + return fullPath; + } } diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalStorage.java b/src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalStorage.java similarity index 90% rename from src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalStorage.java rename to src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalStorage.java index dbbd858fc..54805d985 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/local/LocalStorage.java +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalStorage.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.sink.blob.writer.local; +package org.raystack.firehose.sink.blob.writer.local; import com.google.protobuf.Descriptors; -import io.odpf.firehose.config.BlobSinkConfig; -import io.odpf.firehose.exception.ConfigurationException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.blob.writer.local.policy.WriterPolicy; +import org.raystack.firehose.config.BlobSinkConfig; +import org.raystack.firehose.exception.ConfigurationException; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.blob.writer.local.policy.WriterPolicy; import lombok.AllArgsConstructor; import java.io.IOException; diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/local/path/TimePartitionedPathUtils.java b/src/main/java/org/raystack/firehose/sink/blob/writer/local/path/TimePartitionedPathUtils.java similarity index 91% rename from src/main/java/io/odpf/firehose/sink/blob/writer/local/path/TimePartitionedPathUtils.java rename to src/main/java/org/raystack/firehose/sink/blob/writer/local/path/TimePartitionedPathUtils.java index 731c6637c..7b45a26f9 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/local/path/TimePartitionedPathUtils.java +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/local/path/TimePartitionedPathUtils.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.blob.writer.local.path; +package org.raystack.firehose.sink.blob.writer.local.path; -import io.odpf.firehose.config.BlobSinkConfig; -import io.odpf.firehose.sink.blob.Constants; -import io.odpf.firehose.sink.blob.message.Record; +import org.raystack.firehose.config.BlobSinkConfig; +import org.raystack.firehose.sink.blob.message.Record; +import org.raystack.firehose.sink.blob.Constants; import lombok.AllArgsConstructor; import java.nio.file.Path; diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/local/policy/SizeBasedRotatingPolicy.java b/src/main/java/org/raystack/firehose/sink/blob/writer/local/policy/SizeBasedRotatingPolicy.java similarity index 77% rename from src/main/java/io/odpf/firehose/sink/blob/writer/local/policy/SizeBasedRotatingPolicy.java rename to src/main/java/org/raystack/firehose/sink/blob/writer/local/policy/SizeBasedRotatingPolicy.java index f62e352c9..7daa05be7 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/local/policy/SizeBasedRotatingPolicy.java +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/local/policy/SizeBasedRotatingPolicy.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink.blob.writer.local.policy; +package org.raystack.firehose.sink.blob.writer.local.policy; -import io.odpf.firehose.sink.blob.writer.local.LocalFileMetadata; +import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata; public class SizeBasedRotatingPolicy implements WriterPolicy { diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/local/policy/TimeBasedRotatingPolicy.java b/src/main/java/org/raystack/firehose/sink/blob/writer/local/policy/TimeBasedRotatingPolicy.java similarity index 82% rename from src/main/java/io/odpf/firehose/sink/blob/writer/local/policy/TimeBasedRotatingPolicy.java rename to src/main/java/org/raystack/firehose/sink/blob/writer/local/policy/TimeBasedRotatingPolicy.java index 0599bbd9f..af36ace7c 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/local/policy/TimeBasedRotatingPolicy.java +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/local/policy/TimeBasedRotatingPolicy.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink.blob.writer.local.policy; +package org.raystack.firehose.sink.blob.writer.local.policy; -import io.odpf.firehose.sink.blob.writer.local.LocalFileMetadata; +import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata; public class TimeBasedRotatingPolicy implements WriterPolicy { diff --git a/src/main/java/org/raystack/firehose/sink/blob/writer/local/policy/WriterPolicy.java b/src/main/java/org/raystack/firehose/sink/blob/writer/local/policy/WriterPolicy.java new file mode 100644 index 000000000..ee3bb7733 --- /dev/null +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/local/policy/WriterPolicy.java @@ -0,0 +1,7 @@ +package org.raystack.firehose.sink.blob.writer.local.policy; + +import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata; + +public interface WriterPolicy { + boolean shouldRotate(LocalFileMetadata metadata); +} diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageChecker.java b/src/main/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageChecker.java similarity index 87% rename from src/main/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageChecker.java rename to src/main/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageChecker.java index 8350170a3..84b92b2e8 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageChecker.java +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageChecker.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.blob.writer.remote; +package org.raystack.firehose.sink.blob.writer.remote; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.common.blobstorage.BlobStorage; -import io.odpf.firehose.sink.blob.writer.local.LocalFileMetadata; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.common.blobstorage.BlobStorage; +import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata; import lombok.AllArgsConstructor; import java.util.ArrayList; diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageFailedException.java b/src/main/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageFailedException.java similarity index 73% rename from src/main/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageFailedException.java rename to src/main/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageFailedException.java index 7bc767abe..bff248f3b 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageFailedException.java +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageFailedException.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.blob.writer.remote; +package org.raystack.firehose.sink.blob.writer.remote; public class BlobStorageFailedException extends RuntimeException { public BlobStorageFailedException(Throwable th) { diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageWorker.java b/src/main/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageWorker.java similarity index 74% rename from src/main/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageWorker.java rename to src/main/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageWorker.java index 7c6b8b097..eb933242d 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageWorker.java +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageWorker.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.blob.writer.remote; +package org.raystack.firehose.sink.blob.writer.remote; -import io.odpf.firehose.sink.common.blobstorage.BlobStorage; -import io.odpf.firehose.sink.common.blobstorage.BlobStorageException; -import io.odpf.firehose.sink.blob.writer.local.LocalFileMetadata; +import org.raystack.firehose.sink.common.blobstorage.BlobStorage; +import org.raystack.firehose.sink.common.blobstorage.BlobStorageException; +import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata; import lombok.AllArgsConstructor; import java.nio.file.Paths; diff --git a/src/main/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageWriterFutureHandler.java b/src/main/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageWriterFutureHandler.java similarity index 61% rename from src/main/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageWriterFutureHandler.java rename to src/main/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageWriterFutureHandler.java index 7215acfff..0cad034c1 100644 --- a/src/main/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageWriterFutureHandler.java +++ b/src/main/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageWriterFutureHandler.java @@ -1,18 +1,16 @@ -package io.odpf.firehose.sink.blob.writer.remote; +package org.raystack.firehose.sink.blob.writer.remote; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.common.blobstorage.BlobStorageException; -import io.odpf.firehose.sink.blob.writer.local.LocalFileMetadata; +import org.raystack.firehose.metrics.BlobStorageMetrics; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata; +import org.raystack.firehose.sink.common.blobstorage.BlobStorageException; import lombok.AllArgsConstructor; import lombok.Data; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import static io.odpf.firehose.metrics.Metrics.*; -import static io.odpf.firehose.metrics.Metrics.tag; -import static io.odpf.firehose.metrics.BlobStorageMetrics.*; - @AllArgsConstructor @Data public class BlobStorageWriterFutureHandler { @@ -44,10 +42,10 @@ public boolean isFinished() { private void captureFileUploadSuccessMetric(long totalTime) { firehoseInstrumentation.logInfo("Flushed to blob storage {}", localFileMetadata.getFullPath()); - firehoseInstrumentation.incrementCounter(FILE_UPLOAD_TOTAL, SUCCESS_TAG); - firehoseInstrumentation.captureCount(FILE_UPLOAD_BYTES, localFileMetadata.getSize()); - firehoseInstrumentation.captureCount(FILE_UPLOAD_RECORDS_TOTAL, localFileMetadata.getRecordCount()); - firehoseInstrumentation.captureDuration(FILE_UPLOAD_TIME_MILLISECONDS, totalTime); + firehoseInstrumentation.incrementCounter(BlobStorageMetrics.FILE_UPLOAD_TOTAL, Metrics.SUCCESS_TAG); + firehoseInstrumentation.captureCount(BlobStorageMetrics.FILE_UPLOAD_BYTES, localFileMetadata.getSize()); + firehoseInstrumentation.captureCount(BlobStorageMetrics.FILE_UPLOAD_RECORDS_TOTAL, localFileMetadata.getRecordCount()); + firehoseInstrumentation.captureDuration(BlobStorageMetrics.FILE_UPLOAD_TIME_MILLISECONDS, totalTime); } private void captureUploadFailedMetric(Throwable e) { @@ -58,6 +56,6 @@ private void captureUploadFailedMetric(Throwable e) { } else { errorType = ""; } - firehoseInstrumentation.incrementCounter(FILE_UPLOAD_TOTAL, FAILURE_TAG, tag(BLOB_STORAGE_ERROR_TYPE_TAG, errorType)); + firehoseInstrumentation.incrementCounter(BlobStorageMetrics.FILE_UPLOAD_TOTAL, Metrics.FAILURE_TAG, Metrics.tag(BlobStorageMetrics.BLOB_STORAGE_ERROR_TYPE_TAG, errorType)); } } diff --git a/src/main/java/io/odpf/firehose/sink/common/AbstractHttpSink.java b/src/main/java/org/raystack/firehose/sink/common/AbstractHttpSink.java similarity index 93% rename from src/main/java/io/odpf/firehose/sink/common/AbstractHttpSink.java rename to src/main/java/org/raystack/firehose/sink/common/AbstractHttpSink.java index 501f71447..c9e5bad10 100644 --- a/src/main/java/io/odpf/firehose/sink/common/AbstractHttpSink.java +++ b/src/main/java/org/raystack/firehose/sink/common/AbstractHttpSink.java @@ -1,11 +1,12 @@ -package io.odpf.firehose.sink.common; +package org.raystack.firehose.sink.common; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.NeedToRetry; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.AbstractSink; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.exception.NeedToRetry; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.sink.AbstractSink; +import org.raystack.stencil.client.StencilClient; import joptsimple.internal.Strings; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; @@ -24,8 +25,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.odpf.firehose.metrics.Metrics.SINK_HTTP_RESPONSE_CODE_TOTAL; - public abstract class AbstractHttpSink extends AbstractSink { private final List httpRequests = new ArrayList<>(); @@ -111,7 +110,7 @@ protected String statusCode(HttpResponse response) { private void captureHttpStatusCount(HttpResponse response) { String statusCode = statusCode(response); String httpCodeTag = statusCode.equals("null") ? "status_code=" : "status_code=" + statusCode; - getFirehoseInstrumentation().captureCount(SINK_HTTP_RESPONSE_CODE_TOTAL, 1L, httpCodeTag); + getFirehoseInstrumentation().captureCount(Metrics.SINK_HTTP_RESPONSE_CODE_TOTAL, 1L, httpCodeTag); } private void printRequest(HttpEntityEnclosingRequestBase httpRequest, List contentStringList) throws IOException { diff --git a/src/main/java/io/odpf/firehose/sink/common/KeyOrMessageParser.java b/src/main/java/org/raystack/firehose/sink/common/KeyOrMessageParser.java similarity index 85% rename from src/main/java/io/odpf/firehose/sink/common/KeyOrMessageParser.java rename to src/main/java/org/raystack/firehose/sink/common/KeyOrMessageParser.java index 8881f7144..b013b4358 100644 --- a/src/main/java/io/odpf/firehose/sink/common/KeyOrMessageParser.java +++ b/src/main/java/org/raystack/firehose/sink/common/KeyOrMessageParser.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sink.common; +package org.raystack.firehose.sink.common; -import io.odpf.firehose.config.AppConfig; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.config.AppConfig; +import org.raystack.firehose.message.Message; import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; -import io.odpf.stencil.Parser; +import org.raystack.stencil.Parser; import lombok.AllArgsConstructor; import java.io.IOException; diff --git a/src/main/java/io/odpf/firehose/sink/common/blobstorage/BlobStorage.java b/src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorage.java similarity index 82% rename from src/main/java/io/odpf/firehose/sink/common/blobstorage/BlobStorage.java rename to src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorage.java index 730df138c..c39b58e8a 100644 --- a/src/main/java/io/odpf/firehose/sink/common/blobstorage/BlobStorage.java +++ b/src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorage.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.common.blobstorage; +package org.raystack.firehose.sink.common.blobstorage; /** * Abstraction of any storage that store binary bytes as file. diff --git a/src/main/java/io/odpf/firehose/sink/common/blobstorage/BlobStorageException.java b/src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorageException.java similarity index 90% rename from src/main/java/io/odpf/firehose/sink/common/blobstorage/BlobStorageException.java rename to src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorageException.java index 17d362fe3..f3091c7b1 100644 --- a/src/main/java/io/odpf/firehose/sink/common/blobstorage/BlobStorageException.java +++ b/src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorageException.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.common.blobstorage; +package org.raystack.firehose.sink.common.blobstorage; import lombok.EqualsAndHashCode; import lombok.Getter; diff --git a/src/main/java/io/odpf/firehose/sink/common/blobstorage/BlobStorageFactory.java b/src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorageFactory.java similarity index 79% rename from src/main/java/io/odpf/firehose/sink/common/blobstorage/BlobStorageFactory.java rename to src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorageFactory.java index bbad87f60..94ad0a1ce 100644 --- a/src/main/java/io/odpf/firehose/sink/common/blobstorage/BlobStorageFactory.java +++ b/src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorageFactory.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.sink.common.blobstorage; +package org.raystack.firehose.sink.common.blobstorage; -import io.odpf.firehose.config.GCSConfig; -import io.odpf.firehose.config.S3Config; -import io.odpf.firehose.sink.common.blobstorage.gcs.GoogleCloudStorage; -import io.odpf.firehose.sink.common.blobstorage.s3.S3; +import org.raystack.firehose.config.GCSConfig; +import org.raystack.firehose.config.S3Config; +import org.raystack.firehose.sink.common.blobstorage.gcs.GoogleCloudStorage; +import org.raystack.firehose.sink.common.blobstorage.s3.S3; import org.aeonbits.owner.ConfigFactory; import java.io.IOException; diff --git a/src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorageType.java b/src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorageType.java new file mode 100644 index 000000000..3221ecfc5 --- /dev/null +++ b/src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorageType.java @@ -0,0 +1,6 @@ +package org.raystack.firehose.sink.common.blobstorage; + +public enum BlobStorageType { + GCS, + S3 +} diff --git a/src/main/java/io/odpf/firehose/sink/common/blobstorage/gcs/GoogleCloudStorage.java b/src/main/java/org/raystack/firehose/sink/common/blobstorage/gcs/GoogleCloudStorage.java similarity index 93% rename from src/main/java/io/odpf/firehose/sink/common/blobstorage/gcs/GoogleCloudStorage.java rename to src/main/java/org/raystack/firehose/sink/common/blobstorage/gcs/GoogleCloudStorage.java index 7944b9d16..fb5f6f4ed 100644 --- a/src/main/java/io/odpf/firehose/sink/common/blobstorage/gcs/GoogleCloudStorage.java +++ b/src/main/java/org/raystack/firehose/sink/common/blobstorage/gcs/GoogleCloudStorage.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.common.blobstorage.gcs; +package org.raystack.firehose.sink.common.blobstorage.gcs; import com.google.api.gax.retrying.RetrySettings; import com.google.auth.oauth2.GoogleCredentials; @@ -8,10 +8,10 @@ import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageException; import com.google.cloud.storage.StorageOptions; -import io.odpf.firehose.sink.common.blobstorage.BlobStorage; -import io.odpf.firehose.sink.common.blobstorage.BlobStorageException; -import io.odpf.firehose.config.GCSConfig; -import io.odpf.firehose.sink.common.blobstorage.gcs.error.GCSErrorType; +import org.raystack.firehose.config.GCSConfig; +import org.raystack.firehose.sink.common.blobstorage.BlobStorage; +import org.raystack.firehose.sink.common.blobstorage.BlobStorageException; +import org.raystack.firehose.sink.common.blobstorage.gcs.error.GCSErrorType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.threeten.bp.Duration; diff --git a/src/main/java/io/odpf/firehose/sink/common/blobstorage/gcs/error/GCSErrorType.java b/src/main/java/org/raystack/firehose/sink/common/blobstorage/gcs/error/GCSErrorType.java similarity index 95% rename from src/main/java/io/odpf/firehose/sink/common/blobstorage/gcs/error/GCSErrorType.java rename to src/main/java/org/raystack/firehose/sink/common/blobstorage/gcs/error/GCSErrorType.java index fd5837133..27d847aef 100644 --- a/src/main/java/io/odpf/firehose/sink/common/blobstorage/gcs/error/GCSErrorType.java +++ b/src/main/java/org/raystack/firehose/sink/common/blobstorage/gcs/error/GCSErrorType.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.common.blobstorage.gcs.error; +package org.raystack.firehose.sink.common.blobstorage.gcs.error; import java.util.HashMap; import java.util.Map; diff --git a/src/main/java/io/odpf/firehose/sink/common/blobstorage/s3/S3.java b/src/main/java/org/raystack/firehose/sink/common/blobstorage/s3/S3.java similarity index 94% rename from src/main/java/io/odpf/firehose/sink/common/blobstorage/s3/S3.java rename to src/main/java/org/raystack/firehose/sink/common/blobstorage/s3/S3.java index 699b1003c..4d57b78a9 100644 --- a/src/main/java/io/odpf/firehose/sink/common/blobstorage/s3/S3.java +++ b/src/main/java/org/raystack/firehose/sink/common/blobstorage/s3/S3.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.sink.common.blobstorage.s3; +package org.raystack.firehose.sink.common.blobstorage.s3; -import io.odpf.firehose.config.S3Config; -import io.odpf.firehose.sink.common.blobstorage.BlobStorage; -import io.odpf.firehose.sink.common.blobstorage.BlobStorageException; +import org.raystack.firehose.config.S3Config; +import org.raystack.firehose.sink.common.blobstorage.BlobStorage; +import org.raystack.firehose.sink.common.blobstorage.BlobStorageException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; diff --git a/src/main/java/io/odpf/firehose/sink/dlq/DLQWriterType.java b/src/main/java/org/raystack/firehose/sink/dlq/DLQWriterType.java similarity index 62% rename from src/main/java/io/odpf/firehose/sink/dlq/DLQWriterType.java rename to src/main/java/org/raystack/firehose/sink/dlq/DLQWriterType.java index 2f8a51ff0..e3836fa81 100644 --- a/src/main/java/io/odpf/firehose/sink/dlq/DLQWriterType.java +++ b/src/main/java/org/raystack/firehose/sink/dlq/DLQWriterType.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.dlq; +package org.raystack.firehose.sink.dlq; public enum DLQWriterType { KAFKA, diff --git a/src/main/java/io/odpf/firehose/sink/dlq/DlqWriter.java b/src/main/java/org/raystack/firehose/sink/dlq/DlqWriter.java similarity index 84% rename from src/main/java/io/odpf/firehose/sink/dlq/DlqWriter.java rename to src/main/java/org/raystack/firehose/sink/dlq/DlqWriter.java index bab5db9c4..592033cb9 100644 --- a/src/main/java/io/odpf/firehose/sink/dlq/DlqWriter.java +++ b/src/main/java/org/raystack/firehose/sink/dlq/DlqWriter.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink.dlq; +package org.raystack.firehose.sink.dlq; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.message.Message; import java.io.IOException; import java.util.List; diff --git a/src/main/java/io/odpf/firehose/sink/dlq/DlqWriterFactory.java b/src/main/java/org/raystack/firehose/sink/dlq/DlqWriterFactory.java similarity index 76% rename from src/main/java/io/odpf/firehose/sink/dlq/DlqWriterFactory.java rename to src/main/java/org/raystack/firehose/sink/dlq/DlqWriterFactory.java index bdf444123..cb769e09d 100644 --- a/src/main/java/io/odpf/firehose/sink/dlq/DlqWriterFactory.java +++ b/src/main/java/org/raystack/firehose/sink/dlq/DlqWriterFactory.java @@ -1,15 +1,15 @@ -package io.odpf.firehose.sink.dlq; - -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.DlqConfig; -import io.odpf.firehose.config.DlqKafkaProducerConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.utils.KafkaUtils; -import io.odpf.firehose.sink.common.blobstorage.BlobStorage; -import io.odpf.firehose.sink.common.blobstorage.BlobStorageFactory; -import io.odpf.firehose.sink.dlq.kafka.KafkaDlqWriter; -import io.odpf.firehose.sink.dlq.log.LogDlqWriter; -import io.odpf.firehose.sink.dlq.blobstorage.BlobStorageDlqWriter; +package org.raystack.firehose.sink.dlq; + +import org.raystack.firehose.config.DlqConfig; +import org.raystack.firehose.config.DlqKafkaProducerConfig; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.common.blobstorage.BlobStorage; +import org.raystack.firehose.sink.common.blobstorage.BlobStorageFactory; +import org.raystack.firehose.sink.dlq.blobstorage.BlobStorageDlqWriter; +import org.raystack.firehose.sink.dlq.kafka.KafkaDlqWriter; +import org.raystack.firehose.sink.dlq.log.LogDlqWriter; +import org.raystack.firehose.utils.KafkaUtils; +import org.raystack.depot.metrics.StatsDReporter; import io.opentracing.Tracer; import io.opentracing.contrib.kafka.TracingKafkaProducer; import org.aeonbits.owner.ConfigFactory; diff --git a/src/main/java/io/odpf/firehose/sink/dlq/blobstorage/BlobStorageDlqWriter.java b/src/main/java/org/raystack/firehose/sink/dlq/blobstorage/BlobStorageDlqWriter.java similarity index 91% rename from src/main/java/io/odpf/firehose/sink/dlq/blobstorage/BlobStorageDlqWriter.java rename to src/main/java/org/raystack/firehose/sink/dlq/blobstorage/BlobStorageDlqWriter.java index e4fb722d1..8714534ad 100644 --- a/src/main/java/io/odpf/firehose/sink/dlq/blobstorage/BlobStorageDlqWriter.java +++ b/src/main/java/org/raystack/firehose/sink/dlq/blobstorage/BlobStorageDlqWriter.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sink.dlq.blobstorage; +package org.raystack.firehose.sink.dlq.blobstorage; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.sink.common.blobstorage.BlobStorageException; -import io.odpf.firehose.sink.common.blobstorage.BlobStorage; -import io.odpf.firehose.sink.dlq.DlqWriter; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.sink.common.blobstorage.BlobStorage; +import org.raystack.firehose.sink.common.blobstorage.BlobStorageException; +import org.raystack.firehose.sink.dlq.DlqWriter; import lombok.extern.slf4j.Slf4j; import java.io.IOException; diff --git a/src/main/java/io/odpf/firehose/sink/dlq/blobstorage/DlqMessage.java b/src/main/java/org/raystack/firehose/sink/dlq/blobstorage/DlqMessage.java similarity index 91% rename from src/main/java/io/odpf/firehose/sink/dlq/blobstorage/DlqMessage.java rename to src/main/java/org/raystack/firehose/sink/dlq/blobstorage/DlqMessage.java index f89a5b3da..e8bf416bb 100644 --- a/src/main/java/io/odpf/firehose/sink/dlq/blobstorage/DlqMessage.java +++ b/src/main/java/org/raystack/firehose/sink/dlq/blobstorage/DlqMessage.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.dlq.blobstorage; +package org.raystack.firehose.sink.dlq.blobstorage; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.AllArgsConstructor; diff --git a/src/main/java/io/odpf/firehose/sink/dlq/kafka/KafkaDlqWriter.java b/src/main/java/org/raystack/firehose/sink/dlq/kafka/KafkaDlqWriter.java similarity index 91% rename from src/main/java/io/odpf/firehose/sink/dlq/kafka/KafkaDlqWriter.java rename to src/main/java/org/raystack/firehose/sink/dlq/kafka/KafkaDlqWriter.java index 3f7b72729..52012fa0b 100644 --- a/src/main/java/io/odpf/firehose/sink/dlq/kafka/KafkaDlqWriter.java +++ b/src/main/java/org/raystack/firehose/sink/dlq/kafka/KafkaDlqWriter.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.dlq.kafka; +package org.raystack.firehose.sink.dlq.kafka; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.dlq.DlqWriter; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.dlq.DlqWriter; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; diff --git a/src/main/java/io/odpf/firehose/sink/dlq/log/LogDlqWriter.java b/src/main/java/org/raystack/firehose/sink/dlq/log/LogDlqWriter.java similarity index 82% rename from src/main/java/io/odpf/firehose/sink/dlq/log/LogDlqWriter.java rename to src/main/java/org/raystack/firehose/sink/dlq/log/LogDlqWriter.java index 3923b5ad6..a462b17bd 100644 --- a/src/main/java/io/odpf/firehose/sink/dlq/log/LogDlqWriter.java +++ b/src/main/java/org/raystack/firehose/sink/dlq/log/LogDlqWriter.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.sink.dlq.log; +package org.raystack.firehose.sink.dlq.log; -import io.odpf.depot.error.ErrorInfo; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.dlq.DlqWriter; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.depot.error.ErrorInfo; +import org.raystack.firehose.sink.dlq.DlqWriter; import org.apache.commons.lang3.exception.ExceptionUtils; import java.io.IOException; diff --git a/src/main/java/io/odpf/firehose/sink/elasticsearch/EsSink.java b/src/main/java/org/raystack/firehose/sink/elasticsearch/EsSink.java similarity index 89% rename from src/main/java/io/odpf/firehose/sink/elasticsearch/EsSink.java rename to src/main/java/org/raystack/firehose/sink/elasticsearch/EsSink.java index 2acacd0bc..a1c5c65dd 100644 --- a/src/main/java/io/odpf/firehose/sink/elasticsearch/EsSink.java +++ b/src/main/java/org/raystack/firehose/sink/elasticsearch/EsSink.java @@ -1,10 +1,11 @@ -package io.odpf.firehose.sink.elasticsearch; +package org.raystack.firehose.sink.elasticsearch; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.NeedToRetry; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.AbstractSink; -import io.odpf.firehose.sink.elasticsearch.request.EsRequestHandler; +import org.raystack.firehose.exception.NeedToRetry; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.sink.elasticsearch.request.EsRequestHandler; +import org.raystack.firehose.sink.AbstractSink; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -15,8 +16,6 @@ import java.util.ArrayList; import java.util.List; -import static io.odpf.firehose.metrics.Metrics.SINK_MESSAGES_DROP_TOTAL; - /** * Elastic search sink. */ @@ -85,7 +84,7 @@ private void handleResponse(BulkResponse bulkResponse) throws NeedToRetry { String responseStatus = String.valueOf(response.status().getStatus()); if (esRetryStatusCodeBlacklist.contains(responseStatus)) { getFirehoseInstrumentation().logInfo("Not retrying due to response status: {} is under blacklisted status code", responseStatus); - getFirehoseInstrumentation().incrementCounter(SINK_MESSAGES_DROP_TOTAL, "cause=" + response.status().name()); + getFirehoseInstrumentation().incrementCounter(Metrics.SINK_MESSAGES_DROP_TOTAL, "cause=" + response.status().name()); getFirehoseInstrumentation().logInfo("Message dropped because of status code: " + responseStatus); } else { throw new NeedToRetry(bulkResponse.buildFailureMessage()); diff --git a/src/main/java/io/odpf/firehose/sink/elasticsearch/EsSinkFactory.java b/src/main/java/org/raystack/firehose/sink/elasticsearch/EsSinkFactory.java similarity index 88% rename from src/main/java/io/odpf/firehose/sink/elasticsearch/EsSinkFactory.java rename to src/main/java/org/raystack/firehose/sink/elasticsearch/EsSinkFactory.java index a0d9b6804..989f40732 100644 --- a/src/main/java/io/odpf/firehose/sink/elasticsearch/EsSinkFactory.java +++ b/src/main/java/org/raystack/firehose/sink/elasticsearch/EsSinkFactory.java @@ -1,15 +1,15 @@ -package io.odpf.firehose.sink.elasticsearch; +package org.raystack.firehose.sink.elasticsearch; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.EsSinkConfig; -import io.odpf.firehose.config.enums.SinkType; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.serializer.MessageToJson; -import io.odpf.firehose.sink.Sink; -import io.odpf.firehose.sink.elasticsearch.request.EsRequestHandler; -import io.odpf.firehose.sink.elasticsearch.request.EsRequestHandlerFactory; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.config.EsSinkConfig; +import org.raystack.firehose.config.enums.SinkType; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.serializer.MessageToJson; +import org.raystack.firehose.sink.elasticsearch.request.EsRequestHandler; +import org.raystack.firehose.sink.elasticsearch.request.EsRequestHandlerFactory; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.sink.Sink; +import org.raystack.stencil.client.StencilClient; import org.aeonbits.owner.ConfigFactory; import org.apache.http.HttpHost; import org.elasticsearch.client.RestClient; diff --git a/src/main/java/io/odpf/firehose/sink/elasticsearch/request/EsRequestHandler.java b/src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsRequestHandler.java similarity index 84% rename from src/main/java/io/odpf/firehose/sink/elasticsearch/request/EsRequestHandler.java rename to src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsRequestHandler.java index cf003bd8c..0bc44523b 100644 --- a/src/main/java/io/odpf/firehose/sink/elasticsearch/request/EsRequestHandler.java +++ b/src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsRequestHandler.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.sink.elasticsearch.request; +package org.raystack.firehose.sink.elasticsearch.request; -import io.odpf.firehose.config.enums.EsSinkMessageType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.JsonParseException; -import io.odpf.firehose.serializer.MessageToJson; +import org.raystack.firehose.config.enums.EsSinkMessageType; +import org.raystack.firehose.exception.JsonParseException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.serializer.MessageToJson; import org.elasticsearch.action.DocWriteRequest; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; diff --git a/src/main/java/io/odpf/firehose/sink/elasticsearch/request/EsRequestHandlerFactory.java b/src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsRequestHandlerFactory.java similarity index 71% rename from src/main/java/io/odpf/firehose/sink/elasticsearch/request/EsRequestHandlerFactory.java rename to src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsRequestHandlerFactory.java index 94a687d99..cbd87317d 100644 --- a/src/main/java/io/odpf/firehose/sink/elasticsearch/request/EsRequestHandlerFactory.java +++ b/src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsRequestHandlerFactory.java @@ -1,17 +1,14 @@ -package io.odpf.firehose.sink.elasticsearch.request; +package org.raystack.firehose.sink.elasticsearch.request; -import io.odpf.firehose.config.EsSinkConfig; -import io.odpf.firehose.config.enums.EsSinkMessageType; -import io.odpf.firehose.config.enums.EsSinkRequestType; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.serializer.MessageToJson; +import org.raystack.firehose.config.EsSinkConfig; +import org.raystack.firehose.config.enums.EsSinkMessageType; +import org.raystack.firehose.config.enums.EsSinkRequestType; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.serializer.MessageToJson; import lombok.AllArgsConstructor; import java.util.ArrayList; -import static io.odpf.firehose.config.enums.EsSinkRequestType.INSERT_OR_UPDATE; -import static io.odpf.firehose.config.enums.EsSinkRequestType.UPDATE_ONLY; - @AllArgsConstructor public class EsRequestHandlerFactory { @@ -25,7 +22,7 @@ public class EsRequestHandlerFactory { private final String esRoutingKeyName; public EsRequestHandler getRequestHandler() { - EsSinkRequestType esSinkRequestType = esSinkConfig.isSinkEsModeUpdateOnlyEnable() ? UPDATE_ONLY : INSERT_OR_UPDATE; + EsSinkRequestType esSinkRequestType = esSinkConfig.isSinkEsModeUpdateOnlyEnable() ? EsSinkRequestType.UPDATE_ONLY : EsSinkRequestType.INSERT_OR_UPDATE; firehoseInstrumentation.logInfo("ES request mode: {}", esSinkRequestType); ArrayList esRequestHandlers = new ArrayList<>(); diff --git a/src/main/java/io/odpf/firehose/sink/elasticsearch/request/EsUpdateRequestHandler.java b/src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsUpdateRequestHandler.java similarity index 84% rename from src/main/java/io/odpf/firehose/sink/elasticsearch/request/EsUpdateRequestHandler.java rename to src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsUpdateRequestHandler.java index 69742a5cc..e1aad5eb8 100644 --- a/src/main/java/io/odpf/firehose/sink/elasticsearch/request/EsUpdateRequestHandler.java +++ b/src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsUpdateRequestHandler.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.sink.elasticsearch.request; +package org.raystack.firehose.sink.elasticsearch.request; -import io.odpf.firehose.config.enums.EsSinkMessageType; -import io.odpf.firehose.config.enums.EsSinkRequestType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.serializer.MessageToJson; +import org.raystack.firehose.config.enums.EsSinkMessageType; +import org.raystack.firehose.config.enums.EsSinkRequestType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.serializer.MessageToJson; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.update.UpdateRequest; diff --git a/src/main/java/io/odpf/firehose/sink/elasticsearch/request/EsUpsertRequestHandler.java b/src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsUpsertRequestHandler.java similarity index 84% rename from src/main/java/io/odpf/firehose/sink/elasticsearch/request/EsUpsertRequestHandler.java rename to src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsUpsertRequestHandler.java index ff7a58c49..3e56728b6 100644 --- a/src/main/java/io/odpf/firehose/sink/elasticsearch/request/EsUpsertRequestHandler.java +++ b/src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsUpsertRequestHandler.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.sink.elasticsearch.request; +package org.raystack.firehose.sink.elasticsearch.request; -import io.odpf.firehose.config.enums.EsSinkMessageType; -import io.odpf.firehose.config.enums.EsSinkRequestType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.serializer.MessageToJson; +import org.raystack.firehose.config.enums.EsSinkMessageType; +import org.raystack.firehose.config.enums.EsSinkRequestType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.serializer.MessageToJson; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.index.IndexRequest; diff --git a/src/main/java/io/odpf/firehose/sink/grpc/GrpcSink.java b/src/main/java/org/raystack/firehose/sink/grpc/GrpcSink.java similarity index 82% rename from src/main/java/io/odpf/firehose/sink/grpc/GrpcSink.java rename to src/main/java/org/raystack/firehose/sink/grpc/GrpcSink.java index 055d7a0f7..add95a6bf 100644 --- a/src/main/java/io/odpf/firehose/sink/grpc/GrpcSink.java +++ b/src/main/java/org/raystack/firehose/sink/grpc/GrpcSink.java @@ -1,14 +1,14 @@ -package io.odpf.firehose.sink.grpc; +package org.raystack.firehose.sink.grpc; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.AbstractSink; -import io.odpf.firehose.sink.grpc.client.GrpcClient; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.AbstractSink; +import org.raystack.firehose.sink.grpc.client.GrpcClient; import com.google.protobuf.DynamicMessage; -import io.odpf.stencil.client.StencilClient; +import org.raystack.stencil.client.StencilClient; import java.io.IOException; import java.util.ArrayList; @@ -16,7 +16,7 @@ /** * GrpcSink allows messages consumed from kafka to be relayed to a http service. - * The related configurations for HTTPSink can be found here: {@see io.odpf.firehose.config.HTTPSinkConfig} + * The related configurations for HTTPSink can be found here: {@see org.raystack.firehose.config.HTTPSinkConfig} */ public class GrpcSink extends AbstractSink { diff --git a/src/main/java/io/odpf/firehose/sink/grpc/GrpcSinkFactory.java b/src/main/java/org/raystack/firehose/sink/grpc/GrpcSinkFactory.java similarity index 82% rename from src/main/java/io/odpf/firehose/sink/grpc/GrpcSinkFactory.java rename to src/main/java/org/raystack/firehose/sink/grpc/GrpcSinkFactory.java index 023862730..43e6b5a13 100644 --- a/src/main/java/io/odpf/firehose/sink/grpc/GrpcSinkFactory.java +++ b/src/main/java/org/raystack/firehose/sink/grpc/GrpcSinkFactory.java @@ -1,14 +1,14 @@ -package io.odpf.firehose.sink.grpc; +package org.raystack.firehose.sink.grpc; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.GrpcSinkConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.AbstractSink; -import io.odpf.firehose.sink.grpc.client.GrpcClient; +import org.raystack.firehose.config.GrpcSinkConfig; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.grpc.client.GrpcClient; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.sink.AbstractSink; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; -import io.odpf.stencil.client.StencilClient; +import org.raystack.stencil.client.StencilClient; import org.aeonbits.owner.ConfigFactory; import java.util.Map; diff --git a/src/main/java/io/odpf/firehose/sink/grpc/client/GrpcClient.java b/src/main/java/org/raystack/firehose/sink/grpc/client/GrpcClient.java similarity index 93% rename from src/main/java/io/odpf/firehose/sink/grpc/client/GrpcClient.java rename to src/main/java/org/raystack/firehose/sink/grpc/client/GrpcClient.java index bf4c5971d..2c782cb95 100644 --- a/src/main/java/io/odpf/firehose/sink/grpc/client/GrpcClient.java +++ b/src/main/java/org/raystack/firehose/sink/grpc/client/GrpcClient.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.sink.grpc.client; +package org.raystack.firehose.sink.grpc.client; -import io.odpf.firehose.config.GrpcSinkConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.config.GrpcSinkConfig; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import com.google.protobuf.DynamicMessage; import io.grpc.ManagedChannel; @@ -14,7 +14,7 @@ import io.grpc.Channel; import io.grpc.ClientInterceptors; import io.grpc.CallOptions; -import io.odpf.stencil.client.StencilClient; +import org.raystack.stencil.client.StencilClient; import org.apache.commons.io.IOUtils; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.Headers; diff --git a/src/main/java/io/odpf/firehose/sink/http/HttpSink.java b/src/main/java/org/raystack/firehose/sink/http/HttpSink.java similarity index 82% rename from src/main/java/io/odpf/firehose/sink/http/HttpSink.java rename to src/main/java/org/raystack/firehose/sink/http/HttpSink.java index bba87b921..eaffdcd31 100644 --- a/src/main/java/io/odpf/firehose/sink/http/HttpSink.java +++ b/src/main/java/org/raystack/firehose/sink/http/HttpSink.java @@ -1,12 +1,13 @@ -package io.odpf.firehose.sink.http; +package org.raystack.firehose.sink.http; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.common.AbstractHttpSink; -import io.odpf.firehose.sink.http.request.types.Request; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.sink.common.AbstractHttpSink; +import org.raystack.firehose.sink.http.request.types.Request; +import org.raystack.stencil.client.StencilClient; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; @@ -23,8 +24,6 @@ import java.util.Map; import java.util.stream.Collectors; -import static io.odpf.firehose.metrics.Metrics.SINK_MESSAGES_DROP_TOTAL; - /** * HttpSink implement {@link AbstractHttpSink } lifecycle for HTTP. @@ -72,7 +71,7 @@ protected void captureMessageDropCount(HttpResponse response, List conte List result = Arrays.asList(requestBody.replaceAll("^\\[|]$", "").split("},\\s*\\{")); - getFirehoseInstrumentation().captureCount(SINK_MESSAGES_DROP_TOTAL, (long) result.size(), "cause= " + statusCode(response)); + getFirehoseInstrumentation().captureCount(Metrics.SINK_MESSAGES_DROP_TOTAL, (long) result.size(), "cause= " + statusCode(response)); getFirehoseInstrumentation().logInfo("Message dropped because of status code: " + statusCode(response)); } } diff --git a/src/main/java/io/odpf/firehose/sink/http/HttpSinkFactory.java b/src/main/java/org/raystack/firehose/sink/http/HttpSinkFactory.java similarity index 86% rename from src/main/java/io/odpf/firehose/sink/http/HttpSinkFactory.java rename to src/main/java/org/raystack/firehose/sink/http/HttpSinkFactory.java index 3f6c8d964..e3b930aa8 100644 --- a/src/main/java/io/odpf/firehose/sink/http/HttpSinkFactory.java +++ b/src/main/java/org/raystack/firehose/sink/http/HttpSinkFactory.java @@ -1,15 +1,15 @@ -package io.odpf.firehose.sink.http; +package org.raystack.firehose.sink.http; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.AbstractSink; -import io.odpf.firehose.sink.http.auth.OAuth2Credential; -import io.odpf.firehose.sink.http.request.types.Request; -import io.odpf.firehose.sink.http.request.RequestFactory; -import io.odpf.firehose.sink.http.request.uri.UriParser; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.http.auth.OAuth2Credential; +import org.raystack.firehose.sink.http.request.RequestFactory; +import org.raystack.firehose.sink.http.request.types.Request; +import org.raystack.firehose.sink.http.request.uri.UriParser; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.sink.AbstractSink; +import org.raystack.stencil.client.StencilClient; import org.aeonbits.owner.ConfigFactory; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; diff --git a/src/main/java/io/odpf/firehose/sink/http/auth/OAuth2AccessToken.java b/src/main/java/org/raystack/firehose/sink/http/auth/OAuth2AccessToken.java similarity index 94% rename from src/main/java/io/odpf/firehose/sink/http/auth/OAuth2AccessToken.java rename to src/main/java/org/raystack/firehose/sink/http/auth/OAuth2AccessToken.java index 090ee9f4e..d2b6caf82 100644 --- a/src/main/java/io/odpf/firehose/sink/http/auth/OAuth2AccessToken.java +++ b/src/main/java/org/raystack/firehose/sink/http/auth/OAuth2AccessToken.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.http.auth; +package org.raystack.firehose.sink.http.auth; import org.joda.time.DateTimeUtils; diff --git a/src/main/java/io/odpf/firehose/sink/http/auth/OAuth2Client.java b/src/main/java/org/raystack/firehose/sink/http/auth/OAuth2Client.java similarity index 96% rename from src/main/java/io/odpf/firehose/sink/http/auth/OAuth2Client.java rename to src/main/java/org/raystack/firehose/sink/http/auth/OAuth2Client.java index 043922b68..3456f5c23 100644 --- a/src/main/java/io/odpf/firehose/sink/http/auth/OAuth2Client.java +++ b/src/main/java/org/raystack/firehose/sink/http/auth/OAuth2Client.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink.http.auth; +package org.raystack.firehose.sink.http.auth; -import io.odpf.firehose.exception.OAuth2Exception; +import org.raystack.firehose.exception.OAuth2Exception; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; diff --git a/src/main/java/io/odpf/firehose/sink/http/auth/OAuth2Credential.java b/src/main/java/org/raystack/firehose/sink/http/auth/OAuth2Credential.java similarity index 96% rename from src/main/java/io/odpf/firehose/sink/http/auth/OAuth2Credential.java rename to src/main/java/org/raystack/firehose/sink/http/auth/OAuth2Credential.java index 461327f91..730e0f359 100644 --- a/src/main/java/io/odpf/firehose/sink/http/auth/OAuth2Credential.java +++ b/src/main/java/org/raystack/firehose/sink/http/auth/OAuth2Credential.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink.http.auth; +package org.raystack.firehose.sink.http.auth; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import okhttp3.Interceptor; import okhttp3.Request; import okhttp3.Response; diff --git a/src/main/java/io/odpf/firehose/sink/http/factory/SerializerFactory.java b/src/main/java/org/raystack/firehose/sink/http/factory/SerializerFactory.java similarity index 73% rename from src/main/java/io/odpf/firehose/sink/http/factory/SerializerFactory.java rename to src/main/java/org/raystack/firehose/sink/http/factory/SerializerFactory.java index 9c7db35bf..a82e1d972 100644 --- a/src/main/java/io/odpf/firehose/sink/http/factory/SerializerFactory.java +++ b/src/main/java/org/raystack/firehose/sink/http/factory/SerializerFactory.java @@ -1,15 +1,15 @@ -package io.odpf.firehose.sink.http.factory; - -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.config.enums.HttpSinkDataFormatType; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.serializer.MessageSerializer; -import io.odpf.firehose.serializer.MessageToJson; -import io.odpf.firehose.serializer.MessageToTemplatizedJson; -import io.odpf.firehose.serializer.JsonWrappedProtoByte; -import io.odpf.stencil.client.StencilClient; -import io.odpf.stencil.Parser; +package org.raystack.firehose.sink.http.factory; + +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.config.enums.HttpSinkDataFormatType; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.serializer.JsonWrappedProtoByte; +import org.raystack.firehose.serializer.MessageSerializer; +import org.raystack.firehose.serializer.MessageToJson; +import org.raystack.firehose.serializer.MessageToTemplatizedJson; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.stencil.client.StencilClient; +import org.raystack.stencil.Parser; import lombok.AllArgsConstructor; /** @@ -34,7 +34,7 @@ public MessageSerializer build() { Parser protoParser = stencilClient.getParser(httpSinkConfig.getInputSchemaProtoClass()); if (httpSinkConfig.getSinkHttpJsonBodyTemplate().isEmpty()) { firehoseInstrumentation.logDebug("Serializer type: EsbMessageToJson", HttpSinkDataFormatType.JSON); - return new MessageToJson(protoParser, false, true); + return new MessageToJson(protoParser, false, httpSinkConfig.getSinkHttpSimpleDateFormatEnable()); } else { firehoseInstrumentation.logDebug("Serializer type: EsbMessageToTemplatizedJson"); return MessageToTemplatizedJson.create(new FirehoseInstrumentation(statsDReporter, MessageToTemplatizedJson.class), httpSinkConfig.getSinkHttpJsonBodyTemplate(), protoParser); diff --git a/src/main/java/io/odpf/firehose/sink/http/request/HttpRequestMethodFactory.java b/src/main/java/org/raystack/firehose/sink/http/request/HttpRequestMethodFactory.java similarity index 82% rename from src/main/java/io/odpf/firehose/sink/http/request/HttpRequestMethodFactory.java rename to src/main/java/org/raystack/firehose/sink/http/request/HttpRequestMethodFactory.java index 7362fd84d..37e4dea02 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/HttpRequestMethodFactory.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/HttpRequestMethodFactory.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.sink.http.request; +package org.raystack.firehose.sink.http.request; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; -import io.odpf.firehose.sink.http.request.method.HttpDeleteWithBody; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.sink.http.request.method.HttpDeleteWithBody; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import org.apache.http.client.methods.HttpPatch; import org.apache.http.client.methods.HttpPost; diff --git a/src/main/java/io/odpf/firehose/sink/http/request/RequestFactory.java b/src/main/java/org/raystack/firehose/sink/http/request/RequestFactory.java similarity index 70% rename from src/main/java/io/odpf/firehose/sink/http/request/RequestFactory.java rename to src/main/java/org/raystack/firehose/sink/http/request/RequestFactory.java index b90360432..16e47c944 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/RequestFactory.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/RequestFactory.java @@ -1,25 +1,25 @@ -package io.odpf.firehose.sink.http.request; +package org.raystack.firehose.sink.http.request; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.proto.ProtoToFieldMapper; -import io.odpf.firehose.serializer.MessageSerializer; -import io.odpf.firehose.sink.http.factory.SerializerFactory; -import io.odpf.firehose.sink.http.request.body.JsonBody; -import io.odpf.firehose.sink.http.request.entity.RequestEntityBuilder; -import io.odpf.firehose.sink.http.request.header.HeaderBuilder; -import io.odpf.firehose.sink.http.request.types.DynamicUrlRequest; -import io.odpf.firehose.sink.http.request.types.ParameterizedHeaderRequest; -import io.odpf.firehose.sink.http.request.types.ParameterizedUriRequest; -import io.odpf.firehose.sink.http.request.types.Request; -import io.odpf.firehose.sink.http.request.types.SimpleRequest; -import io.odpf.firehose.sink.http.request.uri.UriBuilder; -import io.odpf.firehose.sink.http.request.uri.UriParser; -import io.odpf.stencil.client.StencilClient; -import io.odpf.stencil.Parser; +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.proto.ProtoToFieldMapper; +import org.raystack.firehose.serializer.MessageSerializer; +import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder; +import org.raystack.firehose.sink.http.request.header.HeaderBuilder; +import org.raystack.firehose.sink.http.request.types.ParameterizedHeaderRequest; +import org.raystack.firehose.sink.http.request.types.ParameterizedUriRequest; +import org.raystack.firehose.sink.http.request.types.Request; +import org.raystack.firehose.sink.http.request.types.SimpleRequest; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.sink.http.factory.SerializerFactory; +import org.raystack.firehose.sink.http.request.body.JsonBody; +import org.raystack.firehose.sink.http.request.types.DynamicUrlRequest; +import org.raystack.firehose.sink.http.request.uri.UriBuilder; +import org.raystack.firehose.sink.http.request.uri.UriParser; +import org.raystack.stencil.client.StencilClient; +import org.raystack.stencil.Parser; import java.util.Arrays; import java.util.List; diff --git a/src/main/java/io/odpf/firehose/sink/http/request/body/JsonBody.java b/src/main/java/org/raystack/firehose/sink/http/request/body/JsonBody.java similarity index 80% rename from src/main/java/io/odpf/firehose/sink/http/request/body/JsonBody.java rename to src/main/java/org/raystack/firehose/sink/http/request/body/JsonBody.java index 8b534dcbb..04c16044c 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/body/JsonBody.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/body/JsonBody.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sink.http.request.body; +package org.raystack.firehose.sink.http.request.body; import java.util.ArrayList; import java.util.List; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.serializer.MessageSerializer; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.serializer.MessageSerializer; /** * JsonBody Serialize the message according to injected serialzier and return it diff --git a/src/main/java/io/odpf/firehose/sink/http/request/create/BatchRequestCreator.java b/src/main/java/org/raystack/firehose/sink/http/request/create/BatchRequestCreator.java similarity index 77% rename from src/main/java/io/odpf/firehose/sink/http/request/create/BatchRequestCreator.java rename to src/main/java/org/raystack/firehose/sink/http/request/create/BatchRequestCreator.java index 0fb22455a..43ba0af46 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/create/BatchRequestCreator.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/create/BatchRequestCreator.java @@ -1,14 +1,14 @@ -package io.odpf.firehose.sink.http.request.create; - -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.http.request.HttpRequestMethodFactory; -import io.odpf.firehose.sink.http.request.body.JsonBody; -import io.odpf.firehose.sink.http.request.entity.RequestEntityBuilder; -import io.odpf.firehose.sink.http.request.header.HeaderBuilder; -import io.odpf.firehose.sink.http.request.uri.UriBuilder; +package org.raystack.firehose.sink.http.request.create; + +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder; +import org.raystack.firehose.sink.http.request.header.HeaderBuilder; +import org.raystack.firehose.sink.http.request.HttpRequestMethodFactory; +import org.raystack.firehose.sink.http.request.body.JsonBody; +import org.raystack.firehose.sink.http.request.uri.UriBuilder; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import java.net.URI; diff --git a/src/main/java/io/odpf/firehose/sink/http/request/create/IndividualRequestCreator.java b/src/main/java/org/raystack/firehose/sink/http/request/create/IndividualRequestCreator.java similarity index 79% rename from src/main/java/io/odpf/firehose/sink/http/request/create/IndividualRequestCreator.java rename to src/main/java/org/raystack/firehose/sink/http/request/create/IndividualRequestCreator.java index 867683ec8..2312b4dff 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/create/IndividualRequestCreator.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/create/IndividualRequestCreator.java @@ -1,14 +1,14 @@ -package io.odpf.firehose.sink.http.request.create; - -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.http.request.HttpRequestMethodFactory; -import io.odpf.firehose.sink.http.request.body.JsonBody; -import io.odpf.firehose.sink.http.request.entity.RequestEntityBuilder; -import io.odpf.firehose.sink.http.request.header.HeaderBuilder; -import io.odpf.firehose.sink.http.request.uri.UriBuilder; +package org.raystack.firehose.sink.http.request.create; + +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder; +import org.raystack.firehose.sink.http.request.header.HeaderBuilder; +import org.raystack.firehose.sink.http.request.HttpRequestMethodFactory; +import org.raystack.firehose.sink.http.request.body.JsonBody; +import org.raystack.firehose.sink.http.request.uri.UriBuilder; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import java.net.URI; diff --git a/src/main/java/io/odpf/firehose/sink/http/request/create/RequestCreator.java b/src/main/java/org/raystack/firehose/sink/http/request/create/RequestCreator.java similarity index 65% rename from src/main/java/io/odpf/firehose/sink/http/request/create/RequestCreator.java rename to src/main/java/org/raystack/firehose/sink/http/request/create/RequestCreator.java index a12bcad65..b8986b2f5 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/create/RequestCreator.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/create/RequestCreator.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.sink.http.request.create; +package org.raystack.firehose.sink.http.request.create; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.sink.http.request.entity.RequestEntityBuilder; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import java.net.URISyntaxException; diff --git a/src/main/java/io/odpf/firehose/sink/http/request/entity/RequestEntityBuilder.java b/src/main/java/org/raystack/firehose/sink/http/request/entity/RequestEntityBuilder.java similarity index 90% rename from src/main/java/io/odpf/firehose/sink/http/request/entity/RequestEntityBuilder.java rename to src/main/java/org/raystack/firehose/sink/http/request/entity/RequestEntityBuilder.java index 3cacb8f58..5f145f5ce 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/entity/RequestEntityBuilder.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/entity/RequestEntityBuilder.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink.http.request.entity; +package org.raystack.firehose.sink.http.request.entity; -import io.odpf.firehose.exception.DeserializerException; +import org.raystack.firehose.exception.DeserializerException; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; diff --git a/src/main/java/io/odpf/firehose/sink/http/request/header/HeaderBuilder.java b/src/main/java/org/raystack/firehose/sink/http/request/header/HeaderBuilder.java similarity index 89% rename from src/main/java/io/odpf/firehose/sink/http/request/header/HeaderBuilder.java rename to src/main/java/org/raystack/firehose/sink/http/request/header/HeaderBuilder.java index 0284df72b..31ef05265 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/header/HeaderBuilder.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/header/HeaderBuilder.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.http.request.header; +package org.raystack.firehose.sink.http.request.header; -import io.odpf.firehose.config.enums.HttpSinkParameterSourceType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.proto.ProtoToFieldMapper; +import org.raystack.firehose.config.enums.HttpSinkParameterSourceType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.proto.ProtoToFieldMapper; import java.util.Arrays; import java.util.Map; diff --git a/src/main/java/io/odpf/firehose/sink/http/request/method/HttpDeleteWithBody.java b/src/main/java/org/raystack/firehose/sink/http/request/method/HttpDeleteWithBody.java similarity index 87% rename from src/main/java/io/odpf/firehose/sink/http/request/method/HttpDeleteWithBody.java rename to src/main/java/org/raystack/firehose/sink/http/request/method/HttpDeleteWithBody.java index d2b4380fb..5ebc8045a 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/method/HttpDeleteWithBody.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/method/HttpDeleteWithBody.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.http.request.method; +package org.raystack.firehose.sink.http.request.method; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; diff --git a/src/main/java/io/odpf/firehose/sink/http/request/types/DynamicUrlRequest.java b/src/main/java/org/raystack/firehose/sink/http/request/types/DynamicUrlRequest.java similarity index 71% rename from src/main/java/io/odpf/firehose/sink/http/request/types/DynamicUrlRequest.java rename to src/main/java/org/raystack/firehose/sink/http/request/types/DynamicUrlRequest.java index 5a4f889bf..5e81bf3b0 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/types/DynamicUrlRequest.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/types/DynamicUrlRequest.java @@ -1,18 +1,18 @@ -package io.odpf.firehose.sink.http.request.types; +package org.raystack.firehose.sink.http.request.types; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; -import io.odpf.firehose.config.enums.HttpSinkParameterSourceType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.http.request.body.JsonBody; -import io.odpf.firehose.sink.http.request.create.IndividualRequestCreator; -import io.odpf.firehose.sink.http.request.create.RequestCreator; -import io.odpf.firehose.sink.http.request.entity.RequestEntityBuilder; -import io.odpf.firehose.sink.http.request.header.HeaderBuilder; -import io.odpf.firehose.sink.http.request.uri.UriBuilder; +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.config.enums.HttpSinkParameterSourceType; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.http.request.body.JsonBody; +import org.raystack.firehose.sink.http.request.create.IndividualRequestCreator; +import org.raystack.firehose.sink.http.request.create.RequestCreator; +import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder; +import org.raystack.firehose.sink.http.request.header.HeaderBuilder; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.sink.http.request.uri.UriBuilder; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import java.net.URISyntaxException; diff --git a/src/main/java/io/odpf/firehose/sink/http/request/types/ParameterizedHeaderRequest.java b/src/main/java/org/raystack/firehose/sink/http/request/types/ParameterizedHeaderRequest.java similarity index 72% rename from src/main/java/io/odpf/firehose/sink/http/request/types/ParameterizedHeaderRequest.java rename to src/main/java/org/raystack/firehose/sink/http/request/types/ParameterizedHeaderRequest.java index 1d9c83efb..3d069e33c 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/types/ParameterizedHeaderRequest.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/types/ParameterizedHeaderRequest.java @@ -1,26 +1,25 @@ -package io.odpf.firehose.sink.http.request.types; +package org.raystack.firehose.sink.http.request.types; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; -import io.odpf.firehose.config.enums.HttpSinkParameterSourceType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.proto.ProtoToFieldMapper; -import io.odpf.firehose.sink.http.request.body.JsonBody; -import io.odpf.firehose.sink.http.request.create.IndividualRequestCreator; -import io.odpf.firehose.sink.http.request.create.RequestCreator; -import io.odpf.firehose.sink.http.request.entity.RequestEntityBuilder; -import io.odpf.firehose.sink.http.request.header.HeaderBuilder; -import io.odpf.firehose.sink.http.request.uri.UriBuilder; +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.config.enums.HttpSinkParameterPlacementType; +import org.raystack.firehose.config.enums.HttpSinkParameterSourceType; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.proto.ProtoToFieldMapper; +import org.raystack.firehose.sink.http.request.header.HeaderBuilder; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.sink.http.request.body.JsonBody; +import org.raystack.firehose.sink.http.request.create.IndividualRequestCreator; +import org.raystack.firehose.sink.http.request.create.RequestCreator; +import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder; +import org.raystack.firehose.sink.http.request.uri.UriBuilder; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import java.net.URISyntaxException; import java.util.List; -import static io.odpf.firehose.config.enums.HttpSinkParameterPlacementType.HEADER; - /** * ParameterizedRequest create one HttpPut per-message. Uri and Header are * parametrized according to incoming message. @@ -82,6 +81,6 @@ public Request setRequestStrategy(HeaderBuilder headerBuilder, UriBuilder uriBui @Override public boolean canProcess() { return httpSinkConfig.getSinkHttpParameterSource() != HttpSinkParameterSourceType.DISABLED - && httpSinkConfig.getSinkHttpParameterPlacement() == HEADER; + && httpSinkConfig.getSinkHttpParameterPlacement() == HttpSinkParameterPlacementType.HEADER; } } diff --git a/src/main/java/io/odpf/firehose/sink/http/request/types/ParameterizedUriRequest.java b/src/main/java/org/raystack/firehose/sink/http/request/types/ParameterizedUriRequest.java similarity index 71% rename from src/main/java/io/odpf/firehose/sink/http/request/types/ParameterizedUriRequest.java rename to src/main/java/org/raystack/firehose/sink/http/request/types/ParameterizedUriRequest.java index 055c06d13..0aa15bcbc 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/types/ParameterizedUriRequest.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/types/ParameterizedUriRequest.java @@ -1,26 +1,25 @@ -package io.odpf.firehose.sink.http.request.types; +package org.raystack.firehose.sink.http.request.types; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; -import io.odpf.firehose.config.enums.HttpSinkParameterSourceType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.proto.ProtoToFieldMapper; -import io.odpf.firehose.sink.http.request.body.JsonBody; -import io.odpf.firehose.sink.http.request.create.IndividualRequestCreator; -import io.odpf.firehose.sink.http.request.create.RequestCreator; -import io.odpf.firehose.sink.http.request.entity.RequestEntityBuilder; -import io.odpf.firehose.sink.http.request.header.HeaderBuilder; -import io.odpf.firehose.sink.http.request.uri.UriBuilder; +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.config.enums.HttpSinkParameterPlacementType; +import org.raystack.firehose.config.enums.HttpSinkParameterSourceType; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.proto.ProtoToFieldMapper; +import org.raystack.firehose.sink.http.request.header.HeaderBuilder; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.sink.http.request.body.JsonBody; +import org.raystack.firehose.sink.http.request.create.IndividualRequestCreator; +import org.raystack.firehose.sink.http.request.create.RequestCreator; +import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder; +import org.raystack.firehose.sink.http.request.uri.UriBuilder; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import java.net.URISyntaxException; import java.util.List; -import static io.odpf.firehose.config.enums.HttpSinkParameterPlacementType.QUERY; - /** * The type Parameterized uri request. */ @@ -81,6 +80,6 @@ public Request setRequestStrategy(HeaderBuilder headerBuilder, UriBuilder uriBui @Override public boolean canProcess() { return httpSinkConfig.getSinkHttpParameterSource() != HttpSinkParameterSourceType.DISABLED - && httpSinkConfig.getSinkHttpParameterPlacement() == QUERY; + && httpSinkConfig.getSinkHttpParameterPlacement() == HttpSinkParameterPlacementType.QUERY; } } diff --git a/src/main/java/io/odpf/firehose/sink/http/request/types/Request.java b/src/main/java/org/raystack/firehose/sink/http/request/types/Request.java similarity index 75% rename from src/main/java/io/odpf/firehose/sink/http/request/types/Request.java rename to src/main/java/org/raystack/firehose/sink/http/request/types/Request.java index 8a0a9ee9a..e43b8a875 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/types/Request.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/types/Request.java @@ -1,12 +1,12 @@ -package io.odpf.firehose.sink.http.request.types; +package org.raystack.firehose.sink.http.request.types; -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.config.enums.HttpSinkDataFormatType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.sink.http.request.entity.RequestEntityBuilder; -import io.odpf.firehose.sink.http.request.header.HeaderBuilder; -import io.odpf.firehose.sink.http.request.uri.UriBuilder; +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.config.enums.HttpSinkDataFormatType; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.sink.http.request.header.HeaderBuilder; +import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder; +import org.raystack.firehose.sink.http.request.uri.UriBuilder; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import java.net.URISyntaxException; diff --git a/src/main/java/io/odpf/firehose/sink/http/request/types/SimpleRequest.java b/src/main/java/org/raystack/firehose/sink/http/request/types/SimpleRequest.java similarity index 72% rename from src/main/java/io/odpf/firehose/sink/http/request/types/SimpleRequest.java rename to src/main/java/org/raystack/firehose/sink/http/request/types/SimpleRequest.java index 719b5ca0d..4e40e8d3c 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/types/SimpleRequest.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/types/SimpleRequest.java @@ -1,19 +1,19 @@ -package io.odpf.firehose.sink.http.request.types; +package org.raystack.firehose.sink.http.request.types; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; -import io.odpf.firehose.config.enums.HttpSinkParameterSourceType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.http.request.body.JsonBody; -import io.odpf.firehose.sink.http.request.create.BatchRequestCreator; -import io.odpf.firehose.sink.http.request.create.IndividualRequestCreator; -import io.odpf.firehose.sink.http.request.create.RequestCreator; -import io.odpf.firehose.sink.http.request.entity.RequestEntityBuilder; -import io.odpf.firehose.sink.http.request.header.HeaderBuilder; -import io.odpf.firehose.sink.http.request.uri.UriBuilder; +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.config.enums.HttpSinkParameterSourceType; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.http.request.header.HeaderBuilder; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.sink.http.request.body.JsonBody; +import org.raystack.firehose.sink.http.request.create.BatchRequestCreator; +import org.raystack.firehose.sink.http.request.create.IndividualRequestCreator; +import org.raystack.firehose.sink.http.request.create.RequestCreator; +import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder; +import org.raystack.firehose.sink.http.request.uri.UriBuilder; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import java.net.URISyntaxException; diff --git a/src/main/java/io/odpf/firehose/sink/http/request/uri/UriBuilder.java b/src/main/java/org/raystack/firehose/sink/http/request/uri/UriBuilder.java similarity index 87% rename from src/main/java/io/odpf/firehose/sink/http/request/uri/UriBuilder.java rename to src/main/java/org/raystack/firehose/sink/http/request/uri/UriBuilder.java index c7fe9d508..209c78a43 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/uri/UriBuilder.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/uri/UriBuilder.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.http.request.uri; +package org.raystack.firehose.sink.http.request.uri; -import io.odpf.firehose.config.enums.HttpSinkParameterSourceType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.proto.ProtoToFieldMapper; +import org.raystack.firehose.config.enums.HttpSinkParameterSourceType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.proto.ProtoToFieldMapper; import java.net.URI; import java.net.URISyntaxException; diff --git a/src/main/java/io/odpf/firehose/sink/http/request/uri/UriParser.java b/src/main/java/org/raystack/firehose/sink/http/request/uri/UriParser.java similarity index 96% rename from src/main/java/io/odpf/firehose/sink/http/request/uri/UriParser.java rename to src/main/java/org/raystack/firehose/sink/http/request/uri/UriParser.java index 0a5b21f02..10ae50539 100644 --- a/src/main/java/io/odpf/firehose/sink/http/request/uri/UriParser.java +++ b/src/main/java/org/raystack/firehose/sink/http/request/uri/UriParser.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sink.http.request.uri; +package org.raystack.firehose.sink.http.request.uri; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.message.Message; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; -import io.odpf.stencil.Parser; +import org.raystack.stencil.Parser; import org.apache.commons.lang3.StringUtils; import org.apache.kafka.common.errors.InvalidConfigurationException; diff --git a/src/main/java/io/odpf/firehose/sink/influxdb/InfluxSink.java b/src/main/java/org/raystack/firehose/sink/influxdb/InfluxSink.java similarity index 85% rename from src/main/java/io/odpf/firehose/sink/influxdb/InfluxSink.java rename to src/main/java/org/raystack/firehose/sink/influxdb/InfluxSink.java index 7e5cf3cd8..7959beb16 100644 --- a/src/main/java/io/odpf/firehose/sink/influxdb/InfluxSink.java +++ b/src/main/java/org/raystack/firehose/sink/influxdb/InfluxSink.java @@ -1,15 +1,15 @@ -package io.odpf.firehose.sink.influxdb; +package org.raystack.firehose.sink.influxdb; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.influxdb.builder.PointBuilder; -import io.odpf.firehose.config.InfluxSinkConfig; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.sink.AbstractSink; +import org.raystack.firehose.config.InfluxSinkConfig; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.influxdb.builder.PointBuilder; +import org.raystack.firehose.sink.AbstractSink; import com.google.protobuf.DynamicMessage; -import io.odpf.stencil.client.StencilClient; -import io.odpf.stencil.Parser; +import org.raystack.stencil.client.StencilClient; +import org.raystack.stencil.Parser; import org.influxdb.InfluxDB; import org.influxdb.dto.BatchPoints; import org.influxdb.dto.Point; diff --git a/src/main/java/io/odpf/firehose/sink/influxdb/InfluxSinkFactory.java b/src/main/java/org/raystack/firehose/sink/influxdb/InfluxSinkFactory.java similarity index 81% rename from src/main/java/io/odpf/firehose/sink/influxdb/InfluxSinkFactory.java rename to src/main/java/org/raystack/firehose/sink/influxdb/InfluxSinkFactory.java index 67766c235..e376a2f46 100644 --- a/src/main/java/io/odpf/firehose/sink/influxdb/InfluxSinkFactory.java +++ b/src/main/java/org/raystack/firehose/sink/influxdb/InfluxSinkFactory.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sink.influxdb; +package org.raystack.firehose.sink.influxdb; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.InfluxSinkConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.AbstractSink; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.config.InfluxSinkConfig; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.sink.AbstractSink; +import org.raystack.stencil.client.StencilClient; import org.aeonbits.owner.ConfigFactory; import org.influxdb.InfluxDB; import org.influxdb.InfluxDBFactory; diff --git a/src/main/java/io/odpf/firehose/sink/influxdb/builder/PointBuilder.java b/src/main/java/org/raystack/firehose/sink/influxdb/builder/PointBuilder.java similarity index 97% rename from src/main/java/io/odpf/firehose/sink/influxdb/builder/PointBuilder.java rename to src/main/java/org/raystack/firehose/sink/influxdb/builder/PointBuilder.java index 1960fcf81..96ee20aa6 100644 --- a/src/main/java/io/odpf/firehose/sink/influxdb/builder/PointBuilder.java +++ b/src/main/java/org/raystack/firehose/sink/influxdb/builder/PointBuilder.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.sink.influxdb.builder; +package org.raystack.firehose.sink.influxdb.builder; -import io.odpf.firehose.config.InfluxSinkConfig; -import io.odpf.firehose.exception.ConfigurationException; +import org.raystack.firehose.config.InfluxSinkConfig; +import org.raystack.firehose.exception.ConfigurationException; import com.google.protobuf.Descriptors; import com.google.protobuf.Duration; import com.google.protobuf.DynamicMessage; diff --git a/src/main/java/io/odpf/firehose/sink/jdbc/HikariJdbcConnectionPool.java b/src/main/java/org/raystack/firehose/sink/jdbc/HikariJdbcConnectionPool.java similarity index 98% rename from src/main/java/io/odpf/firehose/sink/jdbc/HikariJdbcConnectionPool.java rename to src/main/java/org/raystack/firehose/sink/jdbc/HikariJdbcConnectionPool.java index 66d11efae..ae1dcda7f 100644 --- a/src/main/java/io/odpf/firehose/sink/jdbc/HikariJdbcConnectionPool.java +++ b/src/main/java/org/raystack/firehose/sink/jdbc/HikariJdbcConnectionPool.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.jdbc; +package org.raystack.firehose.sink.jdbc; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; diff --git a/src/main/java/io/odpf/firehose/sink/jdbc/JdbcConnectionPool.java b/src/main/java/org/raystack/firehose/sink/jdbc/JdbcConnectionPool.java similarity index 94% rename from src/main/java/io/odpf/firehose/sink/jdbc/JdbcConnectionPool.java rename to src/main/java/org/raystack/firehose/sink/jdbc/JdbcConnectionPool.java index fa7a3988c..ec41c71c1 100644 --- a/src/main/java/io/odpf/firehose/sink/jdbc/JdbcConnectionPool.java +++ b/src/main/java/org/raystack/firehose/sink/jdbc/JdbcConnectionPool.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.jdbc; +package org.raystack.firehose.sink.jdbc; import java.sql.Connection; import java.sql.SQLException; diff --git a/src/main/java/io/odpf/firehose/sink/jdbc/JdbcMapper.java b/src/main/java/org/raystack/firehose/sink/jdbc/JdbcMapper.java similarity index 94% rename from src/main/java/io/odpf/firehose/sink/jdbc/JdbcMapper.java rename to src/main/java/org/raystack/firehose/sink/jdbc/JdbcMapper.java index 482b1e927..c5cc5b7bc 100644 --- a/src/main/java/io/odpf/firehose/sink/jdbc/JdbcMapper.java +++ b/src/main/java/org/raystack/firehose/sink/jdbc/JdbcMapper.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink.jdbc; +package org.raystack.firehose.sink.jdbc; -import io.odpf.firehose.sink.jdbc.field.JdbcFieldFactory; +import org.raystack.firehose.sink.jdbc.field.JdbcFieldFactory; import com.google.protobuf.Descriptors; import com.google.protobuf.Message; import lombok.Getter; diff --git a/src/main/java/io/odpf/firehose/sink/jdbc/JdbcSink.java b/src/main/java/org/raystack/firehose/sink/jdbc/JdbcSink.java similarity index 91% rename from src/main/java/io/odpf/firehose/sink/jdbc/JdbcSink.java rename to src/main/java/org/raystack/firehose/sink/jdbc/JdbcSink.java index 8f643c37d..a4f6b6963 100644 --- a/src/main/java/io/odpf/firehose/sink/jdbc/JdbcSink.java +++ b/src/main/java/org/raystack/firehose/sink/jdbc/JdbcSink.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.sink.jdbc; +package org.raystack.firehose.sink.jdbc; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.AbstractSink; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.AbstractSink; +import org.raystack.stencil.client.StencilClient; import java.io.IOException; import java.sql.Connection; @@ -16,7 +16,7 @@ /** * JDBC Sink allows messages consumed from kafka to be persisted to a database. - * The related configurations for JDBC Sink can be found here: {@see io.odpf.firehose.config.JdbcSinkConfig} + * The related configurations for JDBC Sink can be found here: {@see org.raystack.firehose.config.JdbcSinkConfig} */ public class JdbcSink extends AbstractSink { diff --git a/src/main/java/io/odpf/firehose/sink/jdbc/JdbcSinkFactory.java b/src/main/java/org/raystack/firehose/sink/jdbc/JdbcSinkFactory.java similarity index 86% rename from src/main/java/io/odpf/firehose/sink/jdbc/JdbcSinkFactory.java rename to src/main/java/org/raystack/firehose/sink/jdbc/JdbcSinkFactory.java index a40ebfd62..04027b65a 100644 --- a/src/main/java/io/odpf/firehose/sink/jdbc/JdbcSinkFactory.java +++ b/src/main/java/org/raystack/firehose/sink/jdbc/JdbcSinkFactory.java @@ -1,13 +1,13 @@ -package io.odpf.firehose.sink.jdbc; +package org.raystack.firehose.sink.jdbc; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.JdbcSinkConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.AbstractSink; -import io.odpf.firehose.proto.ProtoToFieldMapper; -import io.odpf.stencil.client.StencilClient; -import io.odpf.stencil.Parser; +import org.raystack.firehose.config.JdbcSinkConfig; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.proto.ProtoToFieldMapper; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.sink.AbstractSink; +import org.raystack.stencil.client.StencilClient; +import org.raystack.stencil.Parser; import org.aeonbits.owner.ConfigFactory; import java.util.Map; diff --git a/src/main/java/io/odpf/firehose/sink/jdbc/QueryTemplate.java b/src/main/java/org/raystack/firehose/sink/jdbc/QueryTemplate.java similarity index 96% rename from src/main/java/io/odpf/firehose/sink/jdbc/QueryTemplate.java rename to src/main/java/org/raystack/firehose/sink/jdbc/QueryTemplate.java index bb46fc8a4..e4fb4ae2e 100644 --- a/src/main/java/io/odpf/firehose/sink/jdbc/QueryTemplate.java +++ b/src/main/java/org/raystack/firehose/sink/jdbc/QueryTemplate.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.jdbc; +package org.raystack.firehose.sink.jdbc; -import io.odpf.firehose.config.JdbcSinkConfig; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.proto.ProtoToFieldMapper; +import org.raystack.firehose.config.JdbcSinkConfig; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.proto.ProtoToFieldMapper; import com.samskivert.mustache.Escapers; import com.samskivert.mustache.Mustache; import com.samskivert.mustache.Template; diff --git a/src/main/java/io/odpf/firehose/sink/jdbc/field/JdbcDefaultField.java b/src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcDefaultField.java similarity index 88% rename from src/main/java/io/odpf/firehose/sink/jdbc/field/JdbcDefaultField.java rename to src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcDefaultField.java index 1a1809af3..a247782ca 100644 --- a/src/main/java/io/odpf/firehose/sink/jdbc/field/JdbcDefaultField.java +++ b/src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcDefaultField.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.jdbc.field; +package org.raystack.firehose.sink.jdbc.field; public class JdbcDefaultField implements JdbcField { private Object columnValue; diff --git a/src/main/java/io/odpf/firehose/sink/jdbc/field/JdbcField.java b/src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcField.java similarity index 88% rename from src/main/java/io/odpf/firehose/sink/jdbc/field/JdbcField.java rename to src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcField.java index 8043def4c..e03bd9353 100644 --- a/src/main/java/io/odpf/firehose/sink/jdbc/field/JdbcField.java +++ b/src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcField.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.jdbc.field; +package org.raystack.firehose.sink.jdbc.field; /** * Jdbc field for JDBC sink. diff --git a/src/main/java/io/odpf/firehose/sink/jdbc/field/JdbcFieldFactory.java b/src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcFieldFactory.java similarity index 77% rename from src/main/java/io/odpf/firehose/sink/jdbc/field/JdbcFieldFactory.java rename to src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcFieldFactory.java index a925bb022..cb2d190c0 100644 --- a/src/main/java/io/odpf/firehose/sink/jdbc/field/JdbcFieldFactory.java +++ b/src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcFieldFactory.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.jdbc.field; +package org.raystack.firehose.sink.jdbc.field; -import io.odpf.firehose.sink.jdbc.field.message.JdbcCollectionField; -import io.odpf.firehose.sink.jdbc.field.message.JdbcDefaultMessageField; -import io.odpf.firehose.sink.jdbc.field.message.JdbcTimestampField; +import org.raystack.firehose.sink.jdbc.field.message.JdbcDefaultMessageField; +import org.raystack.firehose.sink.jdbc.field.message.JdbcCollectionField; +import org.raystack.firehose.sink.jdbc.field.message.JdbcTimestampField; import com.google.protobuf.Descriptors; import java.util.Arrays; diff --git a/src/main/java/io/odpf/firehose/sink/jdbc/field/JdbcMapField.java b/src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcMapField.java similarity index 96% rename from src/main/java/io/odpf/firehose/sink/jdbc/field/JdbcMapField.java rename to src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcMapField.java index 4f380c949..1445da020 100644 --- a/src/main/java/io/odpf/firehose/sink/jdbc/field/JdbcMapField.java +++ b/src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcMapField.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.jdbc.field; +package org.raystack.firehose.sink.jdbc.field; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; diff --git a/src/main/java/io/odpf/firehose/sink/jdbc/field/message/JdbcCollectionField.java b/src/main/java/org/raystack/firehose/sink/jdbc/field/message/JdbcCollectionField.java similarity index 92% rename from src/main/java/io/odpf/firehose/sink/jdbc/field/message/JdbcCollectionField.java rename to src/main/java/org/raystack/firehose/sink/jdbc/field/message/JdbcCollectionField.java index 05a79f1db..6416f186b 100644 --- a/src/main/java/io/odpf/firehose/sink/jdbc/field/message/JdbcCollectionField.java +++ b/src/main/java/org/raystack/firehose/sink/jdbc/field/message/JdbcCollectionField.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink.jdbc.field.message; +package org.raystack.firehose.sink.jdbc.field.message; -import io.odpf.firehose.sink.jdbc.field.JdbcField; +import org.raystack.firehose.sink.jdbc.field.JdbcField; import com.google.gson.GsonBuilder; import com.google.protobuf.Descriptors; import com.google.protobuf.Message; diff --git a/src/main/java/io/odpf/firehose/sink/jdbc/field/message/JdbcDefaultMessageField.java b/src/main/java/org/raystack/firehose/sink/jdbc/field/message/JdbcDefaultMessageField.java similarity index 89% rename from src/main/java/io/odpf/firehose/sink/jdbc/field/message/JdbcDefaultMessageField.java rename to src/main/java/org/raystack/firehose/sink/jdbc/field/message/JdbcDefaultMessageField.java index 22a00d7a1..a237da5c6 100644 --- a/src/main/java/io/odpf/firehose/sink/jdbc/field/message/JdbcDefaultMessageField.java +++ b/src/main/java/org/raystack/firehose/sink/jdbc/field/message/JdbcDefaultMessageField.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink.jdbc.field.message; +package org.raystack.firehose.sink.jdbc.field.message; -import io.odpf.firehose.sink.jdbc.field.JdbcField; +import org.raystack.firehose.sink.jdbc.field.JdbcField; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; import com.google.protobuf.util.JsonFormat; diff --git a/src/main/java/io/odpf/firehose/sink/jdbc/field/message/JdbcTimestampField.java b/src/main/java/org/raystack/firehose/sink/jdbc/field/message/JdbcTimestampField.java similarity index 91% rename from src/main/java/io/odpf/firehose/sink/jdbc/field/message/JdbcTimestampField.java rename to src/main/java/org/raystack/firehose/sink/jdbc/field/message/JdbcTimestampField.java index 46b0d05a9..d5737694e 100644 --- a/src/main/java/io/odpf/firehose/sink/jdbc/field/message/JdbcTimestampField.java +++ b/src/main/java/org/raystack/firehose/sink/jdbc/field/message/JdbcTimestampField.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink.jdbc.field.message; +package org.raystack.firehose.sink.jdbc.field.message; -import io.odpf.firehose.sink.jdbc.field.JdbcField; +import org.raystack.firehose.sink.jdbc.field.JdbcField; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import com.google.protobuf.Timestamp; diff --git a/src/main/java/io/odpf/firehose/sink/mongodb/MongoSink.java b/src/main/java/org/raystack/firehose/sink/mongodb/MongoSink.java similarity index 90% rename from src/main/java/io/odpf/firehose/sink/mongodb/MongoSink.java rename to src/main/java/org/raystack/firehose/sink/mongodb/MongoSink.java index 237eadff6..69dc68e1e 100644 --- a/src/main/java/io/odpf/firehose/sink/mongodb/MongoSink.java +++ b/src/main/java/org/raystack/firehose/sink/mongodb/MongoSink.java @@ -1,12 +1,12 @@ -package io.odpf.firehose.sink.mongodb; +package org.raystack.firehose.sink.mongodb; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.mongodb.request.MongoRequestHandler; import com.mongodb.bulk.BulkWriteError; import com.mongodb.client.model.WriteModel; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.AbstractSink; -import io.odpf.firehose.sink.mongodb.client.MongoSinkClient; -import io.odpf.firehose.sink.mongodb.request.MongoRequestHandler; +import org.raystack.firehose.sink.AbstractSink; +import org.raystack.firehose.sink.mongodb.client.MongoSinkClient; import org.bson.Document; import java.io.IOException; diff --git a/src/main/java/io/odpf/firehose/sink/mongodb/MongoSinkFactory.java b/src/main/java/org/raystack/firehose/sink/mongodb/MongoSinkFactory.java similarity index 90% rename from src/main/java/io/odpf/firehose/sink/mongodb/MongoSinkFactory.java rename to src/main/java/org/raystack/firehose/sink/mongodb/MongoSinkFactory.java index ce62fadac..b67d7f709 100644 --- a/src/main/java/io/odpf/firehose/sink/mongodb/MongoSinkFactory.java +++ b/src/main/java/org/raystack/firehose/sink/mongodb/MongoSinkFactory.java @@ -1,20 +1,20 @@ -package io.odpf.firehose.sink.mongodb; +package org.raystack.firehose.sink.mongodb; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.config.MongoSinkConfig; +import org.raystack.firehose.config.enums.SinkType; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.serializer.MessageToJson; +import org.raystack.firehose.sink.mongodb.client.MongoSinkClient; +import org.raystack.firehose.sink.mongodb.request.MongoRequestHandler; +import org.raystack.firehose.sink.mongodb.request.MongoRequestHandlerFactory; +import org.raystack.firehose.sink.mongodb.util.MongoSinkFactoryUtil; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.stencil.client.StencilClient; import com.mongodb.MongoClient; import com.mongodb.MongoClientOptions; import com.mongodb.MongoCredential; import com.mongodb.ServerAddress; -import io.odpf.firehose.config.MongoSinkConfig; -import io.odpf.firehose.config.enums.SinkType; -import io.odpf.firehose.serializer.MessageToJson; -import io.odpf.firehose.sink.Sink; -import io.odpf.firehose.sink.mongodb.client.MongoSinkClient; -import io.odpf.firehose.sink.mongodb.request.MongoRequestHandler; -import io.odpf.firehose.sink.mongodb.request.MongoRequestHandlerFactory; -import io.odpf.firehose.sink.mongodb.util.MongoSinkFactoryUtil; +import org.raystack.firehose.sink.Sink; import org.aeonbits.owner.ConfigFactory; import java.util.List; diff --git a/src/main/java/io/odpf/firehose/sink/mongodb/client/MongoSinkClient.java b/src/main/java/org/raystack/firehose/sink/mongodb/client/MongoSinkClient.java similarity index 90% rename from src/main/java/io/odpf/firehose/sink/mongodb/client/MongoSinkClient.java rename to src/main/java/org/raystack/firehose/sink/mongodb/client/MongoSinkClient.java index 4c71750ed..cf76035f6 100644 --- a/src/main/java/io/odpf/firehose/sink/mongodb/client/MongoSinkClient.java +++ b/src/main/java/org/raystack/firehose/sink/mongodb/client/MongoSinkClient.java @@ -1,5 +1,8 @@ -package io.odpf.firehose.sink.mongodb.client; +package org.raystack.firehose.sink.mongodb.client; +import org.raystack.firehose.config.MongoSinkConfig; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; import com.mongodb.MongoBulkWriteException; import com.mongodb.MongoClient; import com.mongodb.MongoCommandException; @@ -8,8 +11,6 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.WriteModel; -import io.odpf.firehose.config.MongoSinkConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; import lombok.AllArgsConstructor; import org.bson.Document; @@ -19,8 +20,6 @@ import java.util.List; import java.util.stream.Collectors; -import static io.odpf.firehose.metrics.Metrics.*; - /** * The Mongo Sink Client. * This class is responsible for creating and closing the MongoDB sink @@ -132,12 +131,12 @@ private void logResults(BulkWriteResult writeResult, int messageCount) { if (mongoSinkConfig.isSinkMongoModeUpdateOnlyEnable()) { for (int i = 0; i < failureCount; i++) { - firehoseInstrumentation.incrementCounter(SINK_MESSAGES_DROP_TOTAL, "cause=Primary Key value not found"); + firehoseInstrumentation.incrementCounter(Metrics.SINK_MESSAGES_DROP_TOTAL, "cause=Primary Key value not found"); } firehoseInstrumentation.logWarn("Some Messages were dropped because their Primary Key values had no matches"); } else { for (int i = 0; i < failureCount; i++) { - firehoseInstrumentation.incrementCounter(SINK_MESSAGES_DROP_TOTAL); + firehoseInstrumentation.incrementCounter(Metrics.SINK_MESSAGES_DROP_TOTAL); } } } @@ -157,13 +156,13 @@ private void logResults(BulkWriteResult writeResult, int messageCount) { totalWriteCount); for (int i = 0; i < totalInsertedCount; i++) { - firehoseInstrumentation.incrementCounter(SINK_MONGO_INSERTED_TOTAL); + firehoseInstrumentation.incrementCounter(Metrics.SINK_MONGO_INSERTED_TOTAL); } for (int i = 0; i < writeResult.getModifiedCount(); i++) { - firehoseInstrumentation.incrementCounter(SINK_MONGO_UPDATED_TOTAL); + firehoseInstrumentation.incrementCounter(Metrics.SINK_MONGO_UPDATED_TOTAL); } for (int i = 0; i < totalWriteCount; i++) { - firehoseInstrumentation.incrementCounter(SINK_MONGO_MODIFIED_TOTAL); + firehoseInstrumentation.incrementCounter(Metrics.SINK_MONGO_MODIFIED_TOTAL); } } @@ -183,7 +182,7 @@ private void logErrors(List writeErrors) { .filter(writeError -> mongoRetryStatusCodeBlacklist.contains(writeError.getCode())) .forEach(writeError -> { firehoseInstrumentation.logWarn("Non-retriable error due to response status: {} is under blacklisted status code", writeError.getCode()); - firehoseInstrumentation.incrementCounter(SINK_MESSAGES_DROP_TOTAL, "cause=" + writeError.getMessage()); + firehoseInstrumentation.incrementCounter(Metrics.SINK_MESSAGES_DROP_TOTAL, "cause=" + writeError.getMessage()); firehoseInstrumentation.logInfo("Message dropped because of status code: " + writeError.getCode()); }); diff --git a/src/main/java/io/odpf/firehose/sink/mongodb/client/MongoSinkClientUtil.java b/src/main/java/org/raystack/firehose/sink/mongodb/client/MongoSinkClientUtil.java similarity index 96% rename from src/main/java/io/odpf/firehose/sink/mongodb/client/MongoSinkClientUtil.java rename to src/main/java/org/raystack/firehose/sink/mongodb/client/MongoSinkClientUtil.java index 74e1326af..3e3b057a6 100644 --- a/src/main/java/io/odpf/firehose/sink/mongodb/client/MongoSinkClientUtil.java +++ b/src/main/java/org/raystack/firehose/sink/mongodb/client/MongoSinkClientUtil.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.sink.mongodb.client; +package org.raystack.firehose.sink.mongodb.client; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import com.mongodb.MongoClient; import com.mongodb.MongoNamespace; import com.mongodb.client.MongoDatabase; -import io.odpf.firehose.metrics.FirehoseInstrumentation; import lombok.experimental.UtilityClass; import java.util.ArrayList; diff --git a/src/main/java/io/odpf/firehose/sink/mongodb/request/MongoRequestHandler.java b/src/main/java/org/raystack/firehose/sink/mongodb/request/MongoRequestHandler.java similarity index 94% rename from src/main/java/io/odpf/firehose/sink/mongodb/request/MongoRequestHandler.java rename to src/main/java/org/raystack/firehose/sink/mongodb/request/MongoRequestHandler.java index 736509205..b07f057c3 100644 --- a/src/main/java/io/odpf/firehose/sink/mongodb/request/MongoRequestHandler.java +++ b/src/main/java/org/raystack/firehose/sink/mongodb/request/MongoRequestHandler.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.sink.mongodb.request; +package org.raystack.firehose.sink.mongodb.request; +import org.raystack.firehose.config.enums.MongoSinkMessageType; +import org.raystack.firehose.exception.JsonParseException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.serializer.MessageToJson; import com.mongodb.client.model.WriteModel; -import io.odpf.firehose.config.enums.MongoSinkMessageType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.JsonParseException; -import io.odpf.firehose.serializer.MessageToJson; import org.bson.Document; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; diff --git a/src/main/java/io/odpf/firehose/sink/mongodb/request/MongoRequestHandlerFactory.java b/src/main/java/org/raystack/firehose/sink/mongodb/request/MongoRequestHandlerFactory.java similarity index 80% rename from src/main/java/io/odpf/firehose/sink/mongodb/request/MongoRequestHandlerFactory.java rename to src/main/java/org/raystack/firehose/sink/mongodb/request/MongoRequestHandlerFactory.java index fe24c25d3..5e83cde53 100644 --- a/src/main/java/io/odpf/firehose/sink/mongodb/request/MongoRequestHandlerFactory.java +++ b/src/main/java/org/raystack/firehose/sink/mongodb/request/MongoRequestHandlerFactory.java @@ -1,17 +1,14 @@ -package io.odpf.firehose.sink.mongodb.request; +package org.raystack.firehose.sink.mongodb.request; -import io.odpf.firehose.config.MongoSinkConfig; -import io.odpf.firehose.config.enums.MongoSinkMessageType; -import io.odpf.firehose.config.enums.MongoSinkRequestType; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.serializer.MessageToJson; +import org.raystack.firehose.config.MongoSinkConfig; +import org.raystack.firehose.config.enums.MongoSinkMessageType; +import org.raystack.firehose.config.enums.MongoSinkRequestType; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.serializer.MessageToJson; import lombok.AllArgsConstructor; import java.util.ArrayList; -import static io.odpf.firehose.config.enums.MongoSinkRequestType.UPDATE_ONLY; -import static io.odpf.firehose.config.enums.MongoSinkRequestType.UPSERT; - /** * The Mongo request handler factory. */ @@ -45,9 +42,9 @@ public MongoRequestHandler getRequestHandler() { if (!kafkaRecordParserMode.equals("key") && !kafkaRecordParserMode.equals("message")) { throw new IllegalArgumentException("KAFKA_RECORD_PARSER_MODE should be key/message"); } - MongoSinkRequestType mongoSinkRequestType = mongoSinkConfig.isSinkMongoModeUpdateOnlyEnable() ? UPDATE_ONLY : UPSERT; + MongoSinkRequestType mongoSinkRequestType = mongoSinkConfig.isSinkMongoModeUpdateOnlyEnable() ? MongoSinkRequestType.UPDATE_ONLY : MongoSinkRequestType.UPSERT; firehoseInstrumentation.logInfo("Mongo request mode: {}", mongoSinkRequestType); - if (mongoSinkRequestType == UPDATE_ONLY && mongoPrimaryKey == null) { + if (mongoSinkRequestType == MongoSinkRequestType.UPDATE_ONLY && mongoPrimaryKey == null) { throw new IllegalArgumentException("Primary Key cannot be null in Update-Only mode"); } diff --git a/src/main/java/io/odpf/firehose/sink/mongodb/request/MongoUpdateRequestHandler.java b/src/main/java/org/raystack/firehose/sink/mongodb/request/MongoUpdateRequestHandler.java similarity index 86% rename from src/main/java/io/odpf/firehose/sink/mongodb/request/MongoUpdateRequestHandler.java rename to src/main/java/org/raystack/firehose/sink/mongodb/request/MongoUpdateRequestHandler.java index f69808bf3..7d831c8bb 100644 --- a/src/main/java/io/odpf/firehose/sink/mongodb/request/MongoUpdateRequestHandler.java +++ b/src/main/java/org/raystack/firehose/sink/mongodb/request/MongoUpdateRequestHandler.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.sink.mongodb.request; +package org.raystack.firehose.sink.mongodb.request; +import org.raystack.firehose.config.enums.MongoSinkMessageType; +import org.raystack.firehose.config.enums.MongoSinkRequestType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.serializer.MessageToJson; import com.mongodb.client.model.ReplaceOneModel; -import io.odpf.firehose.config.enums.MongoSinkMessageType; -import io.odpf.firehose.config.enums.MongoSinkRequestType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.serializer.MessageToJson; import org.bson.Document; import org.json.simple.JSONObject; diff --git a/src/main/java/io/odpf/firehose/sink/mongodb/request/MongoUpsertRequestHandler.java b/src/main/java/org/raystack/firehose/sink/mongodb/request/MongoUpsertRequestHandler.java similarity index 89% rename from src/main/java/io/odpf/firehose/sink/mongodb/request/MongoUpsertRequestHandler.java rename to src/main/java/org/raystack/firehose/sink/mongodb/request/MongoUpsertRequestHandler.java index f32f309c9..749cc82e4 100644 --- a/src/main/java/io/odpf/firehose/sink/mongodb/request/MongoUpsertRequestHandler.java +++ b/src/main/java/org/raystack/firehose/sink/mongodb/request/MongoUpsertRequestHandler.java @@ -1,13 +1,13 @@ -package io.odpf.firehose.sink.mongodb.request; +package org.raystack.firehose.sink.mongodb.request; +import org.raystack.firehose.config.enums.MongoSinkMessageType; +import org.raystack.firehose.config.enums.MongoSinkRequestType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.serializer.MessageToJson; import com.mongodb.client.model.InsertOneModel; import com.mongodb.client.model.ReplaceOneModel; import com.mongodb.client.model.ReplaceOptions; import com.mongodb.client.model.WriteModel; -import io.odpf.firehose.config.enums.MongoSinkMessageType; -import io.odpf.firehose.config.enums.MongoSinkRequestType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.serializer.MessageToJson; import org.bson.Document; import org.json.simple.JSONObject; diff --git a/src/main/java/io/odpf/firehose/sink/mongodb/util/MongoSinkFactoryUtil.java b/src/main/java/org/raystack/firehose/sink/mongodb/util/MongoSinkFactoryUtil.java similarity index 95% rename from src/main/java/io/odpf/firehose/sink/mongodb/util/MongoSinkFactoryUtil.java rename to src/main/java/org/raystack/firehose/sink/mongodb/util/MongoSinkFactoryUtil.java index c2cceb513..bed92b3c1 100644 --- a/src/main/java/io/odpf/firehose/sink/mongodb/util/MongoSinkFactoryUtil.java +++ b/src/main/java/org/raystack/firehose/sink/mongodb/util/MongoSinkFactoryUtil.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.sink.mongodb.util; +package org.raystack.firehose.sink.mongodb.util; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import com.mongodb.ServerAddress; -import io.odpf.firehose.metrics.FirehoseInstrumentation; import lombok.experimental.UtilityClass; import java.util.ArrayList; diff --git a/src/main/java/io/odpf/firehose/sink/prometheus/PromSink.java b/src/main/java/org/raystack/firehose/sink/prometheus/PromSink.java similarity index 83% rename from src/main/java/io/odpf/firehose/sink/prometheus/PromSink.java rename to src/main/java/org/raystack/firehose/sink/prometheus/PromSink.java index 0a28ffd41..8e2d19c95 100644 --- a/src/main/java/io/odpf/firehose/sink/prometheus/PromSink.java +++ b/src/main/java/org/raystack/firehose/sink/prometheus/PromSink.java @@ -1,14 +1,15 @@ -package io.odpf.firehose.sink.prometheus; +package org.raystack.firehose.sink.prometheus; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.common.AbstractHttpSink; -import io.odpf.firehose.sink.prometheus.request.PromRequest; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.sink.prometheus.request.PromRequest; +import org.raystack.firehose.sink.common.AbstractHttpSink; import com.google.protobuf.DynamicMessage; import cortexpb.Cortex; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.stencil.client.StencilClient; +import org.raystack.stencil.client.StencilClient; import org.apache.commons.io.IOUtils; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; @@ -22,8 +23,6 @@ import java.util.List; import java.util.Map; -import static io.odpf.firehose.metrics.Metrics.SINK_MESSAGES_DROP_TOTAL; - /** * the Prometheus Sink. this sink use prometheus remote write api to send data into Cortex. */ @@ -63,7 +62,7 @@ protected void prepare(List messages) throws DeserializerException, IOE } protected void captureMessageDropCount(HttpResponse response, List contentStringList) { - getFirehoseInstrumentation().captureCount(SINK_MESSAGES_DROP_TOTAL, (long) contentStringList.size(), "cause= " + statusCode(response)); + getFirehoseInstrumentation().captureCount(Metrics.SINK_MESSAGES_DROP_TOTAL, (long) contentStringList.size(), "cause= " + statusCode(response)); getFirehoseInstrumentation().logInfo("Message dropped because of status code: " + statusCode(response)); } diff --git a/src/main/java/io/odpf/firehose/sink/prometheus/PromSinkConstants.java b/src/main/java/org/raystack/firehose/sink/prometheus/PromSinkConstants.java similarity index 94% rename from src/main/java/io/odpf/firehose/sink/prometheus/PromSinkConstants.java rename to src/main/java/org/raystack/firehose/sink/prometheus/PromSinkConstants.java index 37d571c0c..8ce5aaf9e 100644 --- a/src/main/java/io/odpf/firehose/sink/prometheus/PromSinkConstants.java +++ b/src/main/java/org/raystack/firehose/sink/prometheus/PromSinkConstants.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.prometheus; +package org.raystack.firehose.sink.prometheus; public class PromSinkConstants { diff --git a/src/main/java/io/odpf/firehose/sink/prometheus/PromSinkFactory.java b/src/main/java/org/raystack/firehose/sink/prometheus/PromSinkFactory.java similarity index 86% rename from src/main/java/io/odpf/firehose/sink/prometheus/PromSinkFactory.java rename to src/main/java/org/raystack/firehose/sink/prometheus/PromSinkFactory.java index b9b5ccfd7..1dd27e849 100644 --- a/src/main/java/io/odpf/firehose/sink/prometheus/PromSinkFactory.java +++ b/src/main/java/org/raystack/firehose/sink/prometheus/PromSinkFactory.java @@ -1,13 +1,13 @@ -package io.odpf.firehose.sink.prometheus; +package org.raystack.firehose.sink.prometheus; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.PromSinkConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.AbstractSink; -import io.odpf.firehose.sink.prometheus.request.PromRequest; -import io.odpf.firehose.sink.prometheus.request.PromRequestCreator; -import io.odpf.stencil.client.StencilClient; -import io.odpf.stencil.Parser; +import org.raystack.firehose.config.PromSinkConfig; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.prometheus.request.PromRequest; +import org.raystack.firehose.sink.prometheus.request.PromRequestCreator; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.sink.AbstractSink; +import org.raystack.stencil.client.StencilClient; +import org.raystack.stencil.Parser; import org.aeonbits.owner.ConfigFactory; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; diff --git a/src/main/java/io/odpf/firehose/sink/prometheus/builder/HeaderBuilder.java b/src/main/java/org/raystack/firehose/sink/prometheus/builder/HeaderBuilder.java similarity index 89% rename from src/main/java/io/odpf/firehose/sink/prometheus/builder/HeaderBuilder.java rename to src/main/java/org/raystack/firehose/sink/prometheus/builder/HeaderBuilder.java index 43751c9a9..daf8ab22d 100644 --- a/src/main/java/io/odpf/firehose/sink/prometheus/builder/HeaderBuilder.java +++ b/src/main/java/org/raystack/firehose/sink/prometheus/builder/HeaderBuilder.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sink.prometheus.builder; +package org.raystack.firehose.sink.prometheus.builder; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; -import static io.odpf.firehose.sink.prometheus.PromSinkConstants.*; +import static org.raystack.firehose.sink.prometheus.PromSinkConstants.*; /** * Builder for prometheus request header. diff --git a/src/main/java/io/odpf/firehose/sink/prometheus/builder/PrometheusLabel.java b/src/main/java/org/raystack/firehose/sink/prometheus/builder/PrometheusLabel.java similarity index 79% rename from src/main/java/io/odpf/firehose/sink/prometheus/builder/PrometheusLabel.java rename to src/main/java/org/raystack/firehose/sink/prometheus/builder/PrometheusLabel.java index 50ea8112e..c0981faac 100644 --- a/src/main/java/io/odpf/firehose/sink/prometheus/builder/PrometheusLabel.java +++ b/src/main/java/org/raystack/firehose/sink/prometheus/builder/PrometheusLabel.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.prometheus.builder; +package org.raystack.firehose.sink.prometheus.builder; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/src/main/java/io/odpf/firehose/sink/prometheus/builder/PrometheusMetric.java b/src/main/java/org/raystack/firehose/sink/prometheus/builder/PrometheusMetric.java similarity index 79% rename from src/main/java/io/odpf/firehose/sink/prometheus/builder/PrometheusMetric.java rename to src/main/java/org/raystack/firehose/sink/prometheus/builder/PrometheusMetric.java index b7a0487b8..e99af1422 100644 --- a/src/main/java/io/odpf/firehose/sink/prometheus/builder/PrometheusMetric.java +++ b/src/main/java/org/raystack/firehose/sink/prometheus/builder/PrometheusMetric.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.prometheus.builder; +package org.raystack.firehose.sink.prometheus.builder; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/src/main/java/io/odpf/firehose/sink/prometheus/builder/RequestEntityBuilder.java b/src/main/java/org/raystack/firehose/sink/prometheus/builder/RequestEntityBuilder.java similarity index 93% rename from src/main/java/io/odpf/firehose/sink/prometheus/builder/RequestEntityBuilder.java rename to src/main/java/org/raystack/firehose/sink/prometheus/builder/RequestEntityBuilder.java index 4f145d05a..b9d1254af 100644 --- a/src/main/java/io/odpf/firehose/sink/prometheus/builder/RequestEntityBuilder.java +++ b/src/main/java/org/raystack/firehose/sink/prometheus/builder/RequestEntityBuilder.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.prometheus.builder; +package org.raystack.firehose.sink.prometheus.builder; import cortexpb.Cortex; import org.apache.http.entity.ByteArrayEntity; diff --git a/src/main/java/io/odpf/firehose/sink/prometheus/builder/TimeSeriesBuilder.java b/src/main/java/org/raystack/firehose/sink/prometheus/builder/TimeSeriesBuilder.java similarity index 94% rename from src/main/java/io/odpf/firehose/sink/prometheus/builder/TimeSeriesBuilder.java rename to src/main/java/org/raystack/firehose/sink/prometheus/builder/TimeSeriesBuilder.java index 0dd11ee97..246ae88ef 100644 --- a/src/main/java/io/odpf/firehose/sink/prometheus/builder/TimeSeriesBuilder.java +++ b/src/main/java/org/raystack/firehose/sink/prometheus/builder/TimeSeriesBuilder.java @@ -1,18 +1,18 @@ -package io.odpf.firehose.sink.prometheus.builder; +package org.raystack.firehose.sink.prometheus.builder; -import io.odpf.firehose.config.PromSinkConfig; +import org.raystack.firehose.config.PromSinkConfig; +import org.raystack.firehose.exception.ConfigurationException; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; import cortexpb.Cortex; -import io.odpf.firehose.exception.ConfigurationException; import java.util.List; import java.util.Properties; import java.util.Set; import java.util.stream.Collectors; -import static io.odpf.firehose.sink.prometheus.PromSinkConstants.*; +import static org.raystack.firehose.sink.prometheus.PromSinkConstants.*; /** * Builder for Cortex TimeSeries. diff --git a/src/main/java/io/odpf/firehose/sink/prometheus/builder/TimeSeriesBuilderUtils.java b/src/main/java/org/raystack/firehose/sink/prometheus/builder/TimeSeriesBuilderUtils.java similarity index 91% rename from src/main/java/io/odpf/firehose/sink/prometheus/builder/TimeSeriesBuilderUtils.java rename to src/main/java/org/raystack/firehose/sink/prometheus/builder/TimeSeriesBuilderUtils.java index 814c4653a..5b4135e94 100644 --- a/src/main/java/io/odpf/firehose/sink/prometheus/builder/TimeSeriesBuilderUtils.java +++ b/src/main/java/org/raystack/firehose/sink/prometheus/builder/TimeSeriesBuilderUtils.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.prometheus.builder; +package org.raystack.firehose.sink.prometheus.builder; import com.google.protobuf.Descriptors; import com.google.protobuf.Duration; @@ -6,13 +6,12 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; import com.google.protobuf.Timestamp; +import org.raystack.firehose.sink.prometheus.PromSinkConstants; import java.util.HashSet; import java.util.Properties; import java.util.Set; -import static io.odpf.firehose.sink.prometheus.PromSinkConstants.*; - public class TimeSeriesBuilderUtils { public static Set getMetricsFromMessage(Message message, Properties metricNameProtoIndexMapping) { @@ -37,7 +36,7 @@ public static Set getMetricsFromMessage(Message message, Prope public static Set getLabelsFromMessage(Message message, Properties labelNameProtoIndexMapping, int partition) { Set labels = new HashSet<>(); - labels.add(new PrometheusLabel(KAFKA_PARTITION, String.valueOf(partition))); + labels.add(new PrometheusLabel(PromSinkConstants.KAFKA_PARTITION, String.valueOf(partition))); labels.addAll(getLabelsFromMessage(message, labelNameProtoIndexMapping)); return labels; } @@ -102,8 +101,8 @@ private static Descriptors.FieldDescriptor getFieldByIndex(Message message, int } private static Long getMillisFromTimestamp(Timestamp timestamp) { - Long millisFromSeconds = timestamp.getSeconds() * SECONDS_SCALED_TO_MILLI; - Long millisFromNanos = timestamp.getNanos() / MILLIS_SCALED_TO_NANOS; + Long millisFromSeconds = timestamp.getSeconds() * PromSinkConstants.SECONDS_SCALED_TO_MILLI; + Long millisFromNanos = timestamp.getNanos() / PromSinkConstants.MILLIS_SCALED_TO_NANOS; return millisFromSeconds + millisFromNanos; } } diff --git a/src/main/java/io/odpf/firehose/sink/prometheus/builder/WriteRequestBuilder.java b/src/main/java/org/raystack/firehose/sink/prometheus/builder/WriteRequestBuilder.java similarity index 93% rename from src/main/java/io/odpf/firehose/sink/prometheus/builder/WriteRequestBuilder.java rename to src/main/java/org/raystack/firehose/sink/prometheus/builder/WriteRequestBuilder.java index 9a723ad8d..940eea824 100644 --- a/src/main/java/io/odpf/firehose/sink/prometheus/builder/WriteRequestBuilder.java +++ b/src/main/java/org/raystack/firehose/sink/prometheus/builder/WriteRequestBuilder.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sink.prometheus.builder; +package org.raystack.firehose.sink.prometheus.builder; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.message.Message; import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; import cortexpb.Cortex; -import io.odpf.stencil.Parser; +import org.raystack.stencil.Parser; import java.util.ArrayList; import java.util.Comparator; diff --git a/src/main/java/io/odpf/firehose/sink/prometheus/request/PromRequest.java b/src/main/java/org/raystack/firehose/sink/prometheus/request/PromRequest.java similarity index 84% rename from src/main/java/io/odpf/firehose/sink/prometheus/request/PromRequest.java rename to src/main/java/org/raystack/firehose/sink/prometheus/request/PromRequest.java index fb13d7851..7886c1d99 100644 --- a/src/main/java/io/odpf/firehose/sink/prometheus/request/PromRequest.java +++ b/src/main/java/org/raystack/firehose/sink/prometheus/request/PromRequest.java @@ -1,12 +1,12 @@ -package io.odpf.firehose.sink.prometheus.request; +package org.raystack.firehose.sink.prometheus.request; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.prometheus.builder.HeaderBuilder; import cortexpb.Cortex; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.prometheus.builder.HeaderBuilder; -import io.odpf.firehose.sink.prometheus.builder.RequestEntityBuilder; -import io.odpf.firehose.sink.prometheus.builder.WriteRequestBuilder; +import org.raystack.firehose.sink.prometheus.builder.RequestEntityBuilder; +import org.raystack.firehose.sink.prometheus.builder.WriteRequestBuilder; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import org.apache.http.client.methods.HttpPost; diff --git a/src/main/java/io/odpf/firehose/sink/prometheus/request/PromRequestCreator.java b/src/main/java/org/raystack/firehose/sink/prometheus/request/PromRequestCreator.java similarity index 74% rename from src/main/java/io/odpf/firehose/sink/prometheus/request/PromRequestCreator.java rename to src/main/java/org/raystack/firehose/sink/prometheus/request/PromRequestCreator.java index 35510fdc8..9e2c26801 100644 --- a/src/main/java/io/odpf/firehose/sink/prometheus/request/PromRequestCreator.java +++ b/src/main/java/org/raystack/firehose/sink/prometheus/request/PromRequestCreator.java @@ -1,14 +1,14 @@ -package io.odpf.firehose.sink.prometheus.request; +package org.raystack.firehose.sink.prometheus.request; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.PromSinkConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.prometheus.builder.HeaderBuilder; -import io.odpf.firehose.sink.prometheus.builder.RequestEntityBuilder; -import io.odpf.firehose.sink.prometheus.builder.TimeSeriesBuilder; -import io.odpf.firehose.sink.prometheus.builder.WriteRequestBuilder; -import io.odpf.stencil.Parser; +import org.raystack.firehose.config.PromSinkConfig; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.prometheus.builder.TimeSeriesBuilder; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.sink.prometheus.builder.HeaderBuilder; +import org.raystack.firehose.sink.prometheus.builder.RequestEntityBuilder; +import org.raystack.firehose.sink.prometheus.builder.WriteRequestBuilder; +import org.raystack.stencil.Parser; /** * Prometheus Request Creator. diff --git a/src/main/java/io/odpf/firehose/sinkdecorator/BackOff.java b/src/main/java/org/raystack/firehose/sinkdecorator/BackOff.java similarity index 83% rename from src/main/java/io/odpf/firehose/sinkdecorator/BackOff.java rename to src/main/java/org/raystack/firehose/sinkdecorator/BackOff.java index 0d157d47d..942dd6b82 100644 --- a/src/main/java/io/odpf/firehose/sinkdecorator/BackOff.java +++ b/src/main/java/org/raystack/firehose/sinkdecorator/BackOff.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sinkdecorator; +package org.raystack.firehose.sinkdecorator; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import lombok.AllArgsConstructor; diff --git a/src/main/java/io/odpf/firehose/sinkdecorator/BackOffProvider.java b/src/main/java/org/raystack/firehose/sinkdecorator/BackOffProvider.java similarity index 89% rename from src/main/java/io/odpf/firehose/sinkdecorator/BackOffProvider.java rename to src/main/java/org/raystack/firehose/sinkdecorator/BackOffProvider.java index 849f670f3..18139ec72 100644 --- a/src/main/java/io/odpf/firehose/sinkdecorator/BackOffProvider.java +++ b/src/main/java/org/raystack/firehose/sinkdecorator/BackOffProvider.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sinkdecorator; +package org.raystack.firehose.sinkdecorator; /** * Interface to provide back-off functionality. diff --git a/src/main/java/io/odpf/firehose/sinkdecorator/ExponentialBackOffProvider.java b/src/main/java/org/raystack/firehose/sinkdecorator/ExponentialBackOffProvider.java similarity index 91% rename from src/main/java/io/odpf/firehose/sinkdecorator/ExponentialBackOffProvider.java rename to src/main/java/org/raystack/firehose/sinkdecorator/ExponentialBackOffProvider.java index 080f958f4..767f96c94 100644 --- a/src/main/java/io/odpf/firehose/sinkdecorator/ExponentialBackOffProvider.java +++ b/src/main/java/org/raystack/firehose/sinkdecorator/ExponentialBackOffProvider.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sinkdecorator; +package org.raystack.firehose.sinkdecorator; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.FirehoseInstrumentation; -import static io.odpf.firehose.metrics.Metrics.RETRY_SLEEP_TIME_MILLISECONDS; +import static org.raystack.firehose.metrics.Metrics.RETRY_SLEEP_TIME_MILLISECONDS; import static java.lang.Math.toIntExact; /** diff --git a/src/main/java/io/odpf/firehose/sinkdecorator/SinkDecorator.java b/src/main/java/org/raystack/firehose/sinkdecorator/SinkDecorator.java similarity index 83% rename from src/main/java/io/odpf/firehose/sinkdecorator/SinkDecorator.java rename to src/main/java/org/raystack/firehose/sinkdecorator/SinkDecorator.java index 41eeb5c47..bbdc69615 100644 --- a/src/main/java/io/odpf/firehose/sinkdecorator/SinkDecorator.java +++ b/src/main/java/org/raystack/firehose/sinkdecorator/SinkDecorator.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sinkdecorator; +package org.raystack.firehose.sinkdecorator; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.sink.Sink; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.sink.Sink; import java.io.IOException; import java.util.List; diff --git a/src/main/java/io/odpf/firehose/sinkdecorator/SinkFinal.java b/src/main/java/org/raystack/firehose/sinkdecorator/SinkFinal.java similarity index 75% rename from src/main/java/io/odpf/firehose/sinkdecorator/SinkFinal.java rename to src/main/java/org/raystack/firehose/sinkdecorator/SinkFinal.java index 65a62f3b2..5beaba351 100644 --- a/src/main/java/io/odpf/firehose/sinkdecorator/SinkFinal.java +++ b/src/main/java/org/raystack/firehose/sinkdecorator/SinkFinal.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.sinkdecorator; +package org.raystack.firehose.sinkdecorator; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.metrics.Metrics; -import io.odpf.firehose.sink.Sink; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.sink.Sink; import java.io.IOException; import java.util.List; diff --git a/src/main/java/io/odpf/firehose/sinkdecorator/SinkWithDlq.java b/src/main/java/org/raystack/firehose/sinkdecorator/SinkWithDlq.java similarity index 86% rename from src/main/java/io/odpf/firehose/sinkdecorator/SinkWithDlq.java rename to src/main/java/org/raystack/firehose/sinkdecorator/SinkWithDlq.java index e7db335ed..c784cec18 100644 --- a/src/main/java/io/odpf/firehose/sinkdecorator/SinkWithDlq.java +++ b/src/main/java/org/raystack/firehose/sinkdecorator/SinkWithDlq.java @@ -1,14 +1,14 @@ -package io.odpf.firehose.sinkdecorator; +package org.raystack.firehose.sinkdecorator; -import io.odpf.firehose.config.DlqConfig; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.error.ErrorHandler; -import io.odpf.firehose.error.ErrorScope; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.metrics.Metrics; -import io.odpf.firehose.sink.Sink; -import io.odpf.firehose.sink.dlq.DlqWriter; +import org.raystack.firehose.config.DlqConfig; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.error.ErrorHandler; +import org.raystack.firehose.error.ErrorScope; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.sink.Sink; +import org.raystack.firehose.sink.dlq.DlqWriter; import java.io.IOException; import java.util.LinkedList; @@ -16,8 +16,8 @@ import java.util.Map; import java.util.Optional; -import static io.odpf.firehose.metrics.Metrics.DLQ_MESSAGES_TOTAL; -import static io.odpf.firehose.metrics.Metrics.DLQ_RETRY_ATTEMPTS_TOTAL; +import static org.raystack.firehose.metrics.Metrics.DLQ_MESSAGES_TOTAL; +import static org.raystack.firehose.metrics.Metrics.DLQ_RETRY_ATTEMPTS_TOTAL; /** * This Sink pushes failed messages to kafka after retries are exhausted. diff --git a/src/main/java/io/odpf/firehose/sinkdecorator/SinkWithFailHandler.java b/src/main/java/org/raystack/firehose/sinkdecorator/SinkWithFailHandler.java similarity index 78% rename from src/main/java/io/odpf/firehose/sinkdecorator/SinkWithFailHandler.java rename to src/main/java/org/raystack/firehose/sinkdecorator/SinkWithFailHandler.java index 40b9a81e0..817894784 100644 --- a/src/main/java/io/odpf/firehose/sinkdecorator/SinkWithFailHandler.java +++ b/src/main/java/org/raystack/firehose/sinkdecorator/SinkWithFailHandler.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sinkdecorator; +package org.raystack.firehose.sinkdecorator; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.error.ErrorHandler; -import io.odpf.firehose.error.ErrorScope; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.exception.SinkException; -import io.odpf.firehose.sink.Sink; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.error.ErrorHandler; +import org.raystack.firehose.error.ErrorScope; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.exception.SinkException; +import org.raystack.firehose.sink.Sink; import java.io.IOException; import java.util.List; diff --git a/src/main/java/io/odpf/firehose/sinkdecorator/SinkWithRetry.java b/src/main/java/org/raystack/firehose/sinkdecorator/SinkWithRetry.java similarity index 88% rename from src/main/java/io/odpf/firehose/sinkdecorator/SinkWithRetry.java rename to src/main/java/org/raystack/firehose/sinkdecorator/SinkWithRetry.java index 77a37aa70..5aca93731 100644 --- a/src/main/java/io/odpf/firehose/sinkdecorator/SinkWithRetry.java +++ b/src/main/java/org/raystack/firehose/sinkdecorator/SinkWithRetry.java @@ -1,15 +1,15 @@ -package io.odpf.firehose.sinkdecorator; +package org.raystack.firehose.sinkdecorator; import com.google.protobuf.DynamicMessage; -import io.odpf.firehose.config.AppConfig; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.error.ErrorHandler; -import io.odpf.firehose.error.ErrorScope; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.metrics.Metrics; -import io.odpf.firehose.sink.Sink; -import io.odpf.firehose.sink.common.KeyOrMessageParser; +import org.raystack.firehose.config.AppConfig; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.error.ErrorHandler; +import org.raystack.firehose.error.ErrorScope; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.sink.Sink; +import org.raystack.firehose.sink.common.KeyOrMessageParser; import java.io.IOException; import java.util.ArrayList; @@ -18,8 +18,8 @@ import java.util.Map; import java.util.stream.Collectors; -import static io.odpf.firehose.metrics.Metrics.RETRY_MESSAGES_TOTAL; -import static io.odpf.firehose.metrics.Metrics.RETRY_ATTEMPTS_TOTAL; +import static org.raystack.firehose.metrics.Metrics.RETRY_MESSAGES_TOTAL; +import static org.raystack.firehose.metrics.Metrics.RETRY_ATTEMPTS_TOTAL; /** * Pushes messages with configured retry. diff --git a/src/main/java/io/odpf/firehose/tracer/SinkTracer.java b/src/main/java/org/raystack/firehose/tracer/SinkTracer.java similarity index 94% rename from src/main/java/io/odpf/firehose/tracer/SinkTracer.java rename to src/main/java/org/raystack/firehose/tracer/SinkTracer.java index 3016af249..8dc3a8408 100644 --- a/src/main/java/io/odpf/firehose/tracer/SinkTracer.java +++ b/src/main/java/org/raystack/firehose/tracer/SinkTracer.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.tracer; +package org.raystack.firehose.tracer; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.message.Message; import io.opentracing.References; import io.opentracing.Span; import io.opentracing.SpanContext; diff --git a/src/main/java/io/odpf/firehose/tracer/Traceable.java b/src/main/java/org/raystack/firehose/tracer/Traceable.java similarity index 85% rename from src/main/java/io/odpf/firehose/tracer/Traceable.java rename to src/main/java/org/raystack/firehose/tracer/Traceable.java index b044fd1d5..042b4abfd 100644 --- a/src/main/java/io/odpf/firehose/tracer/Traceable.java +++ b/src/main/java/org/raystack/firehose/tracer/Traceable.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.tracer; +package org.raystack.firehose.tracer; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.message.Message; import io.opentracing.Span; import java.util.List; diff --git a/src/main/java/io/odpf/firehose/utils/ConsumerRebalancer.java b/src/main/java/org/raystack/firehose/utils/ConsumerRebalancer.java similarity index 91% rename from src/main/java/io/odpf/firehose/utils/ConsumerRebalancer.java rename to src/main/java/org/raystack/firehose/utils/ConsumerRebalancer.java index 44b5998ba..3c239740a 100644 --- a/src/main/java/io/odpf/firehose/utils/ConsumerRebalancer.java +++ b/src/main/java/org/raystack/firehose/utils/ConsumerRebalancer.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.utils; +package org.raystack.firehose.utils; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import lombok.AllArgsConstructor; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; import org.apache.kafka.common.TopicPartition; diff --git a/src/main/java/io/odpf/firehose/utils/KafkaUtils.java b/src/main/java/org/raystack/firehose/utils/KafkaUtils.java similarity index 93% rename from src/main/java/io/odpf/firehose/utils/KafkaUtils.java rename to src/main/java/org/raystack/firehose/utils/KafkaUtils.java index c0043d1bc..b5200980a 100644 --- a/src/main/java/io/odpf/firehose/utils/KafkaUtils.java +++ b/src/main/java/org/raystack/firehose/utils/KafkaUtils.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.utils; +package org.raystack.firehose.utils; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.DlqKafkaProducerConfig; -import io.odpf.firehose.config.KafkaConsumerConfig; -import io.odpf.firehose.consumer.kafka.FirehoseKafkaConsumer; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.parser.KafkaEnvironmentVariables; +import org.raystack.firehose.config.DlqKafkaProducerConfig; +import org.raystack.firehose.config.KafkaConsumerConfig; +import org.raystack.firehose.consumer.kafka.FirehoseKafkaConsumer; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.parser.KafkaEnvironmentVariables; +import org.raystack.depot.metrics.StatsDReporter; import io.opentracing.Tracer; import io.opentracing.contrib.kafka.TracingKafkaConsumer; import org.apache.kafka.clients.consumer.KafkaConsumer; diff --git a/src/main/java/io/odpf/firehose/utils/StencilUtils.java b/src/main/java/org/raystack/firehose/utils/StencilUtils.java similarity index 86% rename from src/main/java/io/odpf/firehose/utils/StencilUtils.java rename to src/main/java/org/raystack/firehose/utils/StencilUtils.java index 9a172bb06..b80ec103f 100644 --- a/src/main/java/io/odpf/firehose/utils/StencilUtils.java +++ b/src/main/java/org/raystack/firehose/utils/StencilUtils.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.utils; +package org.raystack.firehose.utils; +import org.raystack.firehose.config.AppConfig; import com.timgroup.statsd.StatsDClient; -import io.odpf.firehose.config.AppConfig; -import io.odpf.stencil.SchemaUpdateListener; -import io.odpf.stencil.config.StencilConfig; +import org.raystack.stencil.SchemaUpdateListener; +import org.raystack.stencil.config.StencilConfig; public class StencilUtils { public static StencilConfig getStencilConfig( diff --git a/src/test/java/io/odpf/firehose/test/categories/IntegrationTest.java b/src/test/java/io/odpf/firehose/test/categories/IntegrationTest.java deleted file mode 100644 index 2cbc68911..000000000 --- a/src/test/java/io/odpf/firehose/test/categories/IntegrationTest.java +++ /dev/null @@ -1,4 +0,0 @@ -package io.odpf.firehose.test.categories; - -public interface IntegrationTest { -} diff --git a/src/test/java/io/odpf/firehose/config/GCSConfigTest.java b/src/test/java/org/raystack/firehose/config/GCSConfigTest.java similarity index 97% rename from src/test/java/io/odpf/firehose/config/GCSConfigTest.java rename to src/test/java/org/raystack/firehose/config/GCSConfigTest.java index d3cc0cd62..6d1ff6c01 100644 --- a/src/test/java/io/odpf/firehose/config/GCSConfigTest.java +++ b/src/test/java/org/raystack/firehose/config/GCSConfigTest.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; import org.aeonbits.owner.ConfigFactory; import org.junit.Assert; diff --git a/src/test/java/io/odpf/firehose/config/HttpSinkDataFormatTypeConverterTest.java b/src/test/java/org/raystack/firehose/config/HttpSinkDataFormatTypeConverterTest.java similarity index 90% rename from src/test/java/io/odpf/firehose/config/HttpSinkDataFormatTypeConverterTest.java rename to src/test/java/org/raystack/firehose/config/HttpSinkDataFormatTypeConverterTest.java index 50fdda264..94de5b730 100644 --- a/src/test/java/io/odpf/firehose/config/HttpSinkDataFormatTypeConverterTest.java +++ b/src/test/java/org/raystack/firehose/config/HttpSinkDataFormatTypeConverterTest.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; -import io.odpf.firehose.config.enums.HttpSinkDataFormatType; -import io.odpf.firehose.config.converter.HttpSinkDataFormatTypeConverter; +import org.raystack.firehose.config.converter.HttpSinkDataFormatTypeConverter; +import org.raystack.firehose.config.enums.HttpSinkDataFormatType; import org.gradle.internal.impldep.org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/config/ProtoIndexToFieldMapConverterTest.java b/src/test/java/org/raystack/firehose/config/ProtoIndexToFieldMapConverterTest.java similarity index 97% rename from src/test/java/io/odpf/firehose/config/ProtoIndexToFieldMapConverterTest.java rename to src/test/java/org/raystack/firehose/config/ProtoIndexToFieldMapConverterTest.java index 9a5262626..192d6c55f 100644 --- a/src/test/java/io/odpf/firehose/config/ProtoIndexToFieldMapConverterTest.java +++ b/src/test/java/org/raystack/firehose/config/ProtoIndexToFieldMapConverterTest.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; -import io.odpf.firehose.config.converter.ProtoIndexToFieldMapConverter; +import org.raystack.firehose.config.converter.ProtoIndexToFieldMapConverter; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/src/test/java/io/odpf/firehose/config/RangeToHashMapConverterTest.java b/src/test/java/org/raystack/firehose/config/RangeToHashMapConverterTest.java similarity index 86% rename from src/test/java/io/odpf/firehose/config/RangeToHashMapConverterTest.java rename to src/test/java/org/raystack/firehose/config/RangeToHashMapConverterTest.java index f4e451033..01de49f52 100644 --- a/src/test/java/io/odpf/firehose/config/RangeToHashMapConverterTest.java +++ b/src/test/java/org/raystack/firehose/config/RangeToHashMapConverterTest.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; -import io.odpf.firehose.config.converter.RangeToHashMapConverter; +import org.raystack.firehose.config.converter.RangeToHashMapConverter; import org.junit.Test; import java.util.Map; diff --git a/src/test/java/io/odpf/firehose/config/S3ConfigTest.java b/src/test/java/org/raystack/firehose/config/S3ConfigTest.java similarity index 98% rename from src/test/java/io/odpf/firehose/config/S3ConfigTest.java rename to src/test/java/org/raystack/firehose/config/S3ConfigTest.java index 701d4ea7b..6a8fdfa36 100644 --- a/src/test/java/io/odpf/firehose/config/S3ConfigTest.java +++ b/src/test/java/org/raystack/firehose/config/S3ConfigTest.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.config; +package org.raystack.firehose.config; import org.aeonbits.owner.ConfigFactory; import org.junit.Assert; diff --git a/src/test/java/io/odpf/firehose/config/converter/InputSchemaTypeConverterTest.java b/src/test/java/org/raystack/firehose/config/converter/InputSchemaTypeConverterTest.java similarity index 87% rename from src/test/java/io/odpf/firehose/config/converter/InputSchemaTypeConverterTest.java rename to src/test/java/org/raystack/firehose/config/converter/InputSchemaTypeConverterTest.java index 2eb31b5e1..3bd72e356 100644 --- a/src/test/java/io/odpf/firehose/config/converter/InputSchemaTypeConverterTest.java +++ b/src/test/java/org/raystack/firehose/config/converter/InputSchemaTypeConverterTest.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; -import io.odpf.firehose.config.enums.InputSchemaType; +import org.raystack.firehose.config.enums.InputSchemaType; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/config/converter/SchemaRegistryHeadersConverterTest.java b/src/test/java/org/raystack/firehose/config/converter/SchemaRegistryHeadersConverterTest.java similarity index 94% rename from src/test/java/io/odpf/firehose/config/converter/SchemaRegistryHeadersConverterTest.java rename to src/test/java/org/raystack/firehose/config/converter/SchemaRegistryHeadersConverterTest.java index 748d01252..691d4299c 100644 --- a/src/test/java/io/odpf/firehose/config/converter/SchemaRegistryHeadersConverterTest.java +++ b/src/test/java/org/raystack/firehose/config/converter/SchemaRegistryHeadersConverterTest.java @@ -1,15 +1,14 @@ -package io.odpf.firehose.config.converter; +package org.raystack.firehose.config.converter; import java.util.HashMap; import java.util.Map; +import org.raystack.firehose.config.AppConfig; import org.aeonbits.owner.ConfigFactory; import org.apache.http.message.BasicHeader; import org.junit.Assert; import org.junit.Test; -import io.odpf.firehose.config.AppConfig; - public class SchemaRegistryHeadersConverterTest { @Test public void testConvertIfFetchHeadersValueEmpty() { diff --git a/src/test/java/io/odpf/firehose/consumer/ConsumerAndOffsetManagerTest.java b/src/test/java/org/raystack/firehose/consumer/ConsumerAndOffsetManagerTest.java similarity index 94% rename from src/test/java/io/odpf/firehose/consumer/ConsumerAndOffsetManagerTest.java rename to src/test/java/org/raystack/firehose/consumer/ConsumerAndOffsetManagerTest.java index 2092d0ac5..fe1382441 100644 --- a/src/test/java/io/odpf/firehose/consumer/ConsumerAndOffsetManagerTest.java +++ b/src/test/java/org/raystack/firehose/consumer/ConsumerAndOffsetManagerTest.java @@ -1,12 +1,12 @@ -package io.odpf.firehose.consumer; +package org.raystack.firehose.consumer; -import io.odpf.firehose.config.KafkaConsumerConfig; -import io.odpf.firehose.consumer.kafka.ConsumerAndOffsetManager; -import io.odpf.firehose.consumer.kafka.FirehoseKafkaConsumer; -import io.odpf.firehose.consumer.kafka.OffsetManager; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.Sink; +import org.raystack.firehose.config.KafkaConsumerConfig; +import org.raystack.firehose.consumer.kafka.ConsumerAndOffsetManager; +import org.raystack.firehose.consumer.kafka.FirehoseKafkaConsumer; +import org.raystack.firehose.consumer.kafka.OffsetManager; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.Sink; import org.aeonbits.owner.ConfigFactory; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; diff --git a/src/test/java/io/odpf/firehose/consumer/FirehoseAsyncConsumerTest.java b/src/test/java/org/raystack/firehose/consumer/FirehoseAsyncConsumerTest.java similarity index 91% rename from src/test/java/io/odpf/firehose/consumer/FirehoseAsyncConsumerTest.java rename to src/test/java/org/raystack/firehose/consumer/FirehoseAsyncConsumerTest.java index c2e00e998..bbe6b164e 100644 --- a/src/test/java/io/odpf/firehose/consumer/FirehoseAsyncConsumerTest.java +++ b/src/test/java/org/raystack/firehose/consumer/FirehoseAsyncConsumerTest.java @@ -1,15 +1,15 @@ -package io.odpf.firehose.consumer; - -import io.odpf.firehose.consumer.kafka.ConsumerAndOffsetManager; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.SinkPool; -import io.odpf.firehose.exception.SinkTaskFailedException; -import io.odpf.firehose.filter.FilterException; -import io.odpf.firehose.filter.FilteredMessages; -import io.odpf.firehose.filter.NoOpFilter; -import io.odpf.firehose.metrics.Metrics; -import io.odpf.firehose.tracer.SinkTracer; +package org.raystack.firehose.consumer; + +import org.raystack.firehose.consumer.kafka.ConsumerAndOffsetManager; +import org.raystack.firehose.exception.SinkTaskFailedException; +import org.raystack.firehose.filter.FilterException; +import org.raystack.firehose.filter.FilteredMessages; +import org.raystack.firehose.filter.NoOpFilter; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.sink.SinkPool; +import org.raystack.firehose.tracer.SinkTracer; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/consumer/FirehoseFilterTest.java b/src/test/java/org/raystack/firehose/consumer/FirehoseFilterTest.java similarity index 93% rename from src/test/java/io/odpf/firehose/consumer/FirehoseFilterTest.java rename to src/test/java/org/raystack/firehose/consumer/FirehoseFilterTest.java index fb5a43120..d96072272 100644 --- a/src/test/java/io/odpf/firehose/consumer/FirehoseFilterTest.java +++ b/src/test/java/org/raystack/firehose/consumer/FirehoseFilterTest.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.consumer; +package org.raystack.firehose.consumer; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.filter.Filter; -import io.odpf.firehose.filter.FilterException; -import io.odpf.firehose.filter.FilteredMessages; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.filter.Filter; +import org.raystack.firehose.filter.FilterException; +import org.raystack.firehose.filter.FilteredMessages; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; diff --git a/src/test/java/io/odpf/firehose/consumer/FirehoseSyncConsumerTest.java b/src/test/java/org/raystack/firehose/consumer/FirehoseSyncConsumerTest.java similarity index 90% rename from src/test/java/io/odpf/firehose/consumer/FirehoseSyncConsumerTest.java rename to src/test/java/org/raystack/firehose/consumer/FirehoseSyncConsumerTest.java index 58dc136b0..b653b9aa3 100644 --- a/src/test/java/io/odpf/firehose/consumer/FirehoseSyncConsumerTest.java +++ b/src/test/java/org/raystack/firehose/consumer/FirehoseSyncConsumerTest.java @@ -1,16 +1,16 @@ -package io.odpf.firehose.consumer; +package org.raystack.firehose.consumer; -import io.odpf.firehose.config.KafkaConsumerConfig; -import io.odpf.firehose.consumer.kafka.ConsumerAndOffsetManager; -import io.odpf.firehose.consumer.kafka.FirehoseKafkaConsumer; -import io.odpf.firehose.consumer.kafka.OffsetManager; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.filter.FilteredMessages; -import io.odpf.firehose.filter.NoOpFilter; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.metrics.Metrics; -import io.odpf.firehose.sink.Sink; -import io.odpf.firehose.tracer.SinkTracer; +import org.raystack.firehose.config.KafkaConsumerConfig; +import org.raystack.firehose.consumer.kafka.ConsumerAndOffsetManager; +import org.raystack.firehose.consumer.kafka.FirehoseKafkaConsumer; +import org.raystack.firehose.consumer.kafka.OffsetManager; +import org.raystack.firehose.filter.FilteredMessages; +import org.raystack.firehose.filter.NoOpFilter; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.sink.Sink; +import org.raystack.firehose.tracer.SinkTracer; import org.aeonbits.owner.ConfigFactory; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/consumer/kafka/FirehoseKafkaConsumerTest.java b/src/test/java/org/raystack/firehose/consumer/kafka/FirehoseKafkaConsumerTest.java similarity index 97% rename from src/test/java/io/odpf/firehose/consumer/kafka/FirehoseKafkaConsumerTest.java rename to src/test/java/org/raystack/firehose/consumer/kafka/FirehoseKafkaConsumerTest.java index d432abdc6..187404dc8 100644 --- a/src/test/java/io/odpf/firehose/consumer/kafka/FirehoseKafkaConsumerTest.java +++ b/src/test/java/org/raystack/firehose/consumer/kafka/FirehoseKafkaConsumerTest.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.consumer.kafka; - -import io.odpf.firehose.config.KafkaConsumerConfig; -import io.odpf.firehose.consumer.TestKey; -import io.odpf.firehose.consumer.TestMessage; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.metrics.Metrics; +package org.raystack.firehose.consumer.kafka; + +import org.raystack.firehose.config.KafkaConsumerConfig; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.consumer.TestKey; +import org.raystack.firehose.consumer.TestMessage; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.OffsetAndMetadata; diff --git a/src/test/java/io/odpf/firehose/consumer/kafka/OffsetManagerTest.java b/src/test/java/org/raystack/firehose/consumer/kafka/OffsetManagerTest.java similarity index 99% rename from src/test/java/io/odpf/firehose/consumer/kafka/OffsetManagerTest.java rename to src/test/java/org/raystack/firehose/consumer/kafka/OffsetManagerTest.java index fd2ef48f3..22055cb09 100644 --- a/src/test/java/io/odpf/firehose/consumer/kafka/OffsetManagerTest.java +++ b/src/test/java/org/raystack/firehose/consumer/kafka/OffsetManagerTest.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.consumer.kafka; +package org.raystack.firehose.consumer.kafka; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.message.Message; import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/src/test/java/io/odpf/firehose/converter/ProtoTimeConverterTest.java b/src/test/java/org/raystack/firehose/converter/ProtoTimeConverterTest.java similarity index 94% rename from src/test/java/io/odpf/firehose/converter/ProtoTimeConverterTest.java rename to src/test/java/org/raystack/firehose/converter/ProtoTimeConverterTest.java index b5c99eac8..cabe1642c 100644 --- a/src/test/java/io/odpf/firehose/converter/ProtoTimeConverterTest.java +++ b/src/test/java/org/raystack/firehose/converter/ProtoTimeConverterTest.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.converter; +package org.raystack.firehose.converter; import com.google.protobuf.Timestamp; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/error/ErrorHandlerTest.java b/src/test/java/org/raystack/firehose/error/ErrorHandlerTest.java similarity index 95% rename from src/test/java/io/odpf/firehose/error/ErrorHandlerTest.java rename to src/test/java/org/raystack/firehose/error/ErrorHandlerTest.java index 8cdb1ceb6..ab3c760ab 100644 --- a/src/test/java/io/odpf/firehose/error/ErrorHandlerTest.java +++ b/src/test/java/org/raystack/firehose/error/ErrorHandlerTest.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.error; +package org.raystack.firehose.error; -import io.odpf.depot.error.ErrorInfo; -import io.odpf.depot.error.ErrorType; -import io.odpf.firehose.config.ErrorConfig; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.config.ErrorConfig; +import org.raystack.firehose.message.Message; +import org.raystack.depot.error.ErrorInfo; +import org.raystack.depot.error.ErrorType; import org.aeonbits.owner.ConfigFactory; import org.junit.Assert; import org.junit.Before; diff --git a/src/test/java/io/odpf/firehose/filter/NoOpFilterTest.java b/src/test/java/org/raystack/firehose/filter/NoOpFilterTest.java similarity index 92% rename from src/test/java/io/odpf/firehose/filter/NoOpFilterTest.java rename to src/test/java/org/raystack/firehose/filter/NoOpFilterTest.java index 885595dca..44df278a6 100644 --- a/src/test/java/io/odpf/firehose/filter/NoOpFilterTest.java +++ b/src/test/java/org/raystack/firehose/filter/NoOpFilterTest.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.filter; +package org.raystack.firehose.filter; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestKey; -import io.odpf.firehose.consumer.TestMessage; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.consumer.TestKey; +import org.raystack.firehose.consumer.TestMessage; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; diff --git a/src/test/java/io/odpf/firehose/filter/jexl/JexlFilterTest.java b/src/test/java/org/raystack/firehose/filter/jexl/JexlFilterTest.java similarity index 88% rename from src/test/java/io/odpf/firehose/filter/jexl/JexlFilterTest.java rename to src/test/java/org/raystack/firehose/filter/jexl/JexlFilterTest.java index 2818d56c2..d7f9da3c8 100644 --- a/src/test/java/io/odpf/firehose/filter/jexl/JexlFilterTest.java +++ b/src/test/java/org/raystack/firehose/filter/jexl/JexlFilterTest.java @@ -1,16 +1,16 @@ -package io.odpf.firehose.filter.jexl; +package org.raystack.firehose.filter.jexl; -import io.odpf.firehose.config.FilterConfig; -import io.odpf.firehose.config.enums.FilterDataSourceType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestBookingLogKey; -import io.odpf.firehose.consumer.TestBookingLogMessage; -import io.odpf.firehose.consumer.TestKey; -import io.odpf.firehose.consumer.TestMessage; -import io.odpf.firehose.filter.Filter; -import io.odpf.firehose.filter.FilterException; -import io.odpf.firehose.filter.FilteredMessages; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.config.FilterConfig; +import org.raystack.firehose.config.enums.FilterDataSourceType; +import org.raystack.firehose.filter.Filter; +import org.raystack.firehose.filter.FilterException; +import org.raystack.firehose.filter.FilteredMessages; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.consumer.TestBookingLogKey; +import org.raystack.firehose.consumer.TestBookingLogMessage; +import org.raystack.firehose.consumer.TestKey; +import org.raystack.firehose.consumer.TestMessage; import org.aeonbits.owner.ConfigFactory; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/filter/json/JsonFilterTest.java b/src/test/java/org/raystack/firehose/filter/json/JsonFilterTest.java similarity index 95% rename from src/test/java/io/odpf/firehose/filter/json/JsonFilterTest.java rename to src/test/java/org/raystack/firehose/filter/json/JsonFilterTest.java index 9346369de..ed2d48eca 100644 --- a/src/test/java/io/odpf/firehose/filter/json/JsonFilterTest.java +++ b/src/test/java/org/raystack/firehose/filter/json/JsonFilterTest.java @@ -1,17 +1,17 @@ -package io.odpf.firehose.filter.json; +package org.raystack.firehose.filter.json; -import io.odpf.firehose.config.FilterConfig; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestBookingLogKey; -import io.odpf.firehose.consumer.TestBookingLogMessage; -import io.odpf.firehose.consumer.TestKey; -import io.odpf.firehose.consumer.TestLocation; -import io.odpf.firehose.consumer.TestMessage; -import io.odpf.firehose.filter.FilterException; -import io.odpf.firehose.filter.FilteredMessages; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.stencil.StencilClientFactory; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.config.FilterConfig; +import org.raystack.firehose.filter.FilterException; +import org.raystack.firehose.filter.FilteredMessages; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.consumer.TestBookingLogKey; +import org.raystack.firehose.consumer.TestBookingLogMessage; +import org.raystack.firehose.consumer.TestKey; +import org.raystack.firehose.consumer.TestLocation; +import org.raystack.firehose.consumer.TestMessage; +import org.raystack.stencil.StencilClientFactory; +import org.raystack.stencil.client.StencilClient; import org.aeonbits.owner.ConfigFactory; import org.junit.Before; import org.junit.Rule; diff --git a/src/test/java/io/odpf/firehose/filter/json/JsonFilterUtilTest.java b/src/test/java/org/raystack/firehose/filter/json/JsonFilterUtilTest.java similarity index 93% rename from src/test/java/io/odpf/firehose/filter/json/JsonFilterUtilTest.java rename to src/test/java/org/raystack/firehose/filter/json/JsonFilterUtilTest.java index b5900de6e..936dab817 100644 --- a/src/test/java/io/odpf/firehose/filter/json/JsonFilterUtilTest.java +++ b/src/test/java/org/raystack/firehose/filter/json/JsonFilterUtilTest.java @@ -1,9 +1,10 @@ -package io.odpf.firehose.filter.json; +package org.raystack.firehose.filter.json; -import io.odpf.firehose.config.FilterConfig; -import io.odpf.firehose.config.enums.FilterDataSourceType; -import io.odpf.firehose.consumer.TestMessage; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.config.FilterConfig; +import org.raystack.firehose.config.enums.FilterDataSourceType; +import org.raystack.firehose.config.enums.FilterMessageFormatType; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.consumer.TestMessage; import org.aeonbits.owner.ConfigFactory; import org.junit.Before; import org.junit.Rule; @@ -15,7 +16,6 @@ import java.util.HashMap; import java.util.Map; -import static io.odpf.firehose.config.enums.FilterMessageFormatType.PROTOBUF; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -46,7 +46,7 @@ public void shouldLogFilterConfigsForValidConfiguration() { verify(firehoseInstrumentation, times(1)).logInfo("\n\tFilter data source type: {}", FilterDataSourceType.MESSAGE); verify(firehoseInstrumentation, times(1)).logInfo("\n\tMessage Proto class: {}", TestMessage.class.getName()); verify(firehoseInstrumentation, times(1)).logInfo("\n\tFilter JSON Schema: {}", "{\"properties\":{\"order_number\":{\"const\":\"123\"}}}"); - verify(firehoseInstrumentation, times(1)).logInfo("\n\tFilter ESB message format: {}", PROTOBUF); + verify(firehoseInstrumentation, times(1)).logInfo("\n\tFilter ESB message format: {}", FilterMessageFormatType.PROTOBUF); } @Test diff --git a/src/test/java/io/odpf/firehose/launch/TaskTest.java b/src/test/java/org/raystack/firehose/launch/TaskTest.java similarity index 95% rename from src/test/java/io/odpf/firehose/launch/TaskTest.java rename to src/test/java/org/raystack/firehose/launch/TaskTest.java index c20887788..e3dbbc64c 100644 --- a/src/test/java/io/odpf/firehose/launch/TaskTest.java +++ b/src/test/java/org/raystack/firehose/launch/TaskTest.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.launch; +package org.raystack.firehose.launch; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/io/odpf/firehose/message/FirehoseMessageUtilsTest.java b/src/test/java/org/raystack/firehose/message/FirehoseMessageUtilsTest.java similarity index 75% rename from src/test/java/io/odpf/firehose/message/FirehoseMessageUtilsTest.java rename to src/test/java/org/raystack/firehose/message/FirehoseMessageUtilsTest.java index ddad28121..198806ee3 100644 --- a/src/test/java/io/odpf/firehose/message/FirehoseMessageUtilsTest.java +++ b/src/test/java/org/raystack/firehose/message/FirehoseMessageUtilsTest.java @@ -1,8 +1,7 @@ -package io.odpf.firehose.message; +package org.raystack.firehose.message; -import io.odpf.depot.common.Tuple; -import io.odpf.depot.message.OdpfMessage; +import org.raystack.depot.common.Tuple; import org.gradle.internal.impldep.org.junit.Assert; import org.junit.Test; @@ -14,7 +13,7 @@ public class FirehoseMessageUtilsTest { @Test - public void shouldConvertToOdpfMessage() { + public void shouldConvertToDepotMessage() { Message m1 = new Message( "testKey".getBytes(), "testMessage".getBytes(), @@ -29,12 +28,12 @@ public void shouldConvertToOdpfMessage() { 11, 2 ); - List actualMessages = FirehoseMessageUtils.convertToOdpfMessage(new ArrayList() {{ + List actualMessages = FirehoseMessageUtils.convertToDepotMessage(new ArrayList() {{ add(m1); add(m2); }}); Assert.assertEquals(2, actualMessages.size()); - OdpfMessage expectedMessage1 = new OdpfMessage( + org.raystack.depot.message.Message expectedMessage1 = new org.raystack.depot.message.Message( "testKey".getBytes(), "testMessage".getBytes(), new Tuple<>("message_topic", "topic"), @@ -43,7 +42,7 @@ public void shouldConvertToOdpfMessage() { new Tuple<>("message_headers", null), new Tuple<>("message_timestamp", 0L), new Tuple<>("load_time", 0L)); - OdpfMessage expectedMessage2 = new OdpfMessage( + org.raystack.depot.message.Message expectedMessage2 = new org.raystack.depot.message.Message( "testKey1".getBytes(), "testMessage1".getBytes(), new Tuple<>("message_topic", "topic1"), @@ -53,8 +52,8 @@ public void shouldConvertToOdpfMessage() { new Tuple<>("message_timestamp", 0L), new Tuple<>("load_time", 0L)); - OdpfMessage actualMessage1 = actualMessages.get(0); - OdpfMessage actualMessage2 = actualMessages.get(1); + org.raystack.depot.message.Message actualMessage1 = actualMessages.get(0); + org.raystack.depot.message.Message actualMessage2 = actualMessages.get(1); Assert.assertTrue(Arrays.equals((byte[]) expectedMessage1.getLogKey(), (byte[]) actualMessage1.getLogKey())); Assert.assertTrue(Arrays.equals((byte[]) expectedMessage1.getLogMessage(), (byte[]) actualMessage1.getLogMessage())); @@ -67,7 +66,7 @@ public void shouldConvertToOdpfMessage() { @Test public void shouldReturnEmptyList() { - List actualMessages = FirehoseMessageUtils.convertToOdpfMessage(new ArrayList<>()); + List actualMessages = FirehoseMessageUtils.convertToDepotMessage(new ArrayList<>()); Assert.assertEquals(Collections.emptyList(), actualMessages); } } diff --git a/src/test/java/io/odpf/firehose/message/MessageTest.java b/src/test/java/org/raystack/firehose/message/MessageTest.java similarity index 88% rename from src/test/java/io/odpf/firehose/message/MessageTest.java rename to src/test/java/org/raystack/firehose/message/MessageTest.java index 71744f48b..1d7349dcc 100644 --- a/src/test/java/io/odpf/firehose/message/MessageTest.java +++ b/src/test/java/org/raystack/firehose/message/MessageTest.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.message; +package org.raystack.firehose.message; -import io.odpf.depot.error.ErrorType; -import io.odpf.firehose.consumer.TestKey; -import io.odpf.firehose.consumer.TestMessage; -import io.odpf.firehose.exception.DefaultException; +import org.raystack.firehose.exception.DefaultException; +import org.raystack.depot.error.ErrorType; +import org.raystack.firehose.consumer.TestKey; +import org.raystack.firehose.consumer.TestMessage; import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/metrics/FirehoseInstrumentationTest.java b/src/test/java/org/raystack/firehose/metrics/FirehoseInstrumentationTest.java similarity index 94% rename from src/test/java/io/odpf/firehose/metrics/FirehoseInstrumentationTest.java rename to src/test/java/org/raystack/firehose/metrics/FirehoseInstrumentationTest.java index 0c898f517..41eaf93cc 100644 --- a/src/test/java/io/odpf/firehose/metrics/FirehoseInstrumentationTest.java +++ b/src/test/java/org/raystack/firehose/metrics/FirehoseInstrumentationTest.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.metrics; +package org.raystack.firehose.metrics; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.message.Message; +import org.raystack.depot.metrics.StatsDReporter; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -16,8 +16,6 @@ import java.util.List; -import static io.odpf.firehose.metrics.Metrics.GLOBAL_MESSAGES_TOTAL; -import static io.odpf.firehose.metrics.Metrics.MESSAGE_SCOPE_TAG; import static org.mockito.Mockito.*; @RunWith(MockitoJUnitRunner.class) @@ -138,7 +136,7 @@ public void shouldCaptureLifetimeTillSink() { @Test public void shouldCaptureGlobalMetrics() { firehoseInstrumentation.captureGlobalMessageMetrics(Metrics.MessageScope.CONSUMER, 1); - verify(statsDReporter, times(1)).captureCount(GLOBAL_MESSAGES_TOTAL, 1L, String.format(MESSAGE_SCOPE_TAG, Metrics.MessageScope.CONSUMER)); + verify(statsDReporter, times(1)).captureCount(Metrics.GLOBAL_MESSAGES_TOTAL, 1L, String.format(Metrics.MESSAGE_SCOPE_TAG, Metrics.MessageScope.CONSUMER)); } @Test diff --git a/src/test/java/io/odpf/firehose/parser/KafkaEnvironmentVariablesTest.java b/src/test/java/org/raystack/firehose/parser/KafkaEnvironmentVariablesTest.java similarity index 98% rename from src/test/java/io/odpf/firehose/parser/KafkaEnvironmentVariablesTest.java rename to src/test/java/org/raystack/firehose/parser/KafkaEnvironmentVariablesTest.java index 076a6d73a..9ae5590e0 100644 --- a/src/test/java/io/odpf/firehose/parser/KafkaEnvironmentVariablesTest.java +++ b/src/test/java/org/raystack/firehose/parser/KafkaEnvironmentVariablesTest.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.parser; +package org.raystack.firehose.parser; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/proto/ProtoMessageTest.java b/src/test/java/org/raystack/firehose/proto/ProtoMessageTest.java similarity index 89% rename from src/test/java/io/odpf/firehose/proto/ProtoMessageTest.java rename to src/test/java/org/raystack/firehose/proto/ProtoMessageTest.java index 6005919ec..621e9a2e2 100644 --- a/src/test/java/io/odpf/firehose/proto/ProtoMessageTest.java +++ b/src/test/java/org/raystack/firehose/proto/ProtoMessageTest.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.proto; +package org.raystack.firehose.proto; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestBookingLogKey; -import io.odpf.firehose.consumer.TestFeedbackLogKey; -import io.odpf.firehose.consumer.TestFeedbackLogMessage; -import io.odpf.firehose.exception.DeserializerException; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.consumer.TestBookingLogKey; +import org.raystack.firehose.consumer.TestFeedbackLogKey; +import org.raystack.firehose.consumer.TestFeedbackLogMessage; import com.google.protobuf.Timestamp; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/proto/ProtoUtilTest.java b/src/test/java/org/raystack/firehose/proto/ProtoUtilTest.java similarity index 95% rename from src/test/java/io/odpf/firehose/proto/ProtoUtilTest.java rename to src/test/java/org/raystack/firehose/proto/ProtoUtilTest.java index 6e95a69b0..da353dcf8 100644 --- a/src/test/java/io/odpf/firehose/proto/ProtoUtilTest.java +++ b/src/test/java/org/raystack/firehose/proto/ProtoUtilTest.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.proto; +package org.raystack.firehose.proto; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import com.google.protobuf.UnknownFieldSet; -import io.odpf.firehose.consumer.TestBookingLogMessage; -import io.odpf.firehose.consumer.TestLocation; +import org.raystack.firehose.consumer.TestBookingLogMessage; +import org.raystack.firehose.consumer.TestLocation; import org.junit.Test; import static org.junit.Assert.assertFalse; diff --git a/src/test/java/io/odpf/firehose/serializer/JsonWrappedProtoByteTest.java b/src/test/java/org/raystack/firehose/serializer/JsonWrappedProtoByteTest.java similarity index 85% rename from src/test/java/io/odpf/firehose/serializer/JsonWrappedProtoByteTest.java rename to src/test/java/org/raystack/firehose/serializer/JsonWrappedProtoByteTest.java index 1efcafe4b..1be3369fc 100644 --- a/src/test/java/io/odpf/firehose/serializer/JsonWrappedProtoByteTest.java +++ b/src/test/java/org/raystack/firehose/serializer/JsonWrappedProtoByteTest.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.serializer; +package org.raystack.firehose.serializer; import static org.junit.Assert.assertEquals; import java.util.Base64; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/serializer/MessageToJsonTest.java b/src/test/java/org/raystack/firehose/serializer/MessageToJsonTest.java similarity index 94% rename from src/test/java/io/odpf/firehose/serializer/MessageToJsonTest.java rename to src/test/java/org/raystack/firehose/serializer/MessageToJsonTest.java index c14689ca6..552d8f743 100644 --- a/src/test/java/io/odpf/firehose/serializer/MessageToJsonTest.java +++ b/src/test/java/org/raystack/firehose/serializer/MessageToJsonTest.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.serializer; - -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestAggregatedSupplyMessage; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.stencil.StencilClientFactory; -import io.odpf.stencil.client.StencilClient; -import io.odpf.stencil.Parser; +package org.raystack.firehose.serializer; + +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.consumer.TestAggregatedSupplyMessage; +import org.raystack.stencil.StencilClientFactory; +import org.raystack.stencil.client.StencilClient; +import org.raystack.stencil.Parser; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/serializer/MessageToTemplatizedJsonTest.java b/src/test/java/org/raystack/firehose/serializer/MessageToTemplatizedJsonTest.java similarity index 91% rename from src/test/java/io/odpf/firehose/serializer/MessageToTemplatizedJsonTest.java rename to src/test/java/org/raystack/firehose/serializer/MessageToTemplatizedJsonTest.java index d70b984ac..8618f2601 100644 --- a/src/test/java/io/odpf/firehose/serializer/MessageToTemplatizedJsonTest.java +++ b/src/test/java/org/raystack/firehose/serializer/MessageToTemplatizedJsonTest.java @@ -1,16 +1,16 @@ -package io.odpf.firehose.serializer; +package org.raystack.firehose.serializer; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestAggregatedSupplyMessage; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.exception.ConfigurationException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.stencil.StencilClientFactory; -import io.odpf.stencil.client.StencilClient; -import io.odpf.stencil.Parser; +import org.raystack.firehose.exception.ConfigurationException; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.consumer.TestAggregatedSupplyMessage; +import org.raystack.stencil.StencilClientFactory; +import org.raystack.stencil.client.StencilClient; +import org.raystack.stencil.Parser; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; diff --git a/src/test/java/io/odpf/firehose/sink/AbstractSinkTest.java b/src/test/java/org/raystack/firehose/sink/AbstractSinkTest.java similarity index 96% rename from src/test/java/io/odpf/firehose/sink/AbstractSinkTest.java rename to src/test/java/org/raystack/firehose/sink/AbstractSinkTest.java index 69bddaf94..6a517d6e6 100644 --- a/src/test/java/io/odpf/firehose/sink/AbstractSinkTest.java +++ b/src/test/java/org/raystack/firehose/sink/AbstractSinkTest.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sink; - -import io.odpf.depot.error.ErrorInfo; -import io.odpf.depot.error.ErrorType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.metrics.Metrics; +package org.raystack.firehose.sink; + +import org.raystack.depot.error.ErrorInfo; +import org.raystack.depot.error.ErrorType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; import org.junit.Assert; import org.junit.Test; import org.mockito.Mock; diff --git a/src/test/java/io/odpf/firehose/sink/GenericOdpfSinkTest.java b/src/test/java/org/raystack/firehose/sink/GenericSinkTest.java similarity index 78% rename from src/test/java/io/odpf/firehose/sink/GenericOdpfSinkTest.java rename to src/test/java/org/raystack/firehose/sink/GenericSinkTest.java index 1203a9130..fbac27789 100644 --- a/src/test/java/io/odpf/firehose/sink/GenericOdpfSinkTest.java +++ b/src/test/java/org/raystack/firehose/sink/GenericSinkTest.java @@ -1,11 +1,10 @@ -package io.odpf.firehose.sink; +package org.raystack.firehose.sink; -import io.odpf.depot.OdpfSink; -import io.odpf.depot.OdpfSinkResponse; -import io.odpf.depot.error.ErrorInfo; -import io.odpf.depot.error.ErrorType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.depot.SinkResponse; +import org.raystack.depot.error.ErrorInfo; +import org.raystack.depot.error.ErrorType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -15,21 +14,21 @@ import java.util.Collections; import java.util.List; -public class GenericOdpfSinkTest { +public class GenericSinkTest { private FirehoseInstrumentation instrumentation; - private OdpfSink odpfSink; + private org.raystack.depot.Sink depotSink; @Before public void setup() { instrumentation = Mockito.mock(FirehoseInstrumentation.class); - odpfSink = Mockito.mock(OdpfSink.class); + depotSink = Mockito.mock(org.raystack.depot.Sink.class); } @Test public void shouldReturnEmptyListOfMessagesWhenSuccess() throws Exception { - GenericOdpfSink sink = new GenericOdpfSink(instrumentation, "test", odpfSink); - Mockito.when(odpfSink.pushToSink(Mockito.anyList())).thenReturn(new OdpfSinkResponse()); + GenericSink sink = new GenericSink(instrumentation, "test", this.depotSink); + Mockito.when(this.depotSink.pushToSink(Mockito.anyList())).thenReturn(new SinkResponse()); List messages = new ArrayList() {{ Message m1 = new Message(new byte[1], new byte[1], "test", 1, 1); Message m2 = new Message(new byte[1], new byte[1], "test", 1, 2); @@ -51,12 +50,12 @@ public void shouldReturnEmptyListOfMessagesWhenSuccess() throws Exception { @Test public void shouldReturnFailedMessages() throws Exception { - GenericOdpfSink sink = new GenericOdpfSink(instrumentation, "test", odpfSink); - OdpfSinkResponse response = new OdpfSinkResponse(); + GenericSink sink = new GenericSink(instrumentation, "test", this.depotSink); + SinkResponse response = new SinkResponse(); response.addErrors(5, new ErrorInfo(new Exception(), ErrorType.SINK_4XX_ERROR)); response.addErrors(2, new ErrorInfo(new Exception(), ErrorType.DEFAULT_ERROR)); response.addErrors(4, new ErrorInfo(new Exception(), ErrorType.DESERIALIZATION_ERROR)); - Mockito.when(odpfSink.pushToSink(Mockito.anyList())).thenReturn(response); + Mockito.when(this.depotSink.pushToSink(Mockito.anyList())).thenReturn(response); Message m1 = new Message(new byte[1], new byte[1], "test", 1, 1); Message m2 = new Message(new byte[1], new byte[1], "test", 1, 2); Message m3 = new Message(new byte[1], new byte[1], "test", 1, 3); diff --git a/src/test/java/io/odpf/firehose/sink/SinkFactoryUtilsTest.java b/src/test/java/org/raystack/firehose/sink/SinkFactoryUtilsTest.java similarity index 96% rename from src/test/java/io/odpf/firehose/sink/SinkFactoryUtilsTest.java rename to src/test/java/org/raystack/firehose/sink/SinkFactoryUtilsTest.java index d573bd992..b356e2232 100644 --- a/src/test/java/io/odpf/firehose/sink/SinkFactoryUtilsTest.java +++ b/src/test/java/org/raystack/firehose/sink/SinkFactoryUtilsTest.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink; +package org.raystack.firehose.sink; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/SinkPoolTest.java b/src/test/java/org/raystack/firehose/sink/SinkPoolTest.java similarity index 95% rename from src/test/java/io/odpf/firehose/sink/SinkPoolTest.java rename to src/test/java/org/raystack/firehose/sink/SinkPoolTest.java index f57704d7a..f7c70a769 100644 --- a/src/test/java/io/odpf/firehose/sink/SinkPoolTest.java +++ b/src/test/java/org/raystack/firehose/sink/SinkPoolTest.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink; +package org.raystack.firehose.sink; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.message.Message; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -37,7 +37,7 @@ public class SinkPoolTest { @Before public void setup() { MockitoAnnotations.initMocks(this); - sinkPool = new SinkPool(workerSinks, executorService, 5); + sinkPool = new SinkPool(workerSinks, new ArrayList<>(), executorService, 5); } @Test diff --git a/src/test/java/io/odpf/firehose/sink/bigquery/BigquerySinkUtilsTest.java b/src/test/java/org/raystack/firehose/sink/bigquery/BigquerySinkUtilsTest.java similarity index 95% rename from src/test/java/io/odpf/firehose/sink/bigquery/BigquerySinkUtilsTest.java rename to src/test/java/org/raystack/firehose/sink/bigquery/BigquerySinkUtilsTest.java index 186af6888..304363ef3 100644 --- a/src/test/java/io/odpf/firehose/sink/bigquery/BigquerySinkUtilsTest.java +++ b/src/test/java/org/raystack/firehose/sink/bigquery/BigquerySinkUtilsTest.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.bigquery; +package org.raystack.firehose.sink.bigquery; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/blob/BlobSinkTest.java b/src/test/java/org/raystack/firehose/sink/blob/BlobSinkTest.java similarity index 92% rename from src/test/java/io/odpf/firehose/sink/blob/BlobSinkTest.java rename to src/test/java/org/raystack/firehose/sink/blob/BlobSinkTest.java index a72f76f47..40d46cd76 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/BlobSinkTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/BlobSinkTest.java @@ -1,18 +1,18 @@ -package io.odpf.firehose.sink.blob; +package org.raystack.firehose.sink.blob; import com.google.protobuf.DynamicMessage; -import io.odpf.depot.error.ErrorType; -import io.odpf.firehose.TestMessageBQ; -import io.odpf.firehose.consumer.kafka.OffsetManager; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.exception.EmptyMessageException; -import io.odpf.firehose.exception.UnknownFieldsException; -import io.odpf.firehose.exception.SinkException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.blob.message.MessageDeSerializer; -import io.odpf.firehose.sink.blob.message.Record; -import io.odpf.firehose.sink.blob.writer.WriterOrchestrator; +import org.raystack.depot.error.ErrorType; +import org.raystack.firehose.TestMessageBQ; +import org.raystack.firehose.consumer.kafka.OffsetManager; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.exception.EmptyMessageException; +import org.raystack.firehose.exception.UnknownFieldsException; +import org.raystack.firehose.exception.SinkException; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.blob.message.MessageDeSerializer; +import org.raystack.firehose.sink.blob.message.Record; +import org.raystack.firehose.sink.blob.writer.WriterOrchestrator; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.junit.Before; diff --git a/src/test/java/io/odpf/firehose/sink/blob/TestProtoMessage.java b/src/test/java/org/raystack/firehose/sink/blob/TestProtoMessage.java similarity index 92% rename from src/test/java/io/odpf/firehose/sink/blob/TestProtoMessage.java rename to src/test/java/org/raystack/firehose/sink/blob/TestProtoMessage.java index 7f5f47615..c8e074561 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/TestProtoMessage.java +++ b/src/test/java/org/raystack/firehose/sink/blob/TestProtoMessage.java @@ -1,17 +1,17 @@ -package io.odpf.firehose.sink.blob; +package org.raystack.firehose.sink.blob; import com.github.os72.protobuf.dynamic.DynamicSchema; import com.github.os72.protobuf.dynamic.MessageDefinition; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import com.google.protobuf.Timestamp; -import io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils; -import io.odpf.firehose.sink.blob.proto.TimestampMetadataProtoMessage; +import org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils; +import org.raystack.firehose.sink.blob.proto.TimestampMetadataProtoMessage; import java.time.Instant; public class TestProtoMessage { - private static final String PACKAGE = "io.odpf"; + private static final String PACKAGE = "org.raystack"; private static final String FILE_NAME = "booking.proto"; private static final String TYPE_NAME = "BookingLogMessage"; diff --git a/src/test/java/io/odpf/firehose/sink/blob/TestUtils.java b/src/test/java/org/raystack/firehose/sink/blob/TestUtils.java similarity index 81% rename from src/test/java/io/odpf/firehose/sink/blob/TestUtils.java rename to src/test/java/org/raystack/firehose/sink/blob/TestUtils.java index 472690e24..0c94f540a 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/TestUtils.java +++ b/src/test/java/org/raystack/firehose/sink/blob/TestUtils.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.sink.blob; +package org.raystack.firehose.sink.blob; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.sink.blob.message.KafkaMetadataUtils; -import io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.sink.blob.message.KafkaMetadataUtils; +import org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils; import java.time.Instant; diff --git a/src/test/java/io/odpf/firehose/sink/blob/message/KafkaMetadataUtilsTest.java b/src/test/java/org/raystack/firehose/sink/blob/message/KafkaMetadataUtilsTest.java similarity index 94% rename from src/test/java/io/odpf/firehose/sink/blob/message/KafkaMetadataUtilsTest.java rename to src/test/java/org/raystack/firehose/sink/blob/message/KafkaMetadataUtilsTest.java index f2e8b1545..e84c086d7 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/message/KafkaMetadataUtilsTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/message/KafkaMetadataUtilsTest.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sink.blob.message; +package org.raystack.firehose.sink.blob.message; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import com.google.protobuf.Timestamp; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessage; -import io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessage; +import org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils; import org.junit.Test; import java.time.Instant; diff --git a/src/test/java/io/odpf/firehose/sink/blob/message/MessageDeSerializerTest.java b/src/test/java/org/raystack/firehose/sink/blob/message/MessageDeSerializerTest.java similarity index 92% rename from src/test/java/io/odpf/firehose/sink/blob/message/MessageDeSerializerTest.java rename to src/test/java/org/raystack/firehose/sink/blob/message/MessageDeSerializerTest.java index b42d99b81..7a712ba84 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/message/MessageDeSerializerTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/message/MessageDeSerializerTest.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.blob.message; +package org.raystack.firehose.sink.blob.message; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; @@ -6,13 +6,13 @@ import com.google.protobuf.StringValue; import com.google.protobuf.UnknownFieldSet; -import io.odpf.firehose.config.BlobSinkConfig; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessage; -import io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils; -import io.odpf.firehose.sink.blob.proto.NestedKafkaMetadataProtoMessage; -import io.odpf.stencil.Parser; +import org.raystack.firehose.config.BlobSinkConfig; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessage; +import org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils; +import org.raystack.firehose.sink.blob.proto.NestedKafkaMetadataProtoMessage; +import org.raystack.stencil.Parser; import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/blob/message/RecordTest.java b/src/test/java/org/raystack/firehose/sink/blob/message/RecordTest.java similarity index 94% rename from src/test/java/io/odpf/firehose/sink/blob/message/RecordTest.java rename to src/test/java/org/raystack/firehose/sink/blob/message/RecordTest.java index d78bc9f60..a3f4685ca 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/message/RecordTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/message/RecordTest.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.sink.blob.message; +package org.raystack.firehose.sink.blob.message; import com.google.protobuf.DynamicMessage; -import io.odpf.firehose.sink.blob.TestUtils; +import org.raystack.firehose.sink.blob.TestUtils; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/blob/proto/KafkaMetadataProtoFirehoseMessageUtilsTest.java b/src/test/java/org/raystack/firehose/sink/blob/proto/KafkaMetadataProtoFirehoseMessageUtilsTest.java similarity index 95% rename from src/test/java/io/odpf/firehose/sink/blob/proto/KafkaMetadataProtoFirehoseMessageUtilsTest.java rename to src/test/java/org/raystack/firehose/sink/blob/proto/KafkaMetadataProtoFirehoseMessageUtilsTest.java index 249f7f22a..e863a883b 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/proto/KafkaMetadataProtoFirehoseMessageUtilsTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/proto/KafkaMetadataProtoFirehoseMessageUtilsTest.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.blob.proto; +package org.raystack.firehose.sink.blob.proto; import com.github.os72.protobuf.dynamic.DynamicSchema; import com.google.protobuf.Descriptors; -import io.odpf.firehose.sink.blob.TestProtoMessage; +import org.raystack.firehose.sink.blob.TestProtoMessage; import org.junit.Test; import static org.junit.Assert.*; diff --git a/src/test/java/io/odpf/firehose/sink/blob/proto/KafkaMetadataProtoMessageTest.java b/src/test/java/org/raystack/firehose/sink/blob/proto/KafkaMetadataProtoMessageTest.java similarity index 93% rename from src/test/java/io/odpf/firehose/sink/blob/proto/KafkaMetadataProtoMessageTest.java rename to src/test/java/org/raystack/firehose/sink/blob/proto/KafkaMetadataProtoMessageTest.java index 4fd485ba3..78db067b0 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/proto/KafkaMetadataProtoMessageTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/proto/KafkaMetadataProtoMessageTest.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.blob.proto; +package org.raystack.firehose.sink.blob.proto; import com.github.os72.protobuf.dynamic.DynamicSchema; import com.github.os72.protobuf.dynamic.MessageDefinition; @@ -9,8 +9,8 @@ import java.time.Instant; -import static io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils.FILE_NAME; -import static io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils.PACKAGE; +import static org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils.FILE_NAME; +import static org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils.PACKAGE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; diff --git a/src/test/java/io/odpf/firehose/sink/blob/proto/NestedKafkaMetadataProtoMessageTest.java b/src/test/java/org/raystack/firehose/sink/blob/proto/NestedKafkaMetadataProtoMessageTest.java similarity index 94% rename from src/test/java/io/odpf/firehose/sink/blob/proto/NestedKafkaMetadataProtoMessageTest.java rename to src/test/java/org/raystack/firehose/sink/blob/proto/NestedKafkaMetadataProtoMessageTest.java index e27503a0d..cb0202cdb 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/proto/NestedKafkaMetadataProtoMessageTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/proto/NestedKafkaMetadataProtoMessageTest.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.blob.proto; +package org.raystack.firehose.sink.blob.proto; import com.github.os72.protobuf.dynamic.DynamicSchema; import com.github.os72.protobuf.dynamic.MessageDefinition; @@ -8,8 +8,8 @@ import java.time.Instant; -import static io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils.FILE_NAME; -import static io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils.PACKAGE; +import static org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils.FILE_NAME; +import static org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils.PACKAGE; import static org.junit.Assert.*; public class NestedKafkaMetadataProtoMessageTest { diff --git a/src/test/java/io/odpf/firehose/sink/blob/proto/TimestampMetadataProtoMessageTest.java b/src/test/java/org/raystack/firehose/sink/blob/proto/TimestampMetadataProtoMessageTest.java similarity index 84% rename from src/test/java/io/odpf/firehose/sink/blob/proto/TimestampMetadataProtoMessageTest.java rename to src/test/java/org/raystack/firehose/sink/blob/proto/TimestampMetadataProtoMessageTest.java index 8f77eeb3e..864cabbaf 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/proto/TimestampMetadataProtoMessageTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/proto/TimestampMetadataProtoMessageTest.java @@ -1,12 +1,12 @@ -package io.odpf.firehose.sink.blob.proto; +package org.raystack.firehose.sink.blob.proto; import com.github.os72.protobuf.dynamic.DynamicSchema; import com.google.protobuf.Descriptors; import com.google.protobuf.Timestamp; import org.junit.Test; -import static io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils.FILE_NAME; -import static io.odpf.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils.PACKAGE; +import static org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils.FILE_NAME; +import static org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils.PACKAGE; import static org.junit.Assert.*; public class TimestampMetadataProtoMessageTest { diff --git a/src/test/java/io/odpf/firehose/sink/blob/writer/WriterOrchestratorStatusTest.java b/src/test/java/org/raystack/firehose/sink/blob/writer/WriterOrchestratorStatusTest.java similarity index 98% rename from src/test/java/io/odpf/firehose/sink/blob/writer/WriterOrchestratorStatusTest.java rename to src/test/java/org/raystack/firehose/sink/blob/writer/WriterOrchestratorStatusTest.java index 74137612b..170c1ad5e 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/writer/WriterOrchestratorStatusTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/writer/WriterOrchestratorStatusTest.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.blob.writer; +package org.raystack.firehose.sink.blob.writer; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/blob/writer/WriterOrchestratorTest.java b/src/test/java/org/raystack/firehose/sink/blob/writer/WriterOrchestratorTest.java similarity index 86% rename from src/test/java/io/odpf/firehose/sink/blob/writer/WriterOrchestratorTest.java rename to src/test/java/org/raystack/firehose/sink/blob/writer/WriterOrchestratorTest.java index e8e8c9cb6..d1e06821f 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/writer/WriterOrchestratorTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/writer/WriterOrchestratorTest.java @@ -1,13 +1,16 @@ -package io.odpf.firehose.sink.blob.writer; +package org.raystack.firehose.sink.blob.writer; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.BlobSinkConfig; -import io.odpf.firehose.sink.common.blobstorage.BlobStorage; -import io.odpf.firehose.sink.blob.Constants; -import io.odpf.firehose.sink.blob.TestProtoMessage; -import io.odpf.firehose.sink.blob.message.Record; -import io.odpf.firehose.sink.blob.writer.local.*; -import io.odpf.firehose.sink.blob.writer.local.path.TimePartitionedPathUtils; +import org.raystack.firehose.config.BlobSinkConfig; +import org.raystack.firehose.sink.blob.Constants; +import org.raystack.firehose.sink.blob.TestProtoMessage; +import org.raystack.firehose.sink.blob.message.Record; +import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata; +import org.raystack.firehose.sink.blob.writer.local.LocalFileWriter; +import org.raystack.firehose.sink.blob.writer.local.LocalFileWriterFailedException; +import org.raystack.firehose.sink.blob.writer.local.LocalStorage; +import org.raystack.firehose.sink.blob.writer.local.path.TimePartitionedPathUtils; +import org.raystack.firehose.sink.common.blobstorage.BlobStorage; +import org.raystack.depot.metrics.StatsDReporter; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -67,7 +70,7 @@ public void shouldCreateLocalFileWriter() throws Exception { Record record = Mockito.mock(Record.class); Mockito.when(record.getTimestamp(timeStampFieldName)).thenReturn(Instant.ofEpochMilli(1L)); Mockito.when(record.getTopic("")).thenReturn(defaultTopic); - Mockito.when(localFileWriter1.getMetadata()).thenReturn(new LocalFileMetadata("/tmp/", "/tmp/test", 0, 0, 0)); + Mockito.when(localFileWriter1.getFullPath()).thenReturn("/tmp/test"); Mockito.when(localStorage.createLocalFileWriter(TimePartitionedPathUtils.getTimePartitionedPath(record, sinkConfig))).thenReturn(localFileWriter1); Mockito.when(localFileWriter1.write(record)).thenReturn(true); try (WriterOrchestrator writerOrchestrator = new WriterOrchestrator(sinkConfig, localStorage, blobStorage, statsDReporter)) { @@ -83,14 +86,14 @@ public void shouldCreateMultipleWriterBasedOnPartition() throws Exception { Mockito.when(record1.getTopic("")).thenReturn(defaultTopic); Mockito.when(localStorage.createLocalFileWriter(TimePartitionedPathUtils.getTimePartitionedPath(record1, sinkConfig))).thenReturn(localFileWriter1); Mockito.when(localFileWriter1.write(record1)).thenReturn(true); - Mockito.when(localFileWriter1.getMetadata()).thenReturn(new LocalFileMetadata("/tmp/", "/tmp/test1", 0, 0, 0)); + Mockito.when(localFileWriter1.getFullPath()).thenReturn("/tmp/test1"); Record record2 = Mockito.mock(Record.class); Mockito.when(record2.getTimestamp(timeStampFieldName)).thenReturn(Instant.ofEpochMilli(7200000L)); Mockito.when(record2.getTopic("")).thenReturn(defaultTopic); Mockito.when(localStorage.createLocalFileWriter(TimePartitionedPathUtils.getTimePartitionedPath(record2, sinkConfig))).thenReturn(localFileWriter2); Mockito.when(localFileWriter2.write(record2)).thenReturn(true); - Mockito.when(localFileWriter2.getMetadata()).thenReturn(new LocalFileMetadata("/tmp/", "/tmp/test2", 0, 0, 0)); + Mockito.when(localFileWriter2.getFullPath()).thenReturn("/tmp/test2"); try (WriterOrchestrator writerOrchestrator = new WriterOrchestrator(sinkConfig, localStorage, blobStorage, statsDReporter)) { Set paths = new HashSet<>(); @@ -131,7 +134,7 @@ public void shouldGetEmptyFlushedPath() throws Exception { Record record = Mockito.mock(Record.class); Mockito.when(record.getTimestamp(timeStampFieldName)).thenReturn(Instant.ofEpochMilli(1L)); Mockito.when(record.getTopic("")).thenReturn(defaultTopic); - Mockito.when(localFileWriter1.getMetadata()).thenReturn(new LocalFileMetadata("/tmp/", "/tmp/test", 0, 0, 0)); + Mockito.when(localFileWriter1.getFullPath()).thenReturn("/tmp/test"); Mockito.when(localStorage.createLocalFileWriter(TimePartitionedPathUtils.getTimePartitionedPath(record, sinkConfig))).thenReturn(localFileWriter1); Mockito.when(localFileWriter1.write(record)).thenReturn(true); try (WriterOrchestrator writerOrchestrator = new WriterOrchestrator(sinkConfig, localStorage, blobStorage, statsDReporter)) { diff --git a/src/test/java/io/odpf/firehose/sink/blob/writer/local/LocalFileCheckerTest.java b/src/test/java/org/raystack/firehose/sink/blob/writer/local/LocalFileCheckerTest.java similarity index 92% rename from src/test/java/io/odpf/firehose/sink/blob/writer/local/LocalFileCheckerTest.java rename to src/test/java/org/raystack/firehose/sink/blob/writer/local/LocalFileCheckerTest.java index f58a632a8..f99608422 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/writer/local/LocalFileCheckerTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/writer/local/LocalFileCheckerTest.java @@ -1,6 +1,8 @@ -package io.odpf.firehose.sink.blob.writer.local; +package org.raystack.firehose.sink.blob.writer.local; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.BlobStorageMetrics; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -17,8 +19,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; -import static io.odpf.firehose.metrics.Metrics.*; -import static io.odpf.firehose.metrics.BlobStorageMetrics.*; import static org.mockito.Mockito.*; import static org.mockito.MockitoAnnotations.initMocks; @@ -147,7 +147,7 @@ public void shouldRecordMetricOfSuccessfullyClosedFiles() throws IOException { when(writer1.getMetadata()).thenReturn(new LocalFileMetadata("/tmp", "/tmp/a/random-file-name-1", 1L, recordCount, fileSize)); when(writer1.closeAndFetchMetaData()).thenReturn(new LocalFileMetadata("/tmp", "/tmp/a/random-file-name-1", 1L, recordCount, fileSize)); worker.run(); - verify(firehoseInstrumentation).incrementCounter(LOCAL_FILE_CLOSE_TOTAL, SUCCESS_TAG); + verify(firehoseInstrumentation).incrementCounter(BlobStorageMetrics.LOCAL_FILE_CLOSE_TOTAL, Metrics.SUCCESS_TAG); } @Test @@ -159,7 +159,7 @@ public void shouldRecordMetricOfClosingTimeDuration() throws IOException { when(writer1.closeAndFetchMetaData()).thenReturn(new LocalFileMetadata("/tmp", "/tmp/a/random-file-name-1", 1L, recordCount, fileSize)); worker.run(); - verify(firehoseInstrumentation, times(1)).captureDurationSince(eq(LOCAL_FILE_CLOSING_TIME_MILLISECONDS), any(Instant.class)); + verify(firehoseInstrumentation, times(1)).captureDurationSince(eq(BlobStorageMetrics.LOCAL_FILE_CLOSING_TIME_MILLISECONDS), any(Instant.class)); } @Test @@ -171,7 +171,7 @@ public void shouldRecordMetricOfFileSizeInBytes() throws IOException { when(writer1.closeAndFetchMetaData()).thenReturn(new LocalFileMetadata("/tmp", "/tmp/a/random-file-name-1", 1L, recordCount, fileSize)); worker.run(); - verify(firehoseInstrumentation, times(1)).captureCount(LOCAL_FILE_SIZE_BYTES, fileSize); + verify(firehoseInstrumentation, times(1)).captureCount(BlobStorageMetrics.LOCAL_FILE_SIZE_BYTES, fileSize); } @Test @@ -187,7 +187,7 @@ public void shouldRecordMetricOfFailedClosedFiles() throws IOException { } catch (LocalFileWriterFailedException ignored) { } - verify(firehoseInstrumentation, times(1)).incrementCounter(LOCAL_FILE_CLOSE_TOTAL, FAILURE_TAG); + verify(firehoseInstrumentation, times(1)).incrementCounter(BlobStorageMetrics.LOCAL_FILE_CLOSE_TOTAL, Metrics.FAILURE_TAG); } @Test @@ -211,7 +211,7 @@ public void shouldCaptureValueOfFileOpenCount() throws IOException { verify(writer2, times(1)).closeAndFetchMetaData(); Assert.assertEquals(2, toBeFlushedToRemotePaths.size()); Assert.assertEquals(0, writerMap.size()); - verify(firehoseInstrumentation, times(3)).captureValue(LOCAL_FILE_OPEN_TOTAL, 2); - verify(firehoseInstrumentation, times(3)).captureValue(LOCAL_FILE_OPEN_TOTAL, 0); + verify(firehoseInstrumentation, times(3)).captureValue(BlobStorageMetrics.LOCAL_FILE_OPEN_TOTAL, 2); + verify(firehoseInstrumentation, times(3)).captureValue(BlobStorageMetrics.LOCAL_FILE_OPEN_TOTAL, 0); } } diff --git a/src/test/java/io/odpf/firehose/sink/blob/writer/local/LocalStorageTest.java b/src/test/java/org/raystack/firehose/sink/blob/writer/local/LocalStorageTest.java similarity index 82% rename from src/test/java/io/odpf/firehose/sink/blob/writer/local/LocalStorageTest.java rename to src/test/java/org/raystack/firehose/sink/blob/writer/local/LocalStorageTest.java index 3e4b06ff9..fb70cf181 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/writer/local/LocalStorageTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/writer/local/LocalStorageTest.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sink.blob.writer.local; +package org.raystack.firehose.sink.blob.writer.local; import com.google.protobuf.Descriptors; -import io.odpf.firehose.config.BlobSinkConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.blob.Constants; -import io.odpf.firehose.sink.blob.writer.local.policy.WriterPolicy; +import org.raystack.firehose.config.BlobSinkConfig; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.blob.Constants; +import org.raystack.firehose.sink.blob.writer.local.policy.WriterPolicy; import org.junit.Test; import org.mockito.Mockito; diff --git a/src/test/java/io/odpf/firehose/sink/blob/writer/local/TimePartitionedPathUtilsTest.java b/src/test/java/org/raystack/firehose/sink/blob/writer/local/TimePartitionedPathUtilsTest.java similarity index 94% rename from src/test/java/io/odpf/firehose/sink/blob/writer/local/TimePartitionedPathUtilsTest.java rename to src/test/java/org/raystack/firehose/sink/blob/writer/local/TimePartitionedPathUtilsTest.java index 8d6d3fa86..3d5907b19 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/writer/local/TimePartitionedPathUtilsTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/writer/local/TimePartitionedPathUtilsTest.java @@ -1,12 +1,12 @@ -package io.odpf.firehose.sink.blob.writer.local; +package org.raystack.firehose.sink.blob.writer.local; import com.google.protobuf.DynamicMessage; -import io.odpf.firehose.config.BlobSinkConfig; -import io.odpf.firehose.sink.blob.Constants; -import io.odpf.firehose.sink.blob.TestUtils; -import io.odpf.firehose.sink.blob.TestProtoMessage; -import io.odpf.firehose.sink.blob.message.Record; -import io.odpf.firehose.sink.blob.writer.local.path.TimePartitionedPathUtils; +import org.raystack.firehose.config.BlobSinkConfig; +import org.raystack.firehose.sink.blob.Constants; +import org.raystack.firehose.sink.blob.TestProtoMessage; +import org.raystack.firehose.sink.blob.TestUtils; +import org.raystack.firehose.sink.blob.message.Record; +import org.raystack.firehose.sink.blob.writer.local.path.TimePartitionedPathUtils; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; diff --git a/src/test/java/io/odpf/firehose/sink/blob/writer/local/policy/SizeBasedRotatingPolicyTest.java b/src/test/java/org/raystack/firehose/sink/blob/writer/local/policy/SizeBasedRotatingPolicyTest.java similarity index 91% rename from src/test/java/io/odpf/firehose/sink/blob/writer/local/policy/SizeBasedRotatingPolicyTest.java rename to src/test/java/org/raystack/firehose/sink/blob/writer/local/policy/SizeBasedRotatingPolicyTest.java index f5011a916..d9de20e4c 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/writer/local/policy/SizeBasedRotatingPolicyTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/writer/local/policy/SizeBasedRotatingPolicyTest.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink.blob.writer.local.policy; +package org.raystack.firehose.sink.blob.writer.local.policy; -import io.odpf.firehose.sink.blob.writer.local.LocalFileMetadata; +import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/blob/writer/local/policy/TimeBasedRotatingPolicyTest.java b/src/test/java/org/raystack/firehose/sink/blob/writer/local/policy/TimeBasedRotatingPolicyTest.java similarity index 92% rename from src/test/java/io/odpf/firehose/sink/blob/writer/local/policy/TimeBasedRotatingPolicyTest.java rename to src/test/java/org/raystack/firehose/sink/blob/writer/local/policy/TimeBasedRotatingPolicyTest.java index b6c0133d4..b48ad75c0 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/writer/local/policy/TimeBasedRotatingPolicyTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/writer/local/policy/TimeBasedRotatingPolicyTest.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink.blob.writer.local.policy; +package org.raystack.firehose.sink.blob.writer.local.policy; -import io.odpf.firehose.sink.blob.writer.local.LocalFileMetadata; +import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageCheckerTest.java b/src/test/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageCheckerTest.java similarity index 88% rename from src/test/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageCheckerTest.java rename to src/test/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageCheckerTest.java index 71a204fe4..213985b7b 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageCheckerTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageCheckerTest.java @@ -1,10 +1,12 @@ -package io.odpf.firehose.sink.blob.writer.remote; - -import io.odpf.firehose.config.BlobSinkConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.common.blobstorage.BlobStorage; -import io.odpf.firehose.sink.blob.Constants; -import io.odpf.firehose.sink.blob.writer.local.LocalFileMetadata; +package org.raystack.firehose.sink.blob.writer.remote; + +import org.raystack.firehose.config.BlobSinkConfig; +import org.raystack.firehose.metrics.BlobStorageMetrics; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.sink.blob.Constants; +import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata; +import org.raystack.firehose.sink.common.blobstorage.BlobStorage; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -21,8 +23,6 @@ import java.util.Set; import java.util.concurrent.*; -import static io.odpf.firehose.metrics.Metrics.*; -import static io.odpf.firehose.metrics.BlobStorageMetrics.*; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.*; @@ -113,7 +113,7 @@ public void shouldRecordMetricOfFileUploadedCount() throws ExecutionException, I when(remoteUploadScheduler.submit(any(Callable.class))).thenReturn(f); worker.run(); - verify(firehoseInstrumentation, times(1)).incrementCounter(FILE_UPLOAD_TOTAL, SUCCESS_TAG); + verify(firehoseInstrumentation, times(1)).incrementCounter(BlobStorageMetrics.FILE_UPLOAD_TOTAL, Metrics.SUCCESS_TAG); } @Test @@ -124,7 +124,7 @@ public void shouldRecordMetricOfFileUploadBytes() throws ExecutionException, Int when(f.get()).thenReturn(10L); when(remoteUploadScheduler.submit(any(Callable.class))).thenReturn(f); worker.run(); - verify(firehoseInstrumentation).captureCount(FILE_UPLOAD_BYTES, localFileMetadata.getSize()); + verify(firehoseInstrumentation).captureCount(BlobStorageMetrics.FILE_UPLOAD_BYTES, localFileMetadata.getSize()); } @Test @@ -137,7 +137,7 @@ public void shouldRecordMetricOfUploadDuration() throws ExecutionException, Inte when(f.get()).thenReturn(totalTime); worker.run(); - verify(firehoseInstrumentation, (times(1))).captureDuration(FILE_UPLOAD_TIME_MILLISECONDS, totalTime); + verify(firehoseInstrumentation, (times(1))).captureDuration(BlobStorageMetrics.FILE_UPLOAD_TIME_MILLISECONDS, totalTime); } @Test @@ -155,9 +155,9 @@ public void shouldRecordMetricOfUploadFailedCountWhenUploadFutureThrowsInterrupt } catch (RuntimeException ignored) { } - verify(firehoseInstrumentation, times(1)).incrementCounter(FILE_UPLOAD_TOTAL, - FAILURE_TAG, - tag(BLOB_STORAGE_ERROR_TYPE_TAG, "")); + verify(firehoseInstrumentation, times(1)).incrementCounter(BlobStorageMetrics.FILE_UPLOAD_TOTAL, + Metrics.FAILURE_TAG, + Metrics.tag(BlobStorageMetrics.BLOB_STORAGE_ERROR_TYPE_TAG, "")); } @Test(expected = BlobStorageFailedException.class) diff --git a/src/test/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageWriterFutureHandlerTest.java b/src/test/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageWriterFutureHandlerTest.java similarity index 91% rename from src/test/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageWriterFutureHandlerTest.java rename to src/test/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageWriterFutureHandlerTest.java index fb10648e5..4cff0975a 100644 --- a/src/test/java/io/odpf/firehose/sink/blob/writer/remote/BlobStorageWriterFutureHandlerTest.java +++ b/src/test/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageWriterFutureHandlerTest.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.sink.blob.writer.remote; +package org.raystack.firehose.sink.blob.writer.remote; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.metrics.Metrics; -import io.odpf.firehose.metrics.BlobStorageMetrics; -import io.odpf.firehose.sink.blob.writer.local.LocalFileMetadata; +import org.raystack.firehose.metrics.BlobStorageMetrics; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata; import org.junit.Assert; import org.junit.Test; import org.junit.jupiter.api.Assertions; diff --git a/src/test/java/io/odpf/firehose/sink/common/KeyOrMessageParserTest.java b/src/test/java/org/raystack/firehose/sink/common/KeyOrMessageParserTest.java similarity index 87% rename from src/test/java/io/odpf/firehose/sink/common/KeyOrMessageParserTest.java rename to src/test/java/org/raystack/firehose/sink/common/KeyOrMessageParserTest.java index 556ca6e00..2d7315f48 100644 --- a/src/test/java/io/odpf/firehose/sink/common/KeyOrMessageParserTest.java +++ b/src/test/java/org/raystack/firehose/sink/common/KeyOrMessageParserTest.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sink.common; +package org.raystack.firehose.sink.common; -import io.odpf.firehose.config.AppConfig; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestMessage; +import org.raystack.firehose.config.AppConfig; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.consumer.TestMessage; import com.google.protobuf.DynamicMessage; -import io.odpf.stencil.Parser; +import org.raystack.stencil.Parser; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/io/odpf/firehose/sink/common/gcs/GoogleCloudStorageTest.java b/src/test/java/org/raystack/firehose/sink/common/gcs/GoogleCloudStorageTest.java similarity index 90% rename from src/test/java/io/odpf/firehose/sink/common/gcs/GoogleCloudStorageTest.java rename to src/test/java/org/raystack/firehose/sink/common/gcs/GoogleCloudStorageTest.java index edd9e1d80..70b8b32bb 100644 --- a/src/test/java/io/odpf/firehose/sink/common/gcs/GoogleCloudStorageTest.java +++ b/src/test/java/org/raystack/firehose/sink/common/gcs/GoogleCloudStorageTest.java @@ -1,12 +1,12 @@ -package io.odpf.firehose.sink.common.gcs; +package org.raystack.firehose.sink.common.gcs; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageException; -import io.odpf.firehose.config.GCSConfig; -import io.odpf.firehose.sink.common.blobstorage.BlobStorageException; -import io.odpf.firehose.sink.common.blobstorage.gcs.GoogleCloudStorage; +import org.raystack.firehose.config.GCSConfig; +import org.raystack.firehose.sink.common.blobstorage.BlobStorageException; +import org.raystack.firehose.sink.common.blobstorage.gcs.GoogleCloudStorage; import org.aeonbits.owner.ConfigFactory; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/common/s3/S3Test.java b/src/test/java/org/raystack/firehose/sink/common/s3/S3Test.java similarity index 94% rename from src/test/java/io/odpf/firehose/sink/common/s3/S3Test.java rename to src/test/java/org/raystack/firehose/sink/common/s3/S3Test.java index 64f3c74da..08e0f110d 100644 --- a/src/test/java/io/odpf/firehose/sink/common/s3/S3Test.java +++ b/src/test/java/org/raystack/firehose/sink/common/s3/S3Test.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.common.s3; +package org.raystack.firehose.sink.common.s3; -import io.odpf.firehose.config.S3Config; -import io.odpf.firehose.sink.common.blobstorage.BlobStorageException; -import io.odpf.firehose.sink.common.blobstorage.s3.S3; +import org.raystack.firehose.config.S3Config; +import org.raystack.firehose.sink.common.blobstorage.BlobStorageException; +import org.raystack.firehose.sink.common.blobstorage.s3.S3; import org.aeonbits.owner.ConfigFactory; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/dlq/KafkaDlqWriterTest.java b/src/test/java/org/raystack/firehose/sink/dlq/KafkaDlqWriterTest.java similarity index 95% rename from src/test/java/io/odpf/firehose/sink/dlq/KafkaDlqWriterTest.java rename to src/test/java/org/raystack/firehose/sink/dlq/KafkaDlqWriterTest.java index 0ac910182..294a3a74e 100644 --- a/src/test/java/io/odpf/firehose/sink/dlq/KafkaDlqWriterTest.java +++ b/src/test/java/org/raystack/firehose/sink/dlq/KafkaDlqWriterTest.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sink.dlq; - -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestKey; -import io.odpf.firehose.consumer.TestMessage; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.dlq.kafka.KafkaDlqWriter; +package org.raystack.firehose.sink.dlq; + +import org.raystack.firehose.message.Message; +import org.raystack.firehose.consumer.TestKey; +import org.raystack.firehose.consumer.TestMessage; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.dlq.kafka.KafkaDlqWriter; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; diff --git a/src/test/java/io/odpf/firehose/sink/dlq/LogDlqWriterTest.java b/src/test/java/org/raystack/firehose/sink/dlq/LogDlqWriterTest.java similarity index 94% rename from src/test/java/io/odpf/firehose/sink/dlq/LogDlqWriterTest.java rename to src/test/java/org/raystack/firehose/sink/dlq/LogDlqWriterTest.java index 1287f865d..936b11d1c 100644 --- a/src/test/java/io/odpf/firehose/sink/dlq/LogDlqWriterTest.java +++ b/src/test/java/org/raystack/firehose/sink/dlq/LogDlqWriterTest.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.sink.dlq; +package org.raystack.firehose.sink.dlq; -import io.odpf.depot.error.ErrorInfo; -import io.odpf.depot.error.ErrorType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.dlq.log.LogDlqWriter; +import org.raystack.depot.error.ErrorInfo; +import org.raystack.depot.error.ErrorType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.dlq.log.LogDlqWriter; import org.apache.commons.lang3.exception.ExceptionUtils; import org.junit.Assert; import org.junit.Before; diff --git a/src/test/java/io/odpf/firehose/sink/dlq/blobstorage/BlobStorageDlqWriterTest.java b/src/test/java/org/raystack/firehose/sink/dlq/blobstorage/BlobStorageDlqWriterTest.java similarity index 94% rename from src/test/java/io/odpf/firehose/sink/dlq/blobstorage/BlobStorageDlqWriterTest.java rename to src/test/java/org/raystack/firehose/sink/dlq/blobstorage/BlobStorageDlqWriterTest.java index 21c7c0303..64c6ff36d 100644 --- a/src/test/java/io/odpf/firehose/sink/dlq/blobstorage/BlobStorageDlqWriterTest.java +++ b/src/test/java/org/raystack/firehose/sink/dlq/blobstorage/BlobStorageDlqWriterTest.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sink.dlq.blobstorage; - -import io.odpf.depot.error.ErrorInfo; -import io.odpf.depot.error.ErrorType; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.sink.common.blobstorage.BlobStorage; -import io.odpf.firehose.sink.common.blobstorage.BlobStorageException; +package org.raystack.firehose.sink.dlq.blobstorage; + +import org.raystack.depot.error.ErrorInfo; +import org.raystack.depot.error.ErrorType; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.sink.common.blobstorage.BlobStorage; +import org.raystack.firehose.sink.common.blobstorage.BlobStorageException; import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/elasticsearch/EsSinkFactoryTest.java b/src/test/java/org/raystack/firehose/sink/elasticsearch/EsSinkFactoryTest.java similarity index 95% rename from src/test/java/io/odpf/firehose/sink/elasticsearch/EsSinkFactoryTest.java rename to src/test/java/org/raystack/firehose/sink/elasticsearch/EsSinkFactoryTest.java index 9ded05d7c..cd71bad70 100644 --- a/src/test/java/io/odpf/firehose/sink/elasticsearch/EsSinkFactoryTest.java +++ b/src/test/java/org/raystack/firehose/sink/elasticsearch/EsSinkFactoryTest.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.sink.elasticsearch; +package org.raystack.firehose.sink.elasticsearch; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.Sink; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.Sink; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.stencil.client.StencilClient; import org.apache.http.HttpHost; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/elasticsearch/EsSinkTest.java b/src/test/java/org/raystack/firehose/sink/elasticsearch/EsSinkTest.java similarity index 97% rename from src/test/java/io/odpf/firehose/sink/elasticsearch/EsSinkTest.java rename to src/test/java/org/raystack/firehose/sink/elasticsearch/EsSinkTest.java index e2b68bbe7..5b18297ad 100644 --- a/src/test/java/io/odpf/firehose/sink/elasticsearch/EsSinkTest.java +++ b/src/test/java/org/raystack/firehose/sink/elasticsearch/EsSinkTest.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.sink.elasticsearch; +package org.raystack.firehose.sink.elasticsearch; -import io.odpf.firehose.config.enums.SinkType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.NeedToRetry; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.elasticsearch.request.EsRequestHandler; +import org.raystack.firehose.config.enums.SinkType; +import org.raystack.firehose.exception.NeedToRetry; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.elasticsearch.request.EsRequestHandler; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.bulk.BulkItemResponse; diff --git a/src/test/java/io/odpf/firehose/sink/elasticsearch/request/ESUpdateRequestHandlerTest.java b/src/test/java/org/raystack/firehose/sink/elasticsearch/request/ESUpdateRequestHandlerTest.java similarity index 96% rename from src/test/java/io/odpf/firehose/sink/elasticsearch/request/ESUpdateRequestHandlerTest.java rename to src/test/java/org/raystack/firehose/sink/elasticsearch/request/ESUpdateRequestHandlerTest.java index 8e165e976..0fd0fe75f 100644 --- a/src/test/java/io/odpf/firehose/sink/elasticsearch/request/ESUpdateRequestHandlerTest.java +++ b/src/test/java/org/raystack/firehose/sink/elasticsearch/request/ESUpdateRequestHandlerTest.java @@ -1,18 +1,18 @@ -package io.odpf.firehose.sink.elasticsearch.request; +package org.raystack.firehose.sink.elasticsearch.request; -import io.odpf.firehose.config.enums.EsSinkMessageType; -import io.odpf.firehose.config.enums.EsSinkRequestType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestAggregatedSupplyMessage; -import io.odpf.firehose.exception.JsonParseException; -import io.odpf.firehose.serializer.MessageToJson; +import org.raystack.firehose.config.enums.EsSinkMessageType; +import org.raystack.firehose.config.enums.EsSinkRequestType; +import org.raystack.firehose.exception.JsonParseException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.serializer.MessageToJson; +import org.raystack.firehose.consumer.TestAggregatedSupplyMessage; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import com.google.protobuf.InvalidProtocolBufferException; -import io.odpf.stencil.StencilClientFactory; -import io.odpf.stencil.client.StencilClient; +import org.raystack.stencil.StencilClientFactory; +import org.raystack.stencil.client.StencilClient; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.common.xcontent.XContentType; diff --git a/src/test/java/io/odpf/firehose/sink/elasticsearch/request/ESUpsertRequestHandlerTest.java b/src/test/java/org/raystack/firehose/sink/elasticsearch/request/ESUpsertRequestHandlerTest.java similarity index 96% rename from src/test/java/io/odpf/firehose/sink/elasticsearch/request/ESUpsertRequestHandlerTest.java rename to src/test/java/org/raystack/firehose/sink/elasticsearch/request/ESUpsertRequestHandlerTest.java index 7d25a8b65..98fd44569 100644 --- a/src/test/java/io/odpf/firehose/sink/elasticsearch/request/ESUpsertRequestHandlerTest.java +++ b/src/test/java/org/raystack/firehose/sink/elasticsearch/request/ESUpsertRequestHandlerTest.java @@ -1,19 +1,19 @@ -package io.odpf.firehose.sink.elasticsearch.request; +package org.raystack.firehose.sink.elasticsearch.request; -import io.odpf.firehose.config.enums.EsSinkMessageType; -import io.odpf.firehose.config.enums.EsSinkRequestType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestAggregatedSupplyMessage; -import io.odpf.firehose.exception.JsonParseException; -import io.odpf.firehose.serializer.MessageToJson; +import org.raystack.firehose.config.enums.EsSinkMessageType; +import org.raystack.firehose.config.enums.EsSinkRequestType; +import org.raystack.firehose.exception.JsonParseException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.serializer.MessageToJson; +import org.raystack.firehose.consumer.TestAggregatedSupplyMessage; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import com.google.protobuf.InvalidProtocolBufferException; -import io.odpf.stencil.StencilClientFactory; -import io.odpf.stencil.client.StencilClient; +import org.raystack.stencil.StencilClientFactory; +import org.raystack.stencil.client.StencilClient; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.common.xcontent.XContentType; diff --git a/src/test/java/io/odpf/firehose/sink/elasticsearch/request/EsRequestHandlerFactoryTest.java b/src/test/java/org/raystack/firehose/sink/elasticsearch/request/EsRequestHandlerFactoryTest.java similarity index 84% rename from src/test/java/io/odpf/firehose/sink/elasticsearch/request/EsRequestHandlerFactoryTest.java rename to src/test/java/org/raystack/firehose/sink/elasticsearch/request/EsRequestHandlerFactoryTest.java index 9822d89cd..dab68f396 100644 --- a/src/test/java/io/odpf/firehose/sink/elasticsearch/request/EsRequestHandlerFactoryTest.java +++ b/src/test/java/org/raystack/firehose/sink/elasticsearch/request/EsRequestHandlerFactoryTest.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.sink.elasticsearch.request; +package org.raystack.firehose.sink.elasticsearch.request; -import io.odpf.firehose.config.EsSinkConfig; -import io.odpf.firehose.config.enums.EsSinkMessageType; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.serializer.MessageToJson; -import io.odpf.firehose.config.enums.EsSinkRequestType; +import org.raystack.firehose.config.EsSinkConfig; +import org.raystack.firehose.config.enums.EsSinkMessageType; +import org.raystack.firehose.config.enums.EsSinkRequestType; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.serializer.MessageToJson; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; diff --git a/src/test/java/io/odpf/firehose/sink/grpc/GrpcClientTest.java b/src/test/java/org/raystack/firehose/sink/grpc/GrpcClientTest.java similarity index 90% rename from src/test/java/io/odpf/firehose/sink/grpc/GrpcClientTest.java rename to src/test/java/org/raystack/firehose/sink/grpc/GrpcClientTest.java index 291d0b6b7..b92747c6d 100644 --- a/src/test/java/io/odpf/firehose/sink/grpc/GrpcClientTest.java +++ b/src/test/java/org/raystack/firehose/sink/grpc/GrpcClientTest.java @@ -1,18 +1,18 @@ -package io.odpf.firehose.sink.grpc; - -import io.odpf.firehose.config.GrpcSinkConfig; -import io.odpf.firehose.consumer.Error; -import io.odpf.firehose.consumer.TestGrpcRequest; -import io.odpf.firehose.consumer.TestGrpcResponse; -import io.odpf.firehose.consumer.TestServerGrpc; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.grpc.client.GrpcClient; +package org.raystack.firehose.sink.grpc; + +import org.raystack.firehose.config.GrpcSinkConfig; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.grpc.client.GrpcClient; +import org.raystack.firehose.consumer.Error; +import org.raystack.firehose.consumer.TestGrpcRequest; +import org.raystack.firehose.consumer.TestGrpcResponse; +import org.raystack.firehose.consumer.TestServerGrpc; import com.google.protobuf.AbstractMessage; import com.google.protobuf.DynamicMessage; import io.grpc.*; import io.grpc.stub.StreamObserver; -import io.odpf.stencil.StencilClientFactory; -import io.odpf.stencil.client.StencilClient; +import org.raystack.stencil.StencilClientFactory; +import org.raystack.stencil.client.StencilClient; import org.aeonbits.owner.ConfigFactory; import org.apache.kafka.common.header.internals.RecordHeader; import org.apache.kafka.common.header.internals.RecordHeaders; @@ -57,8 +57,8 @@ public void setup() throws IOException { Map config = new HashMap<>(); config.put("SINK_GRPC_SERVICE_HOST", "localhost"); config.put("SINK_GRPC_SERVICE_PORT", "5000"); - config.put("SINK_GRPC_METHOD_URL", "io.odpf.firehose.consumer.TestServer/TestRpcMethod"); - config.put("SINK_GRPC_RESPONSE_SCHEMA_PROTO_CLASS", "io.odpf.firehose.consumer.TestGrpcResponse"); + config.put("SINK_GRPC_METHOD_URL", "org.raystack.firehose.consumer.TestServer/TestRpcMethod"); + config.put("SINK_GRPC_RESPONSE_SCHEMA_PROTO_CLASS", "org.raystack.firehose.consumer.TestGrpcResponse"); GrpcSinkConfig grpcSinkConfig = ConfigFactory.create(GrpcSinkConfig.class, config); StencilClient stencilClient = StencilClientFactory.getClient(); diff --git a/src/test/java/io/odpf/firehose/sink/grpc/GrpcSinkFactoryTest.java b/src/test/java/org/raystack/firehose/sink/grpc/GrpcSinkFactoryTest.java similarity index 77% rename from src/test/java/io/odpf/firehose/sink/grpc/GrpcSinkFactoryTest.java rename to src/test/java/org/raystack/firehose/sink/grpc/GrpcSinkFactoryTest.java index a2a6c5be8..f12dc4761 100644 --- a/src/test/java/io/odpf/firehose/sink/grpc/GrpcSinkFactoryTest.java +++ b/src/test/java/org/raystack/firehose/sink/grpc/GrpcSinkFactoryTest.java @@ -1,12 +1,12 @@ -package io.odpf.firehose.sink.grpc; +package org.raystack.firehose.sink.grpc; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.consumer.TestServerGrpc; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.sink.Sink; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.sink.Sink; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.consumer.TestServerGrpc; import io.grpc.Server; import io.grpc.ServerBuilder; -import io.odpf.stencil.client.StencilClient; +import org.raystack.stencil.client.StencilClient; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -49,7 +49,7 @@ public void shouldCreateChannelPoolWithHostAndPort() throws IOException, Deseria .start(); Map config = new HashMap<>(); - config.put("SINK_GRPC_METHOD_URL", "io.odpf.firehose.consumer.TestServer/TestRpcMethod"); + config.put("SINK_GRPC_METHOD_URL", "org.raystack.firehose.consumer.TestServer/TestRpcMethod"); config.put("SINK_GRPC_SERVICE_HOST", "localhost"); config.put("SINK_GRPC_SERVICE_PORT", "5000"); diff --git a/src/test/java/io/odpf/firehose/sink/grpc/GrpcSinkTest.java b/src/test/java/org/raystack/firehose/sink/grpc/GrpcSinkTest.java similarity index 88% rename from src/test/java/io/odpf/firehose/sink/grpc/GrpcSinkTest.java rename to src/test/java/org/raystack/firehose/sink/grpc/GrpcSinkTest.java index 5f731abf1..b1e056d59 100644 --- a/src/test/java/io/odpf/firehose/sink/grpc/GrpcSinkTest.java +++ b/src/test/java/org/raystack/firehose/sink/grpc/GrpcSinkTest.java @@ -1,14 +1,14 @@ -package io.odpf.firehose.sink.grpc; - -import io.odpf.depot.error.ErrorInfo; -import io.odpf.depot.error.ErrorType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestGrpcResponse; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.grpc.client.GrpcClient; +package org.raystack.firehose.sink.grpc; + +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.grpc.client.GrpcClient; +import org.raystack.depot.error.ErrorInfo; +import org.raystack.depot.error.ErrorType; +import org.raystack.firehose.consumer.TestGrpcResponse; import com.google.protobuf.DynamicMessage; -import io.odpf.stencil.client.StencilClient; +import org.raystack.stencil.client.StencilClient; import org.apache.kafka.common.header.internals.RecordHeaders; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/http/HttpSinkFactoryTest.java b/src/test/java/org/raystack/firehose/sink/http/HttpSinkFactoryTest.java similarity index 92% rename from src/test/java/io/odpf/firehose/sink/http/HttpSinkFactoryTest.java rename to src/test/java/org/raystack/firehose/sink/http/HttpSinkFactoryTest.java index 898fc59cc..c32e2e9d5 100644 --- a/src/test/java/io/odpf/firehose/sink/http/HttpSinkFactoryTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/HttpSinkFactoryTest.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.sink.http; +package org.raystack.firehose.sink.http; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.sink.AbstractSink; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.sink.AbstractSink; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.stencil.client.StencilClient; import org.gradle.internal.impldep.org.junit.Before; import org.junit.AfterClass; import org.junit.BeforeClass; diff --git a/src/test/java/io/odpf/firehose/sink/http/HttpSinkTest.java b/src/test/java/org/raystack/firehose/sink/http/HttpSinkTest.java similarity index 98% rename from src/test/java/io/odpf/firehose/sink/http/HttpSinkTest.java rename to src/test/java/org/raystack/firehose/sink/http/HttpSinkTest.java index f6aec6645..5a713c18b 100644 --- a/src/test/java/io/odpf/firehose/sink/http/HttpSinkTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/HttpSinkTest.java @@ -1,13 +1,13 @@ -package io.odpf.firehose.sink.http; +package org.raystack.firehose.sink.http; -import io.odpf.firehose.config.converter.RangeToHashMapConverter; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.exception.NeedToRetry; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.http.request.types.Request; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.config.converter.RangeToHashMapConverter; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.exception.NeedToRetry; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.http.request.types.Request; +import org.raystack.stencil.client.StencilClient; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; diff --git a/src/test/java/io/odpf/firehose/sink/http/auth/OAuth2AccessTokenTest.java b/src/test/java/org/raystack/firehose/sink/http/auth/OAuth2AccessTokenTest.java similarity index 97% rename from src/test/java/io/odpf/firehose/sink/http/auth/OAuth2AccessTokenTest.java rename to src/test/java/org/raystack/firehose/sink/http/auth/OAuth2AccessTokenTest.java index 8f48556f7..cb534000b 100644 --- a/src/test/java/io/odpf/firehose/sink/http/auth/OAuth2AccessTokenTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/auth/OAuth2AccessTokenTest.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.http.auth; +package org.raystack.firehose.sink.http.auth; import org.joda.time.DateTimeUtils; import org.junit.Assert; diff --git a/src/test/java/io/odpf/firehose/sink/http/auth/OAuth2ClientTest.java b/src/test/java/org/raystack/firehose/sink/http/auth/OAuth2ClientTest.java similarity index 97% rename from src/test/java/io/odpf/firehose/sink/http/auth/OAuth2ClientTest.java rename to src/test/java/org/raystack/firehose/sink/http/auth/OAuth2ClientTest.java index 0e7d204ba..9ea36b99b 100644 --- a/src/test/java/io/odpf/firehose/sink/http/auth/OAuth2ClientTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/auth/OAuth2ClientTest.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sink.http.auth; +package org.raystack.firehose.sink.http.auth; -import io.odpf.firehose.exception.OAuth2Exception; +import org.raystack.firehose.exception.OAuth2Exception; import org.joda.time.DateTimeUtils; import org.junit.*; import org.mockserver.integration.ClientAndServer; diff --git a/src/test/java/io/odpf/firehose/sink/http/auth/OAuth2CredentialTest.java b/src/test/java/org/raystack/firehose/sink/http/auth/OAuth2CredentialTest.java similarity index 98% rename from src/test/java/io/odpf/firehose/sink/http/auth/OAuth2CredentialTest.java rename to src/test/java/org/raystack/firehose/sink/http/auth/OAuth2CredentialTest.java index 28bc1ae77..baa4b0e40 100644 --- a/src/test/java/io/odpf/firehose/sink/http/auth/OAuth2CredentialTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/auth/OAuth2CredentialTest.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.sink.http.auth; +package org.raystack.firehose.sink.http.auth; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.depot.metrics.StatsDReporter; import com.google.gson.JsonSyntaxException; import okhttp3.OkHttpClient; import okhttp3.Request; diff --git a/src/test/java/io/odpf/firehose/sink/http/request/RequestFactoryTest.java b/src/test/java/org/raystack/firehose/sink/http/request/RequestFactoryTest.java similarity index 81% rename from src/test/java/io/odpf/firehose/sink/http/request/RequestFactoryTest.java rename to src/test/java/org/raystack/firehose/sink/http/request/RequestFactoryTest.java index a7f7cbedf..015cbcd14 100644 --- a/src/test/java/io/odpf/firehose/sink/http/request/RequestFactoryTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/request/RequestFactoryTest.java @@ -1,15 +1,15 @@ -package io.odpf.firehose.sink.http.request; - - -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.sink.http.request.types.SimpleRequest; -import io.odpf.firehose.sink.http.request.types.DynamicUrlRequest; -import io.odpf.firehose.sink.http.request.types.ParameterizedHeaderRequest; -import io.odpf.firehose.sink.http.request.types.ParameterizedUriRequest; -import io.odpf.firehose.sink.http.request.types.Request; -import io.odpf.firehose.sink.http.request.uri.UriParser; -import io.odpf.stencil.client.StencilClient; +package org.raystack.firehose.sink.http.request; + + +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.sink.http.request.uri.UriParser; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.firehose.sink.http.request.types.SimpleRequest; +import org.raystack.firehose.sink.http.request.types.DynamicUrlRequest; +import org.raystack.firehose.sink.http.request.types.ParameterizedHeaderRequest; +import org.raystack.firehose.sink.http.request.types.ParameterizedUriRequest; +import org.raystack.firehose.sink.http.request.types.Request; +import org.raystack.stencil.client.StencilClient; import org.aeonbits.owner.ConfigFactory; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/http/request/body/JsonBodyTest.java b/src/test/java/org/raystack/firehose/sink/http/request/body/JsonBodyTest.java similarity index 87% rename from src/test/java/io/odpf/firehose/sink/http/request/body/JsonBodyTest.java rename to src/test/java/org/raystack/firehose/sink/http/request/body/JsonBodyTest.java index 2203cdc0a..b81cfd40c 100644 --- a/src/test/java/io/odpf/firehose/sink/http/request/body/JsonBodyTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/request/body/JsonBodyTest.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.http.request.body; +package org.raystack.firehose.sink.http.request.body; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.when; @@ -6,9 +6,9 @@ import java.util.Collections; import java.util.List; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.serializer.MessageSerializer; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.serializer.MessageSerializer; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/http/request/create/BatchRequestCreatorTest.java b/src/test/java/org/raystack/firehose/sink/http/request/create/BatchRequestCreatorTest.java similarity index 94% rename from src/test/java/io/odpf/firehose/sink/http/request/create/BatchRequestCreatorTest.java rename to src/test/java/org/raystack/firehose/sink/http/request/create/BatchRequestCreatorTest.java index f2f5632cf..d3eb72f1b 100644 --- a/src/test/java/io/odpf/firehose/sink/http/request/create/BatchRequestCreatorTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/request/create/BatchRequestCreatorTest.java @@ -1,14 +1,14 @@ -package io.odpf.firehose.sink.http.request.create; - -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.http.request.body.JsonBody; -import io.odpf.firehose.sink.http.request.entity.RequestEntityBuilder; -import io.odpf.firehose.sink.http.request.header.HeaderBuilder; -import io.odpf.firehose.sink.http.request.uri.UriBuilder; +package org.raystack.firehose.sink.http.request.create; + +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.http.request.body.JsonBody; +import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder; +import org.raystack.firehose.sink.http.request.header.HeaderBuilder; +import org.raystack.firehose.sink.http.request.uri.UriBuilder; import org.apache.http.Header; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import org.apache.http.entity.ContentType; diff --git a/src/test/java/io/odpf/firehose/sink/http/request/create/IndividualRequestCreatorTest.java b/src/test/java/org/raystack/firehose/sink/http/request/create/IndividualRequestCreatorTest.java similarity index 96% rename from src/test/java/io/odpf/firehose/sink/http/request/create/IndividualRequestCreatorTest.java rename to src/test/java/org/raystack/firehose/sink/http/request/create/IndividualRequestCreatorTest.java index cb139d8ba..de4b10255 100644 --- a/src/test/java/io/odpf/firehose/sink/http/request/create/IndividualRequestCreatorTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/request/create/IndividualRequestCreatorTest.java @@ -1,14 +1,14 @@ -package io.odpf.firehose.sink.http.request.create; - -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.http.request.body.JsonBody; -import io.odpf.firehose.sink.http.request.entity.RequestEntityBuilder; -import io.odpf.firehose.sink.http.request.header.HeaderBuilder; -import io.odpf.firehose.sink.http.request.uri.UriBuilder; +package org.raystack.firehose.sink.http.request.create; + +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.http.request.body.JsonBody; +import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder; +import org.raystack.firehose.sink.http.request.header.HeaderBuilder; +import org.raystack.firehose.sink.http.request.uri.UriBuilder; import org.apache.commons.io.IOUtils; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import org.junit.Assert; diff --git a/src/test/java/io/odpf/firehose/sink/http/request/entity/RequestEntityBuilderTest.java b/src/test/java/org/raystack/firehose/sink/http/request/entity/RequestEntityBuilderTest.java similarity index 96% rename from src/test/java/io/odpf/firehose/sink/http/request/entity/RequestEntityBuilderTest.java rename to src/test/java/org/raystack/firehose/sink/http/request/entity/RequestEntityBuilderTest.java index a11bba37c..a42148ac7 100644 --- a/src/test/java/io/odpf/firehose/sink/http/request/entity/RequestEntityBuilderTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/request/entity/RequestEntityBuilderTest.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.http.request.entity; +package org.raystack.firehose.sink.http.request.entity; import org.apache.commons.io.IOUtils; import org.apache.http.entity.StringEntity; diff --git a/src/test/java/io/odpf/firehose/sink/http/request/header/HeaderBuilderTest.java b/src/test/java/org/raystack/firehose/sink/http/request/header/HeaderBuilderTest.java similarity index 94% rename from src/test/java/io/odpf/firehose/sink/http/request/header/HeaderBuilderTest.java rename to src/test/java/org/raystack/firehose/sink/http/request/header/HeaderBuilderTest.java index 426bd7f34..835c0fe43 100644 --- a/src/test/java/io/odpf/firehose/sink/http/request/header/HeaderBuilderTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/request/header/HeaderBuilderTest.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.sink.http.request.header; +package org.raystack.firehose.sink.http.request.header; -import io.odpf.firehose.config.enums.HttpSinkParameterSourceType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.proto.ProtoToFieldMapper; -import io.odpf.firehose.sink.http.request.uri.UriParser; +import org.raystack.firehose.config.enums.HttpSinkParameterSourceType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.proto.ProtoToFieldMapper; +import org.raystack.firehose.sink.http.request.uri.UriParser; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; diff --git a/src/test/java/io/odpf/firehose/sink/http/request/types/DynamicUrlRequestTest.java b/src/test/java/org/raystack/firehose/sink/http/request/types/DynamicUrlRequestTest.java similarity index 88% rename from src/test/java/io/odpf/firehose/sink/http/request/types/DynamicUrlRequestTest.java rename to src/test/java/org/raystack/firehose/sink/http/request/types/DynamicUrlRequestTest.java index 4d8f039c1..40c1773f1 100644 --- a/src/test/java/io/odpf/firehose/sink/http/request/types/DynamicUrlRequestTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/request/types/DynamicUrlRequestTest.java @@ -1,15 +1,15 @@ -package io.odpf.firehose.sink.http.request.types; - -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; -import io.odpf.firehose.config.enums.HttpSinkDataFormatType; -import io.odpf.firehose.config.enums.HttpSinkParameterSourceType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.sink.http.request.body.JsonBody; -import io.odpf.firehose.sink.http.request.entity.RequestEntityBuilder; -import io.odpf.firehose.sink.http.request.header.HeaderBuilder; -import io.odpf.firehose.sink.http.request.uri.UriBuilder; +package org.raystack.firehose.sink.http.request.types; + +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.config.enums.HttpSinkDataFormatType; +import org.raystack.firehose.config.enums.HttpSinkParameterSourceType; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.sink.http.request.body.JsonBody; +import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder; +import org.raystack.firehose.sink.http.request.header.HeaderBuilder; +import org.raystack.firehose.sink.http.request.uri.UriBuilder; +import org.raystack.depot.metrics.StatsDReporter; import org.gradle.internal.impldep.org.junit.Assert; import org.junit.Before; import org.mockito.Mock; diff --git a/src/test/java/io/odpf/firehose/sink/http/request/types/ParameterizedHeaderRequestTest.java b/src/test/java/org/raystack/firehose/sink/http/request/types/ParameterizedHeaderRequestTest.java similarity index 89% rename from src/test/java/io/odpf/firehose/sink/http/request/types/ParameterizedHeaderRequestTest.java rename to src/test/java/org/raystack/firehose/sink/http/request/types/ParameterizedHeaderRequestTest.java index 01ca02c59..91a1585b6 100644 --- a/src/test/java/io/odpf/firehose/sink/http/request/types/ParameterizedHeaderRequestTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/request/types/ParameterizedHeaderRequestTest.java @@ -1,17 +1,17 @@ -package io.odpf.firehose.sink.http.request.types; - -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; -import io.odpf.firehose.config.enums.HttpSinkDataFormatType; -import io.odpf.firehose.config.enums.HttpSinkParameterPlacementType; -import io.odpf.firehose.config.enums.HttpSinkParameterSourceType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.proto.ProtoToFieldMapper; -import io.odpf.firehose.sink.http.request.body.JsonBody; -import io.odpf.firehose.sink.http.request.entity.RequestEntityBuilder; -import io.odpf.firehose.sink.http.request.header.HeaderBuilder; -import io.odpf.firehose.sink.http.request.uri.UriBuilder; +package org.raystack.firehose.sink.http.request.types; + +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.config.enums.HttpSinkDataFormatType; +import org.raystack.firehose.config.enums.HttpSinkParameterPlacementType; +import org.raystack.firehose.config.enums.HttpSinkParameterSourceType; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.proto.ProtoToFieldMapper; +import org.raystack.firehose.sink.http.request.body.JsonBody; +import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder; +import org.raystack.firehose.sink.http.request.header.HeaderBuilder; +import org.raystack.firehose.sink.http.request.uri.UriBuilder; +import org.raystack.depot.metrics.StatsDReporter; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; diff --git a/src/test/java/io/odpf/firehose/sink/http/request/types/ParameterizedUriRequestTest.java b/src/test/java/org/raystack/firehose/sink/http/request/types/ParameterizedUriRequestTest.java similarity index 88% rename from src/test/java/io/odpf/firehose/sink/http/request/types/ParameterizedUriRequestTest.java rename to src/test/java/org/raystack/firehose/sink/http/request/types/ParameterizedUriRequestTest.java index ec64c77e0..6d6ce8aa0 100644 --- a/src/test/java/io/odpf/firehose/sink/http/request/types/ParameterizedUriRequestTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/request/types/ParameterizedUriRequestTest.java @@ -1,17 +1,17 @@ -package io.odpf.firehose.sink.http.request.types; - -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; -import io.odpf.firehose.config.enums.HttpSinkDataFormatType; -import io.odpf.firehose.config.enums.HttpSinkParameterPlacementType; -import io.odpf.firehose.config.enums.HttpSinkParameterSourceType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.proto.ProtoToFieldMapper; -import io.odpf.firehose.sink.http.request.body.JsonBody; -import io.odpf.firehose.sink.http.request.entity.RequestEntityBuilder; -import io.odpf.firehose.sink.http.request.header.HeaderBuilder; -import io.odpf.firehose.sink.http.request.uri.UriBuilder; +package org.raystack.firehose.sink.http.request.types; + +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.config.enums.HttpSinkDataFormatType; +import org.raystack.firehose.config.enums.HttpSinkParameterPlacementType; +import org.raystack.firehose.config.enums.HttpSinkParameterSourceType; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.proto.ProtoToFieldMapper; +import org.raystack.firehose.sink.http.request.body.JsonBody; +import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder; +import org.raystack.firehose.sink.http.request.header.HeaderBuilder; +import org.raystack.firehose.sink.http.request.uri.UriBuilder; +import org.raystack.depot.metrics.StatsDReporter; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; diff --git a/src/test/java/io/odpf/firehose/sink/http/request/types/SimpleRequestTest.java b/src/test/java/org/raystack/firehose/sink/http/request/types/SimpleRequestTest.java similarity index 89% rename from src/test/java/io/odpf/firehose/sink/http/request/types/SimpleRequestTest.java rename to src/test/java/org/raystack/firehose/sink/http/request/types/SimpleRequestTest.java index 423a5679a..8a1226245 100644 --- a/src/test/java/io/odpf/firehose/sink/http/request/types/SimpleRequestTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/request/types/SimpleRequestTest.java @@ -1,15 +1,15 @@ -package io.odpf.firehose.sink.http.request.types; - -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.HttpSinkConfig; -import io.odpf.firehose.config.enums.HttpSinkRequestMethodType; -import io.odpf.firehose.config.enums.HttpSinkDataFormatType; -import io.odpf.firehose.config.enums.HttpSinkParameterSourceType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.sink.http.request.body.JsonBody; -import io.odpf.firehose.sink.http.request.entity.RequestEntityBuilder; -import io.odpf.firehose.sink.http.request.header.HeaderBuilder; -import io.odpf.firehose.sink.http.request.uri.UriBuilder; +package org.raystack.firehose.sink.http.request.types; + +import org.raystack.firehose.config.HttpSinkConfig; +import org.raystack.firehose.config.enums.HttpSinkDataFormatType; +import org.raystack.firehose.config.enums.HttpSinkParameterSourceType; +import org.raystack.firehose.config.enums.HttpSinkRequestMethodType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.sink.http.request.body.JsonBody; +import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder; +import org.raystack.firehose.sink.http.request.header.HeaderBuilder; +import org.raystack.firehose.sink.http.request.uri.UriBuilder; +import org.raystack.depot.metrics.StatsDReporter; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; diff --git a/src/test/java/io/odpf/firehose/sink/http/request/uri/UriBuilderTest.java b/src/test/java/org/raystack/firehose/sink/http/request/uri/UriBuilderTest.java similarity index 95% rename from src/test/java/io/odpf/firehose/sink/http/request/uri/UriBuilderTest.java rename to src/test/java/org/raystack/firehose/sink/http/request/uri/UriBuilderTest.java index 9ac1a7247..cd480acc7 100644 --- a/src/test/java/io/odpf/firehose/sink/http/request/uri/UriBuilderTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/request/uri/UriBuilderTest.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.http.request.uri; +package org.raystack.firehose.sink.http.request.uri; -import io.odpf.firehose.config.enums.HttpSinkParameterSourceType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.proto.ProtoToFieldMapper; +import org.raystack.firehose.config.enums.HttpSinkParameterSourceType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.proto.ProtoToFieldMapper; import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/http/request/uri/UriParserTest.java b/src/test/java/org/raystack/firehose/sink/http/request/uri/UriParserTest.java similarity index 93% rename from src/test/java/io/odpf/firehose/sink/http/request/uri/UriParserTest.java rename to src/test/java/org/raystack/firehose/sink/http/request/uri/UriParserTest.java index fa0969200..5ddf9ce93 100644 --- a/src/test/java/io/odpf/firehose/sink/http/request/uri/UriParserTest.java +++ b/src/test/java/org/raystack/firehose/sink/http/request/uri/UriParserTest.java @@ -1,16 +1,13 @@ -package io.odpf.firehose.sink.http.request.uri; +package org.raystack.firehose.sink.http.request.uri; - - - -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestBookingLogMessage; -import io.odpf.firehose.consumer.TestKey; -import io.odpf.firehose.consumer.TestMessage; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.consumer.TestBookingLogMessage; +import org.raystack.firehose.consumer.TestKey; +import org.raystack.firehose.consumer.TestMessage; import com.google.protobuf.InvalidProtocolBufferException; -import io.odpf.stencil.client.ClassLoadStencilClient; -import io.odpf.stencil.client.StencilClient; -import io.odpf.stencil.Parser; +import org.raystack.stencil.client.ClassLoadStencilClient; +import org.raystack.stencil.client.StencilClient; +import org.raystack.stencil.Parser; import org.apache.kafka.common.errors.InvalidConfigurationException; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/influxdb/InfluxSinkTest.java b/src/test/java/org/raystack/firehose/sink/influxdb/InfluxSinkTest.java similarity index 93% rename from src/test/java/io/odpf/firehose/sink/influxdb/InfluxSinkTest.java rename to src/test/java/org/raystack/firehose/sink/influxdb/InfluxSinkTest.java index bcdcb8454..0090fc097 100644 --- a/src/test/java/io/odpf/firehose/sink/influxdb/InfluxSinkTest.java +++ b/src/test/java/org/raystack/firehose/sink/influxdb/InfluxSinkTest.java @@ -1,21 +1,21 @@ -package io.odpf.firehose.sink.influxdb; +package org.raystack.firehose.sink.influxdb; -import io.odpf.firehose.config.InfluxSinkConfig; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestBookingLogMessage; -import io.odpf.firehose.consumer.TestFeedbackLogKey; -import io.odpf.firehose.consumer.TestFeedbackLogMessage; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.Sink; +import org.raystack.firehose.config.InfluxSinkConfig; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.Sink; +import org.raystack.firehose.consumer.TestBookingLogMessage; +import org.raystack.firehose.consumer.TestFeedbackLogKey; +import org.raystack.firehose.consumer.TestFeedbackLogMessage; import com.google.protobuf.DynamicMessage; import com.google.protobuf.Timestamp; -import io.odpf.stencil.StencilClientFactory; -import io.odpf.stencil.client.StencilClient; -import io.odpf.stencil.Parser; +import org.raystack.stencil.StencilClientFactory; +import org.raystack.stencil.client.StencilClient; +import org.raystack.stencil.Parser; import org.aeonbits.owner.ConfigFactory; import org.influxdb.InfluxDB; import org.influxdb.dto.BatchPoints; diff --git a/src/test/java/io/odpf/firehose/sink/influxdb/builder/PointBuilderTest.java b/src/test/java/org/raystack/firehose/sink/influxdb/builder/PointBuilderTest.java similarity index 96% rename from src/test/java/io/odpf/firehose/sink/influxdb/builder/PointBuilderTest.java rename to src/test/java/org/raystack/firehose/sink/influxdb/builder/PointBuilderTest.java index a9cd84578..c5fc3a5b8 100644 --- a/src/test/java/io/odpf/firehose/sink/influxdb/builder/PointBuilderTest.java +++ b/src/test/java/org/raystack/firehose/sink/influxdb/builder/PointBuilderTest.java @@ -1,13 +1,13 @@ -package io.odpf.firehose.sink.influxdb.builder; +package org.raystack.firehose.sink.influxdb.builder; -import io.odpf.firehose.config.InfluxSinkConfig; -import io.odpf.firehose.consumer.TestDurationMessage; -import io.odpf.firehose.consumer.TestFeedbackLogMessage; +import org.raystack.firehose.config.InfluxSinkConfig; +import org.raystack.firehose.consumer.TestDurationMessage; +import org.raystack.firehose.consumer.TestFeedbackLogMessage; import com.google.protobuf.Duration; import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Timestamp; -import io.odpf.firehose.consumer.TestFeedbackSource; +import org.raystack.firehose.consumer.TestFeedbackSource; import org.aeonbits.owner.ConfigFactory; import org.influxdb.dto.Point; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/jdbc/HikariJdbcConnectionPoolTest.java b/src/test/java/org/raystack/firehose/sink/jdbc/HikariJdbcConnectionPoolTest.java similarity index 96% rename from src/test/java/io/odpf/firehose/sink/jdbc/HikariJdbcConnectionPoolTest.java rename to src/test/java/org/raystack/firehose/sink/jdbc/HikariJdbcConnectionPoolTest.java index f2859b182..1eb3507ae 100644 --- a/src/test/java/io/odpf/firehose/sink/jdbc/HikariJdbcConnectionPoolTest.java +++ b/src/test/java/org/raystack/firehose/sink/jdbc/HikariJdbcConnectionPoolTest.java @@ -1,4 +1,4 @@ -package io.odpf.firehose.sink.jdbc; +package org.raystack.firehose.sink.jdbc; import com.zaxxer.hikari.HikariDataSource; import org.junit.Before; diff --git a/src/test/java/io/odpf/firehose/sink/jdbc/JdbcSinkTest.java b/src/test/java/org/raystack/firehose/sink/jdbc/JdbcSinkTest.java similarity index 95% rename from src/test/java/io/odpf/firehose/sink/jdbc/JdbcSinkTest.java rename to src/test/java/org/raystack/firehose/sink/jdbc/JdbcSinkTest.java index 80031cb0b..dad866e47 100644 --- a/src/test/java/io/odpf/firehose/sink/jdbc/JdbcSinkTest.java +++ b/src/test/java/org/raystack/firehose/sink/jdbc/JdbcSinkTest.java @@ -1,10 +1,11 @@ -package io.odpf.firehose.sink.jdbc; +package org.raystack.firehose.sink.jdbc; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.stencil.client.StencilClient; +import junit.framework.TestCase; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -20,7 +21,6 @@ import java.util.Arrays; import java.util.List; -import static junit.framework.TestCase.assertEquals; import static org.mockito.Mockito.*; @RunWith(MockitoJUnitRunner.class) @@ -90,7 +90,7 @@ public void shouldReturnEmptyListWhenNoException() throws IOException, Deseriali List messages = Arrays.asList(new Message(new byte[0], new byte[0], "topic", 0, 100), new Message(new byte[0], new byte[0], "topic", 0, 100)); - assertEquals(jdbcSink.pushMessage(messages).size(), 0); + TestCase.assertEquals(jdbcSink.pushMessage(messages).size(), 0); verify(firehoseInstrumentation, times(1)).captureSinkExecutionTelemetry("db", messages.size()); } diff --git a/src/test/java/io/odpf/firehose/sink/jdbc/ProtoToFieldMapperTest.java b/src/test/java/org/raystack/firehose/sink/jdbc/ProtoToFieldMapperTest.java similarity index 94% rename from src/test/java/io/odpf/firehose/sink/jdbc/ProtoToFieldMapperTest.java rename to src/test/java/org/raystack/firehose/sink/jdbc/ProtoToFieldMapperTest.java index d1f5908af..f785cc8b0 100644 --- a/src/test/java/io/odpf/firehose/sink/jdbc/ProtoToFieldMapperTest.java +++ b/src/test/java/org/raystack/firehose/sink/jdbc/ProtoToFieldMapperTest.java @@ -1,19 +1,19 @@ -package io.odpf.firehose.sink.jdbc; +package org.raystack.firehose.sink.jdbc; -import io.odpf.firehose.consumer.TestMapMessage; -import io.odpf.firehose.consumer.TestMessage; -import io.odpf.firehose.consumer.TestFeedbackLogMessage; -import io.odpf.firehose.consumer.TestBookingLogMessage; -import io.odpf.firehose.consumer.TestNestedMessage; -import io.odpf.firehose.consumer.TestNestedRepeatedMessage; -import io.odpf.firehose.proto.ProtoToFieldMapper; +import org.raystack.firehose.proto.ProtoToFieldMapper; +import org.raystack.firehose.consumer.TestMapMessage; +import org.raystack.firehose.consumer.TestMessage; +import org.raystack.firehose.consumer.TestFeedbackLogMessage; +import org.raystack.firehose.consumer.TestBookingLogMessage; +import org.raystack.firehose.consumer.TestNestedMessage; +import org.raystack.firehose.consumer.TestNestedRepeatedMessage; import com.google.protobuf.Timestamp; -import io.odpf.stencil.StencilClientFactory; -import io.odpf.stencil.client.StencilClient; -import io.odpf.stencil.Parser; +import org.raystack.stencil.StencilClientFactory; +import org.raystack.stencil.client.StencilClient; +import org.raystack.stencil.Parser; import net.minidev.json.JSONObject; import org.gradle.internal.impldep.org.testng.Assert; import org.junit.After; diff --git a/src/test/java/io/odpf/firehose/sink/jdbc/QueryTemplateTest.java b/src/test/java/org/raystack/firehose/sink/jdbc/QueryTemplateTest.java similarity index 97% rename from src/test/java/io/odpf/firehose/sink/jdbc/QueryTemplateTest.java rename to src/test/java/org/raystack/firehose/sink/jdbc/QueryTemplateTest.java index feaa2f5f5..0df5f53c0 100644 --- a/src/test/java/io/odpf/firehose/sink/jdbc/QueryTemplateTest.java +++ b/src/test/java/org/raystack/firehose/sink/jdbc/QueryTemplateTest.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.jdbc; +package org.raystack.firehose.sink.jdbc; -import io.odpf.firehose.config.JdbcSinkConfig; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.proto.ProtoToFieldMapper; +import org.raystack.firehose.config.JdbcSinkConfig; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.proto.ProtoToFieldMapper; import org.gradle.internal.impldep.org.testng.Assert; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/jdbc/field/JdbcMapFieldTest.java b/src/test/java/org/raystack/firehose/sink/jdbc/field/JdbcMapFieldTest.java similarity index 86% rename from src/test/java/io/odpf/firehose/sink/jdbc/field/JdbcMapFieldTest.java rename to src/test/java/org/raystack/firehose/sink/jdbc/field/JdbcMapFieldTest.java index c450b878d..016d7133f 100644 --- a/src/test/java/io/odpf/firehose/sink/jdbc/field/JdbcMapFieldTest.java +++ b/src/test/java/org/raystack/firehose/sink/jdbc/field/JdbcMapFieldTest.java @@ -1,13 +1,13 @@ -package io.odpf.firehose.sink.jdbc.field; +package org.raystack.firehose.sink.jdbc.field; -import io.odpf.firehose.consumer.TestAuditEntityLogMessage; +import org.raystack.firehose.consumer.TestAuditEntityLogMessage; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; -import io.odpf.stencil.StencilClientFactory; -import io.odpf.stencil.client.StencilClient; +import org.raystack.stencil.StencilClientFactory; +import org.raystack.stencil.client.StencilClient; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -34,7 +34,7 @@ public void shouldParseTheMapFieldAsString() throws Exception { TestAuditEntityLogMessage auditEntityLogMessage = TestAuditEntityLogMessage.newBuilder().putAllCurrentState(currentStates).build(); Descriptors.FieldDescriptor currentEntityFieldDescriptor = TestAuditEntityLogMessage.getDescriptor().getFields().get(6); - DynamicMessage auditEntityParsed = stencilClient.getParser("io.odpf.firehose.consumer.TestAuditEntityLogMessage").parse(auditEntityLogMessage.toByteArray()); + DynamicMessage auditEntityParsed = stencilClient.getParser("org.raystack.firehose.consumer.TestAuditEntityLogMessage").parse(auditEntityLogMessage.toByteArray()); Object columnValue = auditEntityParsed.getField(currentEntityFieldDescriptor); JdbcMapField jdbcMapField = new JdbcMapField(columnValue, currentEntityFieldDescriptor); @@ -53,7 +53,7 @@ public void shouldParseTheMapFieldWithEmptyValueAsString() throws Exception { TestAuditEntityLogMessage auditEntityLogMessage = TestAuditEntityLogMessage.newBuilder().putAllCurrentState(currentStates).build(); Descriptors.FieldDescriptor currentEntityFieldDescriptor = TestAuditEntityLogMessage.getDescriptor().getFields().get(6); - DynamicMessage auditEntityParsed = stencilClient.getParser("io.odpf.firehose.consumer.TestAuditEntityLogMessage").parse(auditEntityLogMessage.toByteArray()); + DynamicMessage auditEntityParsed = stencilClient.getParser("org.raystack.firehose.consumer.TestAuditEntityLogMessage").parse(auditEntityLogMessage.toByteArray()); Object columnValue = auditEntityParsed.getField(currentEntityFieldDescriptor); JdbcMapField jdbcMapField = new JdbcMapField(columnValue, currentEntityFieldDescriptor); @@ -73,7 +73,7 @@ public void shouldBeAbleToParseMapFields() throws Exception { TestAuditEntityLogMessage auditEntityLogMessage = TestAuditEntityLogMessage.newBuilder().putAllCurrentState(currentStates).build(); Descriptors.FieldDescriptor currentEntityFieldDescriptor = TestAuditEntityLogMessage.getDescriptor().getFields().get(6); - DynamicMessage auditEntityParsed = stencilClient.getParser("io.odpf.firehose.consumer.TestAuditEntityLogMessage").parse(auditEntityLogMessage.toByteArray()); + DynamicMessage auditEntityParsed = stencilClient.getParser("org.raystack.firehose.consumer.TestAuditEntityLogMessage").parse(auditEntityLogMessage.toByteArray()); Object columnValue = auditEntityParsed.getField(currentEntityFieldDescriptor); JdbcMapField jdbcMapField = new JdbcMapField(columnValue, currentEntityFieldDescriptor); @@ -90,7 +90,7 @@ public void shouldBeAbleToParseStringFields() throws Exception { TestAuditEntityLogMessage auditEntityLogMessage = TestAuditEntityLogMessage.newBuilder().setAuditId("audit_id").putAllCurrentState(currentStates).build(); Descriptors.FieldDescriptor auditIdFieldDescriptor = TestAuditEntityLogMessage.getDescriptor().getFields().get(0); - DynamicMessage auditEntityParsed = stencilClient.getParser("io.odpf.firehose.consumer.TestAuditEntityLogMessage").parse(auditEntityLogMessage.toByteArray()); + DynamicMessage auditEntityParsed = stencilClient.getParser("org.raystack.firehose.consumer.TestAuditEntityLogMessage").parse(auditEntityLogMessage.toByteArray()); Object columnValue = auditEntityParsed.getField(auditIdFieldDescriptor); JdbcMapField jdbcMapField = new JdbcMapField(columnValue, auditIdFieldDescriptor); diff --git a/src/test/java/io/odpf/firehose/sink/jdbc/field/message/DBCollectionFieldTest.java b/src/test/java/org/raystack/firehose/sink/jdbc/field/message/DBCollectionFieldTest.java similarity index 87% rename from src/test/java/io/odpf/firehose/sink/jdbc/field/message/DBCollectionFieldTest.java rename to src/test/java/org/raystack/firehose/sink/jdbc/field/message/DBCollectionFieldTest.java index 35e0a4a66..fc05fa5c7 100644 --- a/src/test/java/io/odpf/firehose/sink/jdbc/field/message/DBCollectionFieldTest.java +++ b/src/test/java/org/raystack/firehose/sink/jdbc/field/message/DBCollectionFieldTest.java @@ -1,12 +1,12 @@ -package io.odpf.firehose.sink.jdbc.field.message; +package org.raystack.firehose.sink.jdbc.field.message; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; -import io.odpf.firehose.consumer.TestFeedbackLogMessage; -import io.odpf.firehose.consumer.TestReason; -import io.odpf.stencil.StencilClientFactory; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.consumer.TestFeedbackLogMessage; +import org.raystack.firehose.consumer.TestReason; +import org.raystack.stencil.StencilClientFactory; +import org.raystack.stencil.client.StencilClient; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -38,7 +38,7 @@ public void shouldParseTheCollectionFieldAsString() throws Exception { Descriptors.FieldDescriptor reasonFieldDescriptor = TestFeedbackLogMessage.getDescriptor().getFields().get(10); - DynamicMessage feedbackParsed = stencilClient.getParser("io.odpf.firehose.consumer.TestFeedbackLogMessage").parse(feedback.toByteArray()); + DynamicMessage feedbackParsed = stencilClient.getParser("org.raystack.firehose.consumer.TestFeedbackLogMessage").parse(feedback.toByteArray()); Object columnValue = feedbackParsed.getField(reasonFieldDescriptor); JdbcCollectionField dbCollectionField = new JdbcCollectionField(columnValue, reasonFieldDescriptor); @@ -62,7 +62,7 @@ public void shouldBeAbleToParseCollectionFields() throws Exception { Descriptors.FieldDescriptor reasonFieldDescriptor = TestFeedbackLogMessage.getDescriptor().getFields().get(10); - DynamicMessage feedbackParsed = stencilClient.getParser("io.odpf.firehose.consumer.TestFeedbackLogMessage").parse(feedback.toByteArray()); + DynamicMessage feedbackParsed = stencilClient.getParser("org.raystack.firehose.consumer.TestFeedbackLogMessage").parse(feedback.toByteArray()); Object columnValue = feedbackParsed.getField(reasonFieldDescriptor); JdbcCollectionField dbCollectionField = new JdbcCollectionField(columnValue, reasonFieldDescriptor); @@ -86,7 +86,7 @@ public void shouldNotBeAbleToParseStringFields() throws Exception { Descriptors.FieldDescriptor orderNumberFieldDescriptor = TestFeedbackLogMessage.getDescriptor().getFields().get(0); - DynamicMessage feedbackParsed = stencilClient.getParser("io.odpf.firehose.consumer.TestFeedbackLogMessage").parse(feedback.toByteArray()); + DynamicMessage feedbackParsed = stencilClient.getParser("org.raystack.firehose.consumer.TestFeedbackLogMessage").parse(feedback.toByteArray()); Object columnValue = feedbackParsed.getField(orderNumberFieldDescriptor); JdbcCollectionField dbCollectionField = new JdbcCollectionField(columnValue, orderNumberFieldDescriptor); diff --git a/src/test/java/io/odpf/firehose/sink/jdbc/field/message/DBDefaultMessageFieldTest.java b/src/test/java/org/raystack/firehose/sink/jdbc/field/message/DBDefaultMessageFieldTest.java similarity index 83% rename from src/test/java/io/odpf/firehose/sink/jdbc/field/message/DBDefaultMessageFieldTest.java rename to src/test/java/org/raystack/firehose/sink/jdbc/field/message/DBDefaultMessageFieldTest.java index 5d5aa3943..ba4904674 100644 --- a/src/test/java/io/odpf/firehose/sink/jdbc/field/message/DBDefaultMessageFieldTest.java +++ b/src/test/java/org/raystack/firehose/sink/jdbc/field/message/DBDefaultMessageFieldTest.java @@ -1,14 +1,14 @@ -package io.odpf.firehose.sink.jdbc.field.message; +package org.raystack.firehose.sink.jdbc.field.message; -import io.odpf.firehose.consumer.TestBookingLogMessage; -import io.odpf.firehose.consumer.TestLocation; +import org.raystack.firehose.consumer.TestBookingLogMessage; +import org.raystack.firehose.consumer.TestLocation; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; -import io.odpf.stencil.StencilClientFactory; -import io.odpf.stencil.client.StencilClient; +import org.raystack.stencil.StencilClientFactory; +import org.raystack.stencil.client.StencilClient; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -27,7 +27,7 @@ public void shouldParseTheMessageFieldAsString() throws Exception { TestBookingLogMessage booking = TestBookingLogMessage.newBuilder().setDriverDropoffLocation(TestLocation.newBuilder().setName("location_name").build()).build(); Descriptors.FieldDescriptor locationFieldDescriptor = TestBookingLogMessage.getDescriptor().getFields().get(12); - DynamicMessage bookingParsed = stencilClient.getParser("io.odpf.firehose.consumer.TestBookingLogMessage").parse(booking.toByteArray()); + DynamicMessage bookingParsed = stencilClient.getParser("org.raystack.firehose.consumer.TestBookingLogMessage").parse(booking.toByteArray()); Object columnValue = bookingParsed.getField(locationFieldDescriptor); JdbcDefaultMessageField dbDefaultMessageField = new JdbcDefaultMessageField(columnValue); @@ -40,7 +40,7 @@ public void shouldBeAbleToParseMessageFields() throws Exception { TestBookingLogMessage booking = TestBookingLogMessage.newBuilder().setDriverDropoffLocation(TestLocation.newBuilder().setName("location_name").build()).build(); Descriptors.FieldDescriptor locationFieldDescriptor = TestBookingLogMessage.getDescriptor().getFields().get(12); - DynamicMessage bookingParsed = stencilClient.getParser("io.odpf.firehose.consumer.TestBookingLogMessage").parse(booking.toByteArray()); + DynamicMessage bookingParsed = stencilClient.getParser("org.raystack.firehose.consumer.TestBookingLogMessage").parse(booking.toByteArray()); Object columnValue = bookingParsed.getField(locationFieldDescriptor); JdbcDefaultMessageField dbDefaultMessageField = new JdbcDefaultMessageField(columnValue); @@ -54,7 +54,7 @@ public void shouldNotBeAbleToParseNormalFields() throws Exception { TestBookingLogMessage booking = TestBookingLogMessage.newBuilder().setDriverDropoffLocation(TestLocation.newBuilder().setName("location_name").build()).setCustomerId("customer_id").build(); Descriptors.FieldDescriptor customerIdFieldDescriptor = TestBookingLogMessage.getDescriptor().getFields().get(5); - DynamicMessage bookingParsed = stencilClient.getParser("io.odpf.firehose.consumer.TestBookingLogMessage").parse(booking.toByteArray()); + DynamicMessage bookingParsed = stencilClient.getParser("org.raystack.firehose.consumer.TestBookingLogMessage").parse(booking.toByteArray()); Object columnValue = bookingParsed.getField(customerIdFieldDescriptor); JdbcDefaultMessageField dbDefaultMessageField = new JdbcDefaultMessageField(columnValue); diff --git a/src/test/java/io/odpf/firehose/sink/jdbc/field/message/DBTimestampFieldTest.java b/src/test/java/org/raystack/firehose/sink/jdbc/field/message/DBTimestampFieldTest.java similarity index 84% rename from src/test/java/io/odpf/firehose/sink/jdbc/field/message/DBTimestampFieldTest.java rename to src/test/java/org/raystack/firehose/sink/jdbc/field/message/DBTimestampFieldTest.java index fce10849e..58231618c 100644 --- a/src/test/java/io/odpf/firehose/sink/jdbc/field/message/DBTimestampFieldTest.java +++ b/src/test/java/org/raystack/firehose/sink/jdbc/field/message/DBTimestampFieldTest.java @@ -1,14 +1,14 @@ -package io.odpf.firehose.sink.jdbc.field.message; +package org.raystack.firehose.sink.jdbc.field.message; -import io.odpf.firehose.consumer.TestAuditEntityLogMessage; +import org.raystack.firehose.consumer.TestAuditEntityLogMessage; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import com.google.protobuf.Timestamp; -import io.odpf.stencil.StencilClientFactory; -import io.odpf.stencil.client.StencilClient; +import org.raystack.stencil.StencilClientFactory; +import org.raystack.stencil.client.StencilClient; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -32,7 +32,7 @@ public void shouldParseTheTimestampFieldAsString() throws Exception { TestAuditEntityLogMessage auditEntityLogMessage = TestAuditEntityLogMessage.newBuilder().setEventTimestamp(timestamp).build(); Descriptors.FieldDescriptor timestampFieldDescriptor = TestAuditEntityLogMessage.getDescriptor().getFields().get(3); - DynamicMessage auditEntityParsed = stencilClient.getParser("io.odpf.firehose.consumer.TestAuditEntityLogMessage").parse(auditEntityLogMessage.toByteArray()); + DynamicMessage auditEntityParsed = stencilClient.getParser("org.raystack.firehose.consumer.TestAuditEntityLogMessage").parse(auditEntityLogMessage.toByteArray()); Object columnValue = auditEntityParsed.getField(timestampFieldDescriptor); JdbcTimestampField dbTime = new JdbcTimestampField(columnValue); @@ -48,7 +48,7 @@ public void shouldBeAbleToParseTimestampFields() throws Exception { TestAuditEntityLogMessage auditEntityLogMessage = TestAuditEntityLogMessage.newBuilder().setEventTimestamp(timestamp).build(); Descriptors.FieldDescriptor timestampFieldDescriptor = TestAuditEntityLogMessage.getDescriptor().getFields().get(3); - DynamicMessage auditEntityParsed = stencilClient.getParser("io.odpf.firehose.consumer.TestAuditEntityLogMessage").parse(auditEntityLogMessage.toByteArray()); + DynamicMessage auditEntityParsed = stencilClient.getParser("org.raystack.firehose.consumer.TestAuditEntityLogMessage").parse(auditEntityLogMessage.toByteArray()); Object columnValue = auditEntityParsed.getField(timestampFieldDescriptor); JdbcTimestampField dbTime = new JdbcTimestampField(columnValue); @@ -64,7 +64,7 @@ public void shouldNotBeAbleToParseStringFields() throws Exception { TestAuditEntityLogMessage auditEntityLogMessage = TestAuditEntityLogMessage.newBuilder().setEventTimestamp(timestamp).setAuditId("audit_id").build(); Descriptors.FieldDescriptor auditIdFieldDescriptor = TestAuditEntityLogMessage.getDescriptor().getFields().get(0); - DynamicMessage auditEntityParsed = stencilClient.getParser("io.odpf.firehose.consumer.TestAuditEntityLogMessage").parse(auditEntityLogMessage.toByteArray()); + DynamicMessage auditEntityParsed = stencilClient.getParser("org.raystack.firehose.consumer.TestAuditEntityLogMessage").parse(auditEntityLogMessage.toByteArray()); Object columnValue = auditEntityParsed.getField(auditIdFieldDescriptor); JdbcTimestampField dbTime = new JdbcTimestampField(columnValue); diff --git a/src/test/java/io/odpf/firehose/sink/mongodb/MongoSinkFactoryTest.java b/src/test/java/org/raystack/firehose/sink/mongodb/MongoSinkFactoryTest.java similarity index 97% rename from src/test/java/io/odpf/firehose/sink/mongodb/MongoSinkFactoryTest.java rename to src/test/java/org/raystack/firehose/sink/mongodb/MongoSinkFactoryTest.java index 3a64e1483..5b65f7d4d 100644 --- a/src/test/java/io/odpf/firehose/sink/mongodb/MongoSinkFactoryTest.java +++ b/src/test/java/org/raystack/firehose/sink/mongodb/MongoSinkFactoryTest.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.mongodb; +package org.raystack.firehose.sink.mongodb; +import org.raystack.firehose.config.MongoSinkConfig; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import com.mongodb.MongoClient; -import io.odpf.firehose.config.MongoSinkConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; import org.aeonbits.owner.ConfigFactory; import org.junit.Before; import org.junit.Rule; diff --git a/src/test/java/io/odpf/firehose/sink/mongodb/MongoSinkTest.java b/src/test/java/org/raystack/firehose/sink/mongodb/MongoSinkTest.java similarity index 96% rename from src/test/java/io/odpf/firehose/sink/mongodb/MongoSinkTest.java rename to src/test/java/org/raystack/firehose/sink/mongodb/MongoSinkTest.java index bfa5ca0a6..b4beebd52 100644 --- a/src/test/java/io/odpf/firehose/sink/mongodb/MongoSinkTest.java +++ b/src/test/java/org/raystack/firehose/sink/mongodb/MongoSinkTest.java @@ -1,14 +1,14 @@ -package io.odpf.firehose.sink.mongodb; +package org.raystack.firehose.sink.mongodb; +import org.raystack.firehose.config.enums.SinkType; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.mongodb.client.MongoSinkClient; +import org.raystack.firehose.sink.mongodb.request.MongoRequestHandler; import com.mongodb.bulk.BulkWriteError; import com.mongodb.client.model.ReplaceOneModel; import com.mongodb.client.model.ReplaceOptions; import com.mongodb.client.model.WriteModel; -import io.odpf.firehose.config.enums.SinkType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.mongodb.client.MongoSinkClient; -import io.odpf.firehose.sink.mongodb.request.MongoRequestHandler; import org.bson.BsonDocument; import org.bson.Document; import org.junit.Before; diff --git a/src/test/java/io/odpf/firehose/sink/mongodb/client/MongoSinkClientTest.java b/src/test/java/org/raystack/firehose/sink/mongodb/client/MongoSinkClientTest.java similarity index 97% rename from src/test/java/io/odpf/firehose/sink/mongodb/client/MongoSinkClientTest.java rename to src/test/java/org/raystack/firehose/sink/mongodb/client/MongoSinkClientTest.java index 00060a299..953ba0ebb 100644 --- a/src/test/java/io/odpf/firehose/sink/mongodb/client/MongoSinkClientTest.java +++ b/src/test/java/org/raystack/firehose/sink/mongodb/client/MongoSinkClientTest.java @@ -1,5 +1,8 @@ -package io.odpf.firehose.sink.mongodb.client; +package org.raystack.firehose.sink.mongodb.client; +import org.raystack.firehose.config.MongoSinkConfig; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; import com.mongodb.MongoBulkWriteException; import com.mongodb.MongoClient; import com.mongodb.ServerAddress; @@ -10,8 +13,6 @@ import com.mongodb.client.model.ReplaceOneModel; import com.mongodb.client.model.ReplaceOptions; import com.mongodb.client.model.WriteModel; -import io.odpf.firehose.config.MongoSinkConfig; -import io.odpf.firehose.metrics.FirehoseInstrumentation; import org.bson.BsonDocument; import org.bson.Document; import org.junit.Assert; @@ -26,7 +27,6 @@ import java.util.Collections; import java.util.List; -import static io.odpf.firehose.metrics.Metrics.*; import static org.mockito.Mockito.*; import static org.mockito.MockitoAnnotations.initMocks; @@ -283,7 +283,7 @@ public void shouldIncrementFailureCounterTagWhenPrimaryKeyNotFoundInUpdateOnlyMo when(mongoCollection.bulkWrite(request)).thenReturn(new BulkWriteResultMock(true, 0, 1, 0)); mongoSinkClient.processRequest(request); - verify(firehoseInstrumentation, times(1)).incrementCounter(SINK_MESSAGES_DROP_TOTAL, "cause=Primary Key value not found"); + verify(firehoseInstrumentation, times(1)).incrementCounter(Metrics.SINK_MESSAGES_DROP_TOTAL, "cause=Primary Key value not found"); } @Test @@ -295,8 +295,8 @@ public void shouldIncrementInsertedCounterTagOnSuccessfulInsertion() { when(mongoCollection.bulkWrite(request)).thenReturn(new BulkWriteResultMock(true, 3, 0, 0)); mongoSinkClient.processRequest(request); - verify(firehoseInstrumentation, times(3)).incrementCounter(SINK_MONGO_INSERTED_TOTAL); - verify(firehoseInstrumentation, times(3)).incrementCounter(SINK_MONGO_MODIFIED_TOTAL); + verify(firehoseInstrumentation, times(3)).incrementCounter(Metrics.SINK_MONGO_INSERTED_TOTAL); + verify(firehoseInstrumentation, times(3)).incrementCounter(Metrics.SINK_MONGO_MODIFIED_TOTAL); } @@ -310,8 +310,8 @@ public void shouldIncrementUpdatedCounterTagOnSuccessfulUpdation() { when(mongoCollection.bulkWrite(request)).thenReturn(new BulkWriteResultMock(true, 0, 3, 0)); mongoSinkClient.processRequest(request); - verify(firehoseInstrumentation, times(3)).incrementCounter(SINK_MONGO_UPDATED_TOTAL); - verify(firehoseInstrumentation, times(3)).incrementCounter(SINK_MONGO_MODIFIED_TOTAL); + verify(firehoseInstrumentation, times(3)).incrementCounter(Metrics.SINK_MONGO_UPDATED_TOTAL); + verify(firehoseInstrumentation, times(3)).incrementCounter(Metrics.SINK_MONGO_MODIFIED_TOTAL); } @Test @@ -324,8 +324,8 @@ public void shouldIncrementInsertedCounterTagOnSuccessfulInsertionInUpsertMode() when(mongoCollection.bulkWrite(request)).thenReturn(new BulkWriteResultMock(true, 0, 0, 3)); mongoSinkClient.processRequest(request); - verify(firehoseInstrumentation, times(3)).incrementCounter(SINK_MONGO_INSERTED_TOTAL); - verify(firehoseInstrumentation, times(3)).incrementCounter(SINK_MONGO_MODIFIED_TOTAL); + verify(firehoseInstrumentation, times(3)).incrementCounter(Metrics.SINK_MONGO_INSERTED_TOTAL); + verify(firehoseInstrumentation, times(3)).incrementCounter(Metrics.SINK_MONGO_MODIFIED_TOTAL); } diff --git a/src/test/java/io/odpf/firehose/sink/mongodb/client/MongoSinkClientUtilTest.java b/src/test/java/org/raystack/firehose/sink/mongodb/client/MongoSinkClientUtilTest.java similarity index 96% rename from src/test/java/io/odpf/firehose/sink/mongodb/client/MongoSinkClientUtilTest.java rename to src/test/java/org/raystack/firehose/sink/mongodb/client/MongoSinkClientUtilTest.java index 70e72dd01..6fd1844e2 100644 --- a/src/test/java/io/odpf/firehose/sink/mongodb/client/MongoSinkClientUtilTest.java +++ b/src/test/java/org/raystack/firehose/sink/mongodb/client/MongoSinkClientUtilTest.java @@ -1,8 +1,8 @@ -package io.odpf.firehose.sink.mongodb.client; +package org.raystack.firehose.sink.mongodb.client; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import com.mongodb.MongoClient; import com.mongodb.client.MongoDatabase; -import io.odpf.firehose.metrics.FirehoseInstrumentation; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/src/test/java/io/odpf/firehose/sink/mongodb/request/MongoRequestHandlerFactoryTest.java b/src/test/java/org/raystack/firehose/sink/mongodb/request/MongoRequestHandlerFactoryTest.java similarity index 93% rename from src/test/java/io/odpf/firehose/sink/mongodb/request/MongoRequestHandlerFactoryTest.java rename to src/test/java/org/raystack/firehose/sink/mongodb/request/MongoRequestHandlerFactoryTest.java index 894bc2d1b..afadf58a5 100644 --- a/src/test/java/io/odpf/firehose/sink/mongodb/request/MongoRequestHandlerFactoryTest.java +++ b/src/test/java/org/raystack/firehose/sink/mongodb/request/MongoRequestHandlerFactoryTest.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.sink.mongodb.request; +package org.raystack.firehose.sink.mongodb.request; -import io.odpf.firehose.config.MongoSinkConfig; -import io.odpf.firehose.config.enums.MongoSinkMessageType; -import io.odpf.firehose.config.enums.MongoSinkRequestType; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.serializer.MessageToJson; +import org.raystack.firehose.config.MongoSinkConfig; +import org.raystack.firehose.config.enums.MongoSinkMessageType; +import org.raystack.firehose.config.enums.MongoSinkRequestType; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.serializer.MessageToJson; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/mongodb/request/MongoUpdateRequestHandlerTest.java b/src/test/java/org/raystack/firehose/sink/mongodb/request/MongoUpdateRequestHandlerTest.java similarity index 94% rename from src/test/java/io/odpf/firehose/sink/mongodb/request/MongoUpdateRequestHandlerTest.java rename to src/test/java/org/raystack/firehose/sink/mongodb/request/MongoUpdateRequestHandlerTest.java index f77004ead..329a18005 100644 --- a/src/test/java/io/odpf/firehose/sink/mongodb/request/MongoUpdateRequestHandlerTest.java +++ b/src/test/java/org/raystack/firehose/sink/mongodb/request/MongoUpdateRequestHandlerTest.java @@ -1,17 +1,17 @@ -package io.odpf.firehose.sink.mongodb.request; +package org.raystack.firehose.sink.mongodb.request; -import io.odpf.stencil.StencilClientFactory; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.config.enums.MongoSinkMessageType; +import org.raystack.firehose.config.enums.MongoSinkRequestType; +import org.raystack.firehose.exception.JsonParseException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.serializer.MessageToJson; +import org.raystack.stencil.StencilClientFactory; +import org.raystack.stencil.client.StencilClient; import com.google.protobuf.InvalidProtocolBufferException; import com.mongodb.BasicDBObject; import com.mongodb.client.model.ReplaceOneModel; -import io.odpf.firehose.config.enums.MongoSinkMessageType; -import io.odpf.firehose.config.enums.MongoSinkRequestType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestAggregatedSupplyMessage; -import io.odpf.firehose.exception.JsonParseException; -import io.odpf.firehose.serializer.MessageToJson; +import org.raystack.firehose.consumer.TestAggregatedSupplyMessage; import org.bson.Document; import org.json.simple.JSONObject; import org.junit.Before; diff --git a/src/test/java/io/odpf/firehose/sink/mongodb/request/MongoUpsertRequestHandlerTest.java b/src/test/java/org/raystack/firehose/sink/mongodb/request/MongoUpsertRequestHandlerTest.java similarity index 95% rename from src/test/java/io/odpf/firehose/sink/mongodb/request/MongoUpsertRequestHandlerTest.java rename to src/test/java/org/raystack/firehose/sink/mongodb/request/MongoUpsertRequestHandlerTest.java index 13d9e9c86..f1e17a51b 100644 --- a/src/test/java/io/odpf/firehose/sink/mongodb/request/MongoUpsertRequestHandlerTest.java +++ b/src/test/java/org/raystack/firehose/sink/mongodb/request/MongoUpsertRequestHandlerTest.java @@ -1,17 +1,17 @@ -package io.odpf.firehose.sink.mongodb.request; - -import io.odpf.stencil.StencilClientFactory; -import io.odpf.stencil.client.StencilClient; +package org.raystack.firehose.sink.mongodb.request; + +import org.raystack.firehose.config.enums.MongoSinkMessageType; +import org.raystack.firehose.config.enums.MongoSinkRequestType; +import org.raystack.firehose.exception.JsonParseException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.serializer.MessageToJson; +import org.raystack.stencil.StencilClientFactory; +import org.raystack.stencil.client.StencilClient; import com.google.protobuf.InvalidProtocolBufferException; import com.mongodb.BasicDBObject; import com.mongodb.client.model.InsertOneModel; import com.mongodb.client.model.ReplaceOneModel; -import io.odpf.firehose.config.enums.MongoSinkMessageType; -import io.odpf.firehose.config.enums.MongoSinkRequestType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestAggregatedSupplyMessage; -import io.odpf.firehose.exception.JsonParseException; -import io.odpf.firehose.serializer.MessageToJson; +import org.raystack.firehose.consumer.TestAggregatedSupplyMessage; import org.bson.Document; import org.json.simple.JSONObject; import org.junit.Before; diff --git a/src/test/java/io/odpf/firehose/sink/mongodb/util/MongoSinkFactoryUtilTest.java b/src/test/java/org/raystack/firehose/sink/mongodb/util/MongoSinkFactoryUtilTest.java similarity index 97% rename from src/test/java/io/odpf/firehose/sink/mongodb/util/MongoSinkFactoryUtilTest.java rename to src/test/java/org/raystack/firehose/sink/mongodb/util/MongoSinkFactoryUtilTest.java index d3374bd1d..2867d5236 100644 --- a/src/test/java/io/odpf/firehose/sink/mongodb/util/MongoSinkFactoryUtilTest.java +++ b/src/test/java/org/raystack/firehose/sink/mongodb/util/MongoSinkFactoryUtilTest.java @@ -1,7 +1,7 @@ -package io.odpf.firehose.sink.mongodb.util; +package org.raystack.firehose.sink.mongodb.util; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import com.mongodb.ServerAddress; -import io.odpf.firehose.metrics.FirehoseInstrumentation; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/prometheus/PromSinkFactoryTest.java b/src/test/java/org/raystack/firehose/sink/prometheus/PromSinkFactoryTest.java similarity index 75% rename from src/test/java/io/odpf/firehose/sink/prometheus/PromSinkFactoryTest.java rename to src/test/java/org/raystack/firehose/sink/prometheus/PromSinkFactoryTest.java index 5be10e72a..7def05649 100644 --- a/src/test/java/io/odpf/firehose/sink/prometheus/PromSinkFactoryTest.java +++ b/src/test/java/org/raystack/firehose/sink/prometheus/PromSinkFactoryTest.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.sink.prometheus; +package org.raystack.firehose.sink.prometheus; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.sink.AbstractSink; -import io.odpf.stencil.client.StencilClient; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.sink.AbstractSink; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.stencil.client.StencilClient; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; diff --git a/src/test/java/io/odpf/firehose/sink/prometheus/PromSinkTest.java b/src/test/java/org/raystack/firehose/sink/prometheus/PromSinkTest.java similarity index 95% rename from src/test/java/io/odpf/firehose/sink/prometheus/PromSinkTest.java rename to src/test/java/org/raystack/firehose/sink/prometheus/PromSinkTest.java index 017d94dd2..07b2a2412 100644 --- a/src/test/java/io/odpf/firehose/sink/prometheus/PromSinkTest.java +++ b/src/test/java/org/raystack/firehose/sink/prometheus/PromSinkTest.java @@ -1,14 +1,14 @@ -package io.odpf.firehose.sink.prometheus; +package org.raystack.firehose.sink.prometheus; +import org.raystack.firehose.config.converter.RangeToHashMapConverter; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.exception.NeedToRetry; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.prometheus.request.PromRequest; import cortexpb.Cortex; -import io.odpf.firehose.config.converter.RangeToHashMapConverter; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.exception.NeedToRetry; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.prometheus.request.PromRequest; -import io.odpf.stencil.client.StencilClient; +import org.raystack.stencil.client.StencilClient; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -34,7 +34,6 @@ import java.util.List; import java.util.Map; -import static io.odpf.firehose.sink.prometheus.PromSinkConstants.PROMETHEUS_LABEL_FOR_METRIC_NAME; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.*; import static org.mockito.MockitoAnnotations.initMocks; @@ -72,7 +71,7 @@ public void setup() { httpPostList = Collections.singletonList(httpPost); Cortex.Sample sample = Cortex.Sample.newBuilder().setValue(10).setTimestampMs(1000000).build(); - Cortex.LabelPair labelPair = Cortex.LabelPair.newBuilder().setName(PROMETHEUS_LABEL_FOR_METRIC_NAME).setValue("test_metric").build(); + Cortex.LabelPair labelPair = Cortex.LabelPair.newBuilder().setName(PromSinkConstants.PROMETHEUS_LABEL_FOR_METRIC_NAME).setValue("test_metric").build(); Cortex.TimeSeries timeSeries = Cortex.TimeSeries.newBuilder() .addSamples(sample).addLabels(labelPair) .build(); diff --git a/src/test/java/io/odpf/firehose/sink/prometheus/builder/HeaderBuilderTest.java b/src/test/java/org/raystack/firehose/sink/prometheus/builder/HeaderBuilderTest.java similarity index 74% rename from src/test/java/io/odpf/firehose/sink/prometheus/builder/HeaderBuilderTest.java rename to src/test/java/org/raystack/firehose/sink/prometheus/builder/HeaderBuilderTest.java index 00d0b1f38..cf79505a3 100644 --- a/src/test/java/io/odpf/firehose/sink/prometheus/builder/HeaderBuilderTest.java +++ b/src/test/java/org/raystack/firehose/sink/prometheus/builder/HeaderBuilderTest.java @@ -1,11 +1,12 @@ -package io.odpf.firehose.sink.prometheus.builder; +package org.raystack.firehose.sink.prometheus.builder; +import org.raystack.firehose.sink.prometheus.PromSinkConstants; +import org.junit.Assert; import org.junit.Test; import java.util.HashMap; import java.util.Map; -import static io.odpf.firehose.sink.prometheus.PromSinkConstants.*; import static org.junit.Assert.assertEquals; public class HeaderBuilderTest { @@ -36,8 +37,8 @@ public void shouldParseWithRequiredPromHeadersInBetween() { { put("foo", "bar"); put("accept", "text/plain"); - put(PROMETHEUS_REMOTE_WRITE_VERSION, PROMETHEUS_REMOTE_WRITE_VERSION_DEFAULT); - put(CONTENT_ENCODING, CONTENT_ENCODING_DEFAULT); + put(PromSinkConstants.PROMETHEUS_REMOTE_WRITE_VERSION, PromSinkConstants.PROMETHEUS_REMOTE_WRITE_VERSION_DEFAULT); + put(PromSinkConstants.CONTENT_ENCODING, PromSinkConstants.CONTENT_ENCODING_DEFAULT); } }; assertEquals(expected, headerBuilder.build()); @@ -49,8 +50,8 @@ public void shouldNotThrowNullPointerExceptionWhenHeaderConfigEmpty() { HeaderBuilder headerBuilder = new HeaderBuilder(headerConfig); Map header = headerBuilder.build(); - assertEquals(PROMETHEUS_REMOTE_WRITE_VERSION_DEFAULT, header.get(PROMETHEUS_REMOTE_WRITE_VERSION)); - assertEquals(CONTENT_ENCODING_DEFAULT, header.get(CONTENT_ENCODING)); + Assert.assertEquals(PromSinkConstants.PROMETHEUS_REMOTE_WRITE_VERSION_DEFAULT, header.get(PromSinkConstants.PROMETHEUS_REMOTE_WRITE_VERSION)); + Assert.assertEquals(PromSinkConstants.CONTENT_ENCODING_DEFAULT, header.get(PromSinkConstants.CONTENT_ENCODING)); } @Test diff --git a/src/test/java/io/odpf/firehose/sink/prometheus/builder/RequestEntityBuilderTest.java b/src/test/java/org/raystack/firehose/sink/prometheus/builder/RequestEntityBuilderTest.java similarity index 86% rename from src/test/java/io/odpf/firehose/sink/prometheus/builder/RequestEntityBuilderTest.java rename to src/test/java/org/raystack/firehose/sink/prometheus/builder/RequestEntityBuilderTest.java index ceceed2bb..fb951ab82 100644 --- a/src/test/java/io/odpf/firehose/sink/prometheus/builder/RequestEntityBuilderTest.java +++ b/src/test/java/org/raystack/firehose/sink/prometheus/builder/RequestEntityBuilderTest.java @@ -1,6 +1,7 @@ -package io.odpf.firehose.sink.prometheus.builder; +package org.raystack.firehose.sink.prometheus.builder; import com.google.protobuf.DynamicMessage; +import org.raystack.firehose.sink.prometheus.PromSinkConstants; import cortexpb.Cortex; import org.apache.commons.io.IOUtils; import org.apache.http.entity.ByteArrayEntity; @@ -10,14 +11,12 @@ import java.io.IOException; -import static io.odpf.firehose.sink.prometheus.PromSinkConstants.PROMETHEUS_LABEL_FOR_METRIC_NAME; - public class RequestEntityBuilderTest { @Test public void shouldCreateSnappyCompressedByteArrayEntity() throws IOException { Cortex.Sample sample = Cortex.Sample.newBuilder().setValue(10).setTimestampMs(System.currentTimeMillis()).build(); - Cortex.LabelPair labelPair = Cortex.LabelPair.newBuilder().setName(PROMETHEUS_LABEL_FOR_METRIC_NAME).setValue("test_metric").build(); + Cortex.LabelPair labelPair = Cortex.LabelPair.newBuilder().setName(PromSinkConstants.PROMETHEUS_LABEL_FOR_METRIC_NAME).setValue("test_metric").build(); Cortex.TimeSeries timeSeries = Cortex.TimeSeries.newBuilder() .addSamples(sample).addLabels(labelPair) .build(); diff --git a/src/test/java/io/odpf/firehose/sink/prometheus/builder/TimeSeriesBuilderTest.java b/src/test/java/org/raystack/firehose/sink/prometheus/builder/TimeSeriesBuilderTest.java similarity index 98% rename from src/test/java/io/odpf/firehose/sink/prometheus/builder/TimeSeriesBuilderTest.java rename to src/test/java/org/raystack/firehose/sink/prometheus/builder/TimeSeriesBuilderTest.java index 19147e56b..04ce1723b 100644 --- a/src/test/java/io/odpf/firehose/sink/prometheus/builder/TimeSeriesBuilderTest.java +++ b/src/test/java/org/raystack/firehose/sink/prometheus/builder/TimeSeriesBuilderTest.java @@ -1,11 +1,11 @@ -package io.odpf.firehose.sink.prometheus.builder; - -import io.odpf.firehose.config.PromSinkConfig; -import io.odpf.firehose.consumer.TestBookingLogMessage; -import io.odpf.firehose.consumer.TestDurationMessage; -import io.odpf.firehose.consumer.TestFeedbackLogMessage; -import io.odpf.firehose.consumer.TestLocation; -import io.odpf.firehose.exception.ConfigurationException; +package org.raystack.firehose.sink.prometheus.builder; + +import org.raystack.firehose.config.PromSinkConfig; +import org.raystack.firehose.exception.ConfigurationException; +import org.raystack.firehose.consumer.TestBookingLogMessage; +import org.raystack.firehose.consumer.TestDurationMessage; +import org.raystack.firehose.consumer.TestFeedbackLogMessage; +import org.raystack.firehose.consumer.TestLocation; import com.google.protobuf.Duration; import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; @@ -239,7 +239,7 @@ public void testMessageWithEnum() throws InvalidProtocolBufferException { TestFeedbackLogMessage feedbackLogMessage = TestFeedbackLogMessage.newBuilder() .setTipAmount(10000) - .setFeedbackSource(io.odpf.firehose.consumer.TestFeedbackSource.Enum.DRIVER) + .setFeedbackSource(org.raystack.firehose.consumer.TestFeedbackSource.Enum.DRIVER) .setEventTimestamp(Timestamp.newBuilder().setSeconds(1000000)) .build(); DynamicMessage dynamicMessage = DynamicMessage.parseFrom(TestFeedbackLogMessage.getDescriptor(), feedbackLogMessage.toByteArray()); diff --git a/src/test/java/io/odpf/firehose/sink/prometheus/builder/TimeSeriesBuilderUtilsTest.java b/src/test/java/org/raystack/firehose/sink/prometheus/builder/TimeSeriesBuilderUtilsTest.java similarity index 96% rename from src/test/java/io/odpf/firehose/sink/prometheus/builder/TimeSeriesBuilderUtilsTest.java rename to src/test/java/org/raystack/firehose/sink/prometheus/builder/TimeSeriesBuilderUtilsTest.java index 5d13a9e2d..782c0f25f 100644 --- a/src/test/java/io/odpf/firehose/sink/prometheus/builder/TimeSeriesBuilderUtilsTest.java +++ b/src/test/java/org/raystack/firehose/sink/prometheus/builder/TimeSeriesBuilderUtilsTest.java @@ -1,10 +1,10 @@ -package io.odpf.firehose.sink.prometheus.builder; +package org.raystack.firehose.sink.prometheus.builder; import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Timestamp; -import io.odpf.firehose.consumer.TestFeedbackLogMessage; -import io.odpf.firehose.sink.prometheus.PromSinkConstants; +import org.raystack.firehose.sink.prometheus.PromSinkConstants; +import org.raystack.firehose.consumer.TestFeedbackLogMessage; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/prometheus/builder/WriteRequestBuilderTest.java b/src/test/java/org/raystack/firehose/sink/prometheus/builder/WriteRequestBuilderTest.java similarity index 89% rename from src/test/java/io/odpf/firehose/sink/prometheus/builder/WriteRequestBuilderTest.java rename to src/test/java/org/raystack/firehose/sink/prometheus/builder/WriteRequestBuilderTest.java index 2050f9e50..fcb5ba299 100644 --- a/src/test/java/io/odpf/firehose/sink/prometheus/builder/WriteRequestBuilderTest.java +++ b/src/test/java/org/raystack/firehose/sink/prometheus/builder/WriteRequestBuilderTest.java @@ -1,13 +1,14 @@ -package io.odpf.firehose.sink.prometheus.builder; +package org.raystack.firehose.sink.prometheus.builder; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.consumer.TestFeedbackLogMessage; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.sink.prometheus.PromSinkConstants; +import org.raystack.firehose.consumer.TestFeedbackLogMessage; import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Timestamp; import cortexpb.Cortex; -import io.odpf.stencil.Parser; +import org.raystack.stencil.Parser; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -19,8 +20,6 @@ import java.util.Collections; import java.util.List; -import static io.odpf.firehose.sink.prometheus.PromSinkConstants.KAFKA_PARTITION; -import static io.odpf.firehose.sink.prometheus.PromSinkConstants.PROMETHEUS_LABEL_FOR_METRIC_NAME; import static org.junit.Assert.*; import static org.mockito.MockitoAnnotations.initMocks; @@ -66,9 +65,9 @@ public void setUp() throws InvalidProtocolBufferException { dynamicMessage1 = DynamicMessage.parseFrom(TestFeedbackLogMessage.getDescriptor(), feedbackLogMessage1.toByteArray()); dynamicMessage2 = DynamicMessage.parseFrom(TestFeedbackLogMessage.getDescriptor(), feedbackLogMessage2.toByteArray()); - labelPair1 = Cortex.LabelPair.newBuilder().setName(PROMETHEUS_LABEL_FOR_METRIC_NAME).setValue("tip_amount").build(); - labelPair2 = Cortex.LabelPair.newBuilder().setName(KAFKA_PARTITION).setValue("0").build(); - labelPair3 = Cortex.LabelPair.newBuilder().setName(KAFKA_PARTITION).setValue("1").build(); + labelPair1 = Cortex.LabelPair.newBuilder().setName(PromSinkConstants.PROMETHEUS_LABEL_FOR_METRIC_NAME).setValue("tip_amount").build(); + labelPair2 = Cortex.LabelPair.newBuilder().setName(PromSinkConstants.KAFKA_PARTITION).setValue("0").build(); + labelPair3 = Cortex.LabelPair.newBuilder().setName(PromSinkConstants.KAFKA_PARTITION).setValue("1").build(); sample1 = Cortex.Sample.newBuilder().setTimestampMs(10000).setValue(100).build(); sample2 = Cortex.Sample.newBuilder().setTimestampMs(8000).setValue(200).build(); } diff --git a/src/test/java/io/odpf/firehose/sink/prometheus/request/PromRequestCreatorTest.java b/src/test/java/org/raystack/firehose/sink/prometheus/request/PromRequestCreatorTest.java similarity index 82% rename from src/test/java/io/odpf/firehose/sink/prometheus/request/PromRequestCreatorTest.java rename to src/test/java/org/raystack/firehose/sink/prometheus/request/PromRequestCreatorTest.java index d62752a72..a8820d3e7 100644 --- a/src/test/java/io/odpf/firehose/sink/prometheus/request/PromRequestCreatorTest.java +++ b/src/test/java/org/raystack/firehose/sink/prometheus/request/PromRequestCreatorTest.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.sink.prometheus.request; +package org.raystack.firehose.sink.prometheus.request; -import io.odpf.depot.metrics.StatsDReporter; -import io.odpf.firehose.config.PromSinkConfig; -import io.odpf.stencil.Parser; +import org.raystack.firehose.config.PromSinkConfig; +import org.raystack.depot.metrics.StatsDReporter; +import org.raystack.stencil.Parser; import org.aeonbits.owner.ConfigFactory; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sink/prometheus/request/PromRequestTest.java b/src/test/java/org/raystack/firehose/sink/prometheus/request/PromRequestTest.java similarity index 75% rename from src/test/java/io/odpf/firehose/sink/prometheus/request/PromRequestTest.java rename to src/test/java/org/raystack/firehose/sink/prometheus/request/PromRequestTest.java index ceef64e79..0bb702099 100644 --- a/src/test/java/io/odpf/firehose/sink/prometheus/request/PromRequestTest.java +++ b/src/test/java/org/raystack/firehose/sink/prometheus/request/PromRequestTest.java @@ -1,12 +1,13 @@ -package io.odpf.firehose.sink.prometheus.request; +package org.raystack.firehose.sink.prometheus.request; import com.google.protobuf.DynamicMessage; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.sink.prometheus.PromSinkConstants; +import org.raystack.firehose.sink.prometheus.builder.HeaderBuilder; +import org.raystack.firehose.sink.prometheus.builder.RequestEntityBuilder; +import org.raystack.firehose.sink.prometheus.builder.WriteRequestBuilder; import cortexpb.Cortex; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.sink.prometheus.builder.HeaderBuilder; -import io.odpf.firehose.sink.prometheus.builder.RequestEntityBuilder; -import io.odpf.firehose.sink.prometheus.builder.WriteRequestBuilder; import org.apache.commons.io.IOUtils; import org.apache.http.Header; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; @@ -25,7 +26,6 @@ import java.util.HashMap; import java.util.List; -import static io.odpf.firehose.sink.prometheus.PromSinkConstants.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.*; @@ -57,8 +57,8 @@ public void setUp() { @Test public void shouldProperlyBuildMessages() throws URISyntaxException, IOException { - Cortex.LabelPair labelPair1 = Cortex.LabelPair.newBuilder().setName(PROMETHEUS_LABEL_FOR_METRIC_NAME).setValue("tip_amount").build(); - Cortex.LabelPair labelPair2 = Cortex.LabelPair.newBuilder().setName(KAFKA_PARTITION).setValue("0").build(); + Cortex.LabelPair labelPair1 = Cortex.LabelPair.newBuilder().setName(PromSinkConstants.PROMETHEUS_LABEL_FOR_METRIC_NAME).setValue("tip_amount").build(); + Cortex.LabelPair labelPair2 = Cortex.LabelPair.newBuilder().setName(PromSinkConstants.KAFKA_PARTITION).setValue("0").build(); Cortex.Sample sample = Cortex.Sample.newBuilder().setTimestampMs(10000).setValue(100).build(); Cortex.TimeSeries timeSeries = Cortex.TimeSeries.newBuilder() @@ -71,8 +71,8 @@ public void shouldProperlyBuildMessages() throws URISyntaxException, IOException .build(); HashMap headerMap = new HashMap<>(); - headerMap.put(CONTENT_ENCODING, CONTENT_ENCODING_DEFAULT); - headerMap.put(PROMETHEUS_REMOTE_WRITE_VERSION, PROMETHEUS_REMOTE_WRITE_VERSION_DEFAULT); + headerMap.put(PromSinkConstants.CONTENT_ENCODING, PromSinkConstants.CONTENT_ENCODING_DEFAULT); + headerMap.put(PromSinkConstants.PROMETHEUS_REMOTE_WRITE_VERSION, PromSinkConstants.PROMETHEUS_REMOTE_WRITE_VERSION_DEFAULT); String url = "dummyEndpoint"; URI uri = new URI(url); @@ -84,8 +84,8 @@ public void shouldProperlyBuildMessages() throws URISyntaxException, IOException PromRequest promRequest = new PromRequest(firehoseInstrumentation, headerBuilder, url, requestEntityBuilder, writeRequestBuilder); HttpEntityEnclosingRequestBase request = promRequest.build(messages).get(0); - BasicHeader header1 = new BasicHeader(CONTENT_ENCODING, CONTENT_ENCODING_DEFAULT); - BasicHeader header2 = new BasicHeader(PROMETHEUS_REMOTE_WRITE_VERSION, PROMETHEUS_REMOTE_WRITE_VERSION_DEFAULT); + BasicHeader header1 = new BasicHeader(PromSinkConstants.CONTENT_ENCODING, PromSinkConstants.CONTENT_ENCODING_DEFAULT); + BasicHeader header2 = new BasicHeader(PromSinkConstants.PROMETHEUS_REMOTE_WRITE_VERSION, PromSinkConstants.PROMETHEUS_REMOTE_WRITE_VERSION_DEFAULT); Header[] headers = new Header[2]; headers[0] = header1; headers[1] = header2; diff --git a/src/test/java/io/odpf/firehose/sinkdecorator/ExponentialBackOffProviderTest.java b/src/test/java/org/raystack/firehose/sinkdecorator/ExponentialBackOffProviderTest.java similarity index 96% rename from src/test/java/io/odpf/firehose/sinkdecorator/ExponentialBackOffProviderTest.java rename to src/test/java/org/raystack/firehose/sinkdecorator/ExponentialBackOffProviderTest.java index 827e7b0c1..c52599a6e 100644 --- a/src/test/java/io/odpf/firehose/sinkdecorator/ExponentialBackOffProviderTest.java +++ b/src/test/java/org/raystack/firehose/sinkdecorator/ExponentialBackOffProviderTest.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.sinkdecorator; +package org.raystack.firehose.sinkdecorator; -import io.odpf.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.FirehoseInstrumentation; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sinkdecorator/SinkFinalTest.java b/src/test/java/org/raystack/firehose/sinkdecorator/SinkFinalTest.java similarity index 81% rename from src/test/java/io/odpf/firehose/sinkdecorator/SinkFinalTest.java rename to src/test/java/org/raystack/firehose/sinkdecorator/SinkFinalTest.java index bf1fcf152..bb7b2c472 100644 --- a/src/test/java/io/odpf/firehose/sinkdecorator/SinkFinalTest.java +++ b/src/test/java/org/raystack/firehose/sinkdecorator/SinkFinalTest.java @@ -1,9 +1,9 @@ -package io.odpf.firehose.sinkdecorator; +package org.raystack.firehose.sinkdecorator; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.metrics.Metrics; -import io.odpf.firehose.sink.Sink; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.sink.Sink; import org.junit.Test; import org.mockito.Mockito; diff --git a/src/test/java/io/odpf/firehose/sinkdecorator/SinkWithDlqTest.java b/src/test/java/org/raystack/firehose/sinkdecorator/SinkWithDlqTest.java similarity index 90% rename from src/test/java/io/odpf/firehose/sinkdecorator/SinkWithDlqTest.java rename to src/test/java/org/raystack/firehose/sinkdecorator/SinkWithDlqTest.java index 25acc9c17..09868e079 100644 --- a/src/test/java/io/odpf/firehose/sinkdecorator/SinkWithDlqTest.java +++ b/src/test/java/org/raystack/firehose/sinkdecorator/SinkWithDlqTest.java @@ -1,14 +1,14 @@ -package io.odpf.firehose.sinkdecorator; - -import io.odpf.depot.error.ErrorInfo; -import io.odpf.depot.error.ErrorType; -import io.odpf.firehose.config.DlqConfig; -import io.odpf.firehose.config.ErrorConfig; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.error.ErrorHandler; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.metrics.Metrics; -import io.odpf.firehose.sink.dlq.DlqWriter; +package org.raystack.firehose.sinkdecorator; + +import org.raystack.firehose.config.DlqConfig; +import org.raystack.firehose.config.ErrorConfig; +import org.raystack.firehose.error.ErrorHandler; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.sink.dlq.DlqWriter; +import org.raystack.depot.error.ErrorInfo; +import org.raystack.depot.error.ErrorType; import org.aeonbits.owner.ConfigFactory; import org.junit.Before; import org.junit.Test; @@ -22,8 +22,6 @@ import java.util.LinkedList; import java.util.List; -import static io.odpf.firehose.metrics.Metrics.DLQ_MESSAGES_TOTAL; -import static io.odpf.firehose.metrics.Metrics.DLQ_RETRY_ATTEMPTS_TOTAL; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.*; import static org.mockito.MockitoAnnotations.initMocks; @@ -74,9 +72,9 @@ public void shouldWriteToDLQWriter() throws Exception { List pushResult = sinkWithDlq.pushMessage(messages); verify(dlqWriter, times(1)).write(messages); assertEquals(0, pushResult.size()); - verify(firehoseInstrumentation, times(2)).captureMessageMetrics(DLQ_MESSAGES_TOTAL, Metrics.MessageType.TOTAL, ErrorType.DESERIALIZATION_ERROR, 1); - verify(firehoseInstrumentation, times(1)).captureMessageMetrics(DLQ_MESSAGES_TOTAL, Metrics.MessageType.SUCCESS, 2); - verify(firehoseInstrumentation, times(1)).incrementCounter(DLQ_RETRY_ATTEMPTS_TOTAL); + verify(firehoseInstrumentation, times(2)).captureMessageMetrics(Metrics.DLQ_MESSAGES_TOTAL, Metrics.MessageType.TOTAL, ErrorType.DESERIALIZATION_ERROR, 1); + verify(firehoseInstrumentation, times(1)).captureMessageMetrics(Metrics.DLQ_MESSAGES_TOTAL, Metrics.MessageType.SUCCESS, 2); + verify(firehoseInstrumentation, times(1)).incrementCounter(Metrics.DLQ_RETRY_ATTEMPTS_TOTAL); verify(firehoseInstrumentation, times(1)).captureGlobalMessageMetrics(Metrics.MessageScope.DLQ, 2); } @@ -127,9 +125,9 @@ public void shouldRetryWriteMessagesToDlqUntilRetryMessagesEmpty() throws IOExce verify(dlqWriter, times(1)).write(dlqRetryMessages); verify(firehoseInstrumentation, times(1)).captureDLQErrors(any(), any()); - verify(firehoseInstrumentation, times(2)).captureMessageMetrics(DLQ_MESSAGES_TOTAL, Metrics.MessageType.TOTAL, ErrorType.DESERIALIZATION_ERROR, 1); - verify(firehoseInstrumentation, times(1)).captureMessageMetrics(DLQ_MESSAGES_TOTAL, Metrics.MessageType.SUCCESS, 2); - verify(firehoseInstrumentation, times(2)).incrementCounter(DLQ_RETRY_ATTEMPTS_TOTAL); + verify(firehoseInstrumentation, times(2)).captureMessageMetrics(Metrics.DLQ_MESSAGES_TOTAL, Metrics.MessageType.TOTAL, ErrorType.DESERIALIZATION_ERROR, 1); + verify(firehoseInstrumentation, times(1)).captureMessageMetrics(Metrics.DLQ_MESSAGES_TOTAL, Metrics.MessageType.SUCCESS, 2); + verify(firehoseInstrumentation, times(2)).incrementCounter(Metrics.DLQ_RETRY_ATTEMPTS_TOTAL); verify(firehoseInstrumentation, times(1)).captureGlobalMessageMetrics(Metrics.MessageScope.DLQ, 2); } @@ -251,10 +249,10 @@ public void shouldInstrumentFailure() throws Exception { List pushResult = sinkWithDlq.pushMessage(messages); verify(dlqWriter, times(10)).write(messages); assertEquals(2, pushResult.size()); - verify(firehoseInstrumentation, times(2)).captureMessageMetrics(DLQ_MESSAGES_TOTAL, Metrics.MessageType.TOTAL, ErrorType.DESERIALIZATION_ERROR, 1); - verify(firehoseInstrumentation, times(1)).captureMessageMetrics(DLQ_MESSAGES_TOTAL, Metrics.MessageType.SUCCESS, 0); - verify(firehoseInstrumentation, times(2)).captureMessageMetrics(DLQ_MESSAGES_TOTAL, Metrics.MessageType.FAILURE, ErrorType.DESERIALIZATION_ERROR, 1); - verify(firehoseInstrumentation, times(10)).incrementCounter(DLQ_RETRY_ATTEMPTS_TOTAL); + verify(firehoseInstrumentation, times(2)).captureMessageMetrics(Metrics.DLQ_MESSAGES_TOTAL, Metrics.MessageType.TOTAL, ErrorType.DESERIALIZATION_ERROR, 1); + verify(firehoseInstrumentation, times(1)).captureMessageMetrics(Metrics.DLQ_MESSAGES_TOTAL, Metrics.MessageType.SUCCESS, 0); + verify(firehoseInstrumentation, times(2)).captureMessageMetrics(Metrics.DLQ_MESSAGES_TOTAL, Metrics.MessageType.FAILURE, ErrorType.DESERIALIZATION_ERROR, 1); + verify(firehoseInstrumentation, times(10)).incrementCounter(Metrics.DLQ_RETRY_ATTEMPTS_TOTAL); verify(firehoseInstrumentation, times(1)).captureGlobalMessageMetrics(Metrics.MessageScope.DLQ, 0); } } diff --git a/src/test/java/io/odpf/firehose/sinkdecorator/SinkWithFailHandlerTest.java b/src/test/java/org/raystack/firehose/sinkdecorator/SinkWithFailHandlerTest.java similarity index 85% rename from src/test/java/io/odpf/firehose/sinkdecorator/SinkWithFailHandlerTest.java rename to src/test/java/org/raystack/firehose/sinkdecorator/SinkWithFailHandlerTest.java index 6d1198c1b..559920f1f 100644 --- a/src/test/java/io/odpf/firehose/sinkdecorator/SinkWithFailHandlerTest.java +++ b/src/test/java/org/raystack/firehose/sinkdecorator/SinkWithFailHandlerTest.java @@ -1,12 +1,12 @@ -package io.odpf.firehose.sinkdecorator; +package org.raystack.firehose.sinkdecorator; -import io.odpf.depot.error.ErrorInfo; -import io.odpf.depot.error.ErrorType; -import io.odpf.firehose.config.ErrorConfig; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.error.ErrorHandler; -import io.odpf.firehose.exception.SinkException; -import io.odpf.firehose.sink.Sink; +import org.raystack.firehose.config.ErrorConfig; +import org.raystack.firehose.error.ErrorHandler; +import org.raystack.firehose.exception.SinkException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.sink.Sink; +import org.raystack.depot.error.ErrorInfo; +import org.raystack.depot.error.ErrorType; import org.aeonbits.owner.ConfigFactory; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/io/odpf/firehose/sinkdecorator/SinkWithRetryTest.java b/src/test/java/org/raystack/firehose/sinkdecorator/SinkWithRetryTest.java similarity index 92% rename from src/test/java/io/odpf/firehose/sinkdecorator/SinkWithRetryTest.java rename to src/test/java/org/raystack/firehose/sinkdecorator/SinkWithRetryTest.java index f36f79999..8a31f9472 100644 --- a/src/test/java/io/odpf/firehose/sinkdecorator/SinkWithRetryTest.java +++ b/src/test/java/org/raystack/firehose/sinkdecorator/SinkWithRetryTest.java @@ -1,16 +1,16 @@ -package io.odpf.firehose.sinkdecorator; - -import io.odpf.depot.error.ErrorInfo; -import io.odpf.depot.error.ErrorType; -import io.odpf.firehose.config.AppConfig; -import io.odpf.firehose.config.ErrorConfig; -import io.odpf.firehose.config.enums.InputSchemaType; -import io.odpf.firehose.message.Message; -import io.odpf.firehose.error.ErrorHandler; -import io.odpf.firehose.exception.DeserializerException; -import io.odpf.firehose.metrics.FirehoseInstrumentation; -import io.odpf.firehose.metrics.Metrics; -import io.odpf.firehose.sink.common.KeyOrMessageParser; +package org.raystack.firehose.sinkdecorator; + +import org.raystack.firehose.config.AppConfig; +import org.raystack.firehose.config.ErrorConfig; +import org.raystack.firehose.config.enums.InputSchemaType; +import org.raystack.firehose.error.ErrorHandler; +import org.raystack.firehose.exception.DeserializerException; +import org.raystack.firehose.message.Message; +import org.raystack.firehose.metrics.FirehoseInstrumentation; +import org.raystack.firehose.metrics.Metrics; +import org.raystack.firehose.sink.common.KeyOrMessageParser; +import org.raystack.depot.error.ErrorInfo; +import org.raystack.depot.error.ErrorType; import org.aeonbits.owner.ConfigFactory; import org.junit.Before; import org.junit.Test; @@ -26,8 +26,6 @@ import java.util.LinkedList; import java.util.List; -import static io.odpf.firehose.metrics.Metrics.RETRY_MESSAGES_TOTAL; -import static io.odpf.firehose.metrics.Metrics.RETRY_ATTEMPTS_TOTAL; import static org.junit.Assert.*; import static org.mockito.Mockito.*; import static org.mockito.MockitoAnnotations.initMocks; @@ -191,9 +189,9 @@ public void shouldAddInstrumentationForRetry() throws Exception { List messageList = sinkWithRetry.pushMessage(Collections.singletonList(message)); assertTrue(messageList.isEmpty()); verify(firehoseInstrumentation, times(1)).logInfo("Maximum retry attempts: {}", 3); - verify(firehoseInstrumentation, times(3)).captureMessageMetrics(RETRY_MESSAGES_TOTAL, Metrics.MessageType.TOTAL, ErrorType.DESERIALIZATION_ERROR, 1); - verify(firehoseInstrumentation, times(2)).incrementCounter(RETRY_ATTEMPTS_TOTAL); - verify(firehoseInstrumentation, times(1)).captureMessageMetrics(RETRY_MESSAGES_TOTAL, Metrics.MessageType.SUCCESS, 3); + verify(firehoseInstrumentation, times(3)).captureMessageMetrics(Metrics.RETRY_MESSAGES_TOTAL, Metrics.MessageType.TOTAL, ErrorType.DESERIALIZATION_ERROR, 1); + verify(firehoseInstrumentation, times(2)).incrementCounter(Metrics.RETRY_ATTEMPTS_TOTAL); + verify(firehoseInstrumentation, times(1)).captureMessageMetrics(Metrics.RETRY_MESSAGES_TOTAL, Metrics.MessageType.SUCCESS, 3); } @Test @@ -209,10 +207,10 @@ public void shouldAddInstrumentationForRetryFailures() throws Exception { List messageList = sinkWithRetry.pushMessage(Collections.singletonList(message)); assertFalse(messageList.isEmpty()); verify(firehoseInstrumentation, times(1)).logInfo("Maximum retry attempts: {}", 1); - verify(firehoseInstrumentation, times(3)).captureMessageMetrics(RETRY_MESSAGES_TOTAL, Metrics.MessageType.TOTAL, ErrorType.DESERIALIZATION_ERROR, 1); - verify(firehoseInstrumentation, times(1)).incrementCounter(RETRY_ATTEMPTS_TOTAL); - verify(firehoseInstrumentation, times(1)).captureMessageMetrics(RETRY_MESSAGES_TOTAL, Metrics.MessageType.SUCCESS, 0); - verify(firehoseInstrumentation, times(3)).captureMessageMetrics(RETRY_MESSAGES_TOTAL, Metrics.MessageType.FAILURE, ErrorType.DESERIALIZATION_ERROR, 1); + verify(firehoseInstrumentation, times(3)).captureMessageMetrics(Metrics.RETRY_MESSAGES_TOTAL, Metrics.MessageType.TOTAL, ErrorType.DESERIALIZATION_ERROR, 1); + verify(firehoseInstrumentation, times(1)).incrementCounter(Metrics.RETRY_ATTEMPTS_TOTAL); + verify(firehoseInstrumentation, times(1)).captureMessageMetrics(Metrics.RETRY_MESSAGES_TOTAL, Metrics.MessageType.SUCCESS, 0); + verify(firehoseInstrumentation, times(3)).captureMessageMetrics(Metrics.RETRY_MESSAGES_TOTAL, Metrics.MessageType.FAILURE, ErrorType.DESERIALIZATION_ERROR, 1); } @Test(expected = IOException.class) diff --git a/src/test/java/org/raystack/firehose/test/categories/IntegrationTest.java b/src/test/java/org/raystack/firehose/test/categories/IntegrationTest.java new file mode 100644 index 000000000..175ee1b49 --- /dev/null +++ b/src/test/java/org/raystack/firehose/test/categories/IntegrationTest.java @@ -0,0 +1,4 @@ +package org.raystack.firehose.test.categories; + +public interface IntegrationTest { +} diff --git a/src/test/java/io/odpf/firehose/tracer/SinkTracerTest.java b/src/test/java/org/raystack/firehose/tracer/SinkTracerTest.java similarity index 95% rename from src/test/java/io/odpf/firehose/tracer/SinkTracerTest.java rename to src/test/java/org/raystack/firehose/tracer/SinkTracerTest.java index 0b46a2368..d554c9e82 100644 --- a/src/test/java/io/odpf/firehose/tracer/SinkTracerTest.java +++ b/src/test/java/org/raystack/firehose/tracer/SinkTracerTest.java @@ -1,6 +1,6 @@ -package io.odpf.firehose.tracer; +package org.raystack.firehose.tracer; -import io.odpf.firehose.message.Message; +import org.raystack.firehose.message.Message; import io.opentracing.Span; import io.opentracing.mock.MockSpan; import io.opentracing.mock.MockTracer; diff --git a/src/test/proto/TestGrpc.proto b/src/test/proto/TestGrpc.proto index 5c70a7653..0e80918c5 100644 --- a/src/test/proto/TestGrpc.proto +++ b/src/test/proto/TestGrpc.proto @@ -1,9 +1,9 @@ syntax = "proto3"; -package io.odpf.firehose.consumer; +package org.raystack.firehose.consumer; option java_multiple_files = true; -option java_package = "io.odpf.firehose.consumer"; +option java_package = "org.raystack.firehose.consumer"; option java_outer_classname = "SampleGrpcServerProto"; service TestServer { diff --git a/src/test/proto/TestLogMessage.proto b/src/test/proto/TestLogMessage.proto index 9d7a2cb68..eade7d56f 100644 --- a/src/test/proto/TestLogMessage.proto +++ b/src/test/proto/TestLogMessage.proto @@ -1,9 +1,9 @@ syntax = "proto3"; -package io.odpf.firehose.consumer; +package org.raystack.firehose.consumer; option java_multiple_files = true; -option java_package = "io.odpf.firehose.consumer"; +option java_package = "org.raystack.firehose.consumer"; option java_outer_classname = "TestLogMessageProto"; import "google/protobuf/timestamp.proto"; diff --git a/src/test/proto/TestMessage.proto b/src/test/proto/TestMessage.proto index bcb7e7f3f..77ab3f0a5 100644 --- a/src/test/proto/TestMessage.proto +++ b/src/test/proto/TestMessage.proto @@ -1,9 +1,9 @@ syntax = "proto3"; -package io.odpf.firehose.consumer; +package org.raystack.firehose.consumer; option java_multiple_files = true; -option java_package = "io.odpf.firehose.consumer"; +option java_package = "org.raystack.firehose.consumer"; option java_outer_classname = "TestMessageProto"; import "google/protobuf/timestamp.proto"; diff --git a/src/test/proto/TestMessageBQ.proto b/src/test/proto/TestMessageBQ.proto index a9bf50f61..c213128a3 100644 --- a/src/test/proto/TestMessageBQ.proto +++ b/src/test/proto/TestMessageBQ.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package odpf.firehose; +package raystack.firehose; import "google/protobuf/timestamp.proto"; import "google/protobuf/duration.proto"; @@ -8,7 +8,7 @@ import "google/protobuf/struct.proto"; import "google/type/date.proto"; option java_multiple_files = true; -option java_package = "io.odpf.firehose"; +option java_package = "org.raystack.firehose"; option java_outer_classname = "TestMessageProtoBQ"; message TestKeyBQ { @@ -21,7 +21,7 @@ message TestMessageBQ { string order_url = 2; string order_details = 3; google.protobuf.Timestamp created_at = 4; - odpf.firehose.StatusBQ status = 5; + raystack.firehose.StatusBQ status = 5; int64 discount = 6; bool success = 7; float price = 8;