From aed037e00078bddbfa44103f082d31d0526577b1 Mon Sep 17 00:00:00 2001 From: Fermin Galan Marquez Date: Tue, 24 Mar 2015 12:45:07 +0100 Subject: [PATCH] MOVE contents under flume/ intro root directory (removing empty flume/ after that) --- ...ANGES_NEXT_RELEASE => CHANGES_NEXT_RELEASE | 0 README.md | 568 +++++++++++++++++- {flume/conf => conf}/README.md | 0 {flume/conf => conf}/agent.conf.template | 0 .../cygnus_instance.conf.template | 0 {flume/conf => conf}/krb5.conf.template | 0 {flume/conf => conf}/krb5_login.conf | 0 .../conf => conf}/log4j.properties.template | 0 .../matching_table.conf.template | 0 {flume/doc => doc}/design/OrionCKANSink.md | 0 {flume/doc => doc}/design/OrionCKANwqSink.md | 0 {flume/doc => doc}/design/OrionHDFSSink.md | 0 {flume/doc => doc}/design/OrionMySQLSink.md | 0 {flume/doc => doc}/design/OrionRestHander.md | 0 {flume/doc => doc}/design/OrionTestSink.md | 0 {flume/doc => doc}/design/architecture.md | 0 {flume/doc => doc}/design/ha.md | 0 {flume/doc => doc}/design/interceptors.md | 0 .../doc => doc}/design/naming_conventions.md | 0 {flume/doc => doc}/devel/add_new_sink.md | 0 .../doc => doc}/images/basic_architecture.jpg | Bin .../doc => doc}/images/basic_architecture.vsd | Bin {flume/doc => doc}/images/ha_architecture.jpg | Bin {flume/doc => doc}/images/ha_architecture.vsd | Bin {flume/doc => doc}/images/kerberos_auth.jpg | Bin {flume/doc => doc}/images/kerberos_auth.vsd | Bin ...e_sinks_multiple_channels_architecture.jpg | Bin ...e_sinks_multiple_channels_architecture.vsd | Bin ...iple_sinks_single_channel_architecture.jpg | Bin ...iple_sinks_single_channel_architecture.vsd | Bin .../images/sequence_diagram_ckan.jpg | Bin .../images/sequence_diagram_ckan.vsd | Bin .../images/sequence_diagram_hdfs.jpg | Bin .../images/sequence_diagram_hdfs.vsd | Bin .../images/sequence_diagram_mysql.jpg | Bin .../images/sequence_diagram_mysql.vsd | Bin ..._diagram_notification_handling_default.jpg | Bin ..._diagram_notification_handling_default.vsd | Bin .../doc => doc}/installation/chef_install.md | 0 .../doc => doc}/installation/rpm_install.md | 0 .../doc => doc}/installation/src_install.md | 0 {flume/doc => doc}/operation/alarms.md | 0 .../operation/cygnus_0.1_to_0.2_migration.md | 0 ...cygnus_0.1_to_0.3_(or_higher)_migration.md | 0 .../operation/hdfs_kerberos_authentication.md | 0 .../operation/ngsi2cosmos_migration.md | 0 .../operation/performance_tuning_tips.md | 0 {flume/doc => doc}/quick_start_guide.md | 0 flume/README.md | 565 ----------------- {flume/neore => neore}/README.md | 0 .../cron.d/cleanup_old_cygnus_logfiles | 0 .../neore => neore}/rpm/SOURCES/init.d/cygnus | 0 .../logrotate.d/logrotate-cygnus-daily | 0 {flume/neore => neore}/rpm/SPECS/cygnus.spec | 0 .../neore => neore}/rpm/get_version_string.sh | 0 {flume/neore => neore}/scripts/check_rpm.sh | 0 .../neore => neore}/scripts/colors_shell.sh | 0 {flume/neore => neore}/scripts/package.sh | 0 flume/pom.xml => pom.xml | 0 .../cygnus/backends/ckan/CKANBackend.java | 0 .../cygnus/backends/ckan/CKANBackendImpl.java | 0 .../cygnus/backends/ckan/CKANCache.java | 0 .../cygnus/backends/ckan/CKANRequester.java | 0 .../cygnus/backends/ckan/CKANResponse.java | 0 .../cygnus/backends/hdfs/HDFSBackend.java | 0 .../cygnus/backends/hdfs/HDFSBackendImpl.java | 0 .../cygnus/backends/hive/HiveBackend.java | 0 .../cygnus/backends/mysql/MySQLBackend.java | 0 .../cygnus/channels/CygnusChannel.java | 0 .../cygnus/channels/CygnusFileChannel.java | 0 .../cygnus/channels/CygnusMemoryChannel.java | 0 .../RoundRobinChannelSelector.java | 0 .../containers/NotifyContextRequest.java | 0 .../NotifyContextRequestSAXHandler.java | 0 .../cygnus/errors/CygnusBadConfiguration.java | 0 .../cygnus/errors/CygnusBadContextData.java | 0 .../cygnus/errors/CygnusPersistenceError.java | 0 .../cygnus/errors/CygnusRuntimeError.java | 0 .../cygnus/handlers/OrionRestHandler.java | 0 .../cygnus/http/HttpClientFactory.java | 0 .../cygnus/http/JettyServer.java | 0 .../interceptors/DestinationExtractor.java | 0 .../cygnus/log/CygnusLogger.java | 0 .../management/ManagementInterface.java | 0 .../cygnus/nodes/CygnusApplication.java | 0 .../cygnus/sinks/OrionCKANSink.java | 0 .../cygnus/sinks/OrionHDFSSink.java | 0 .../cygnus/sinks/OrionMySQLSink.java | 0 .../cygnus/sinks/OrionSink.java | 0 .../cygnus/sinks/OrionTestSink.java | 0 .../cygnus/utils/Constants.java | 0 .../fiwareconnectors/cygnus/utils/Utils.java | 0 .../main/resources/cygnus-flume-ng | 0 .../src => src}/main/resources/pom.properties | 0 .../backends/ckan/CKANBackendImplTest.java | 0 .../cygnus/backends/ckan/CKANCacheTest.java | 0 .../backends/ckan/CKANRequesterTest.java | 0 .../backends/ckan/CKANResponseTest.java | 0 .../backends/hdfs/HDFSBackendImplTest.java | 0 .../backends/mysql/MySQLBackendTest.java | 0 .../RoundRobinChannelSelectorTest.java | 0 .../containers/NotifyContextRequestTest.java | 0 .../cygnus/handlers/OrionRestHandlerTest.java | 0 .../cygnus/http/JettyServerTest.java | 0 .../DestinationExtractorTest.java | 0 .../management/ManagementInterfaceTest.java | 0 .../cygnus/sinks/OrionCKANSinkTest.java | 0 .../cygnus/sinks/OrionHDFSSinkTest.java | 0 .../cygnus/sinks/OrionMySQLSinkTest.java | 0 .../cygnus/utils/TestConstants.java | 0 .../cygnus/utils/TestUtils.java | 0 ...heckstyle.xml => telefonica_checkstyle.xml | 0 112 files changed, 559 insertions(+), 574 deletions(-) rename flume/CHANGES_NEXT_RELEASE => CHANGES_NEXT_RELEASE (100%) rename {flume/conf => conf}/README.md (100%) rename {flume/conf => conf}/agent.conf.template (100%) rename {flume/conf => conf}/cygnus_instance.conf.template (100%) rename {flume/conf => conf}/krb5.conf.template (100%) rename {flume/conf => conf}/krb5_login.conf (100%) rename {flume/conf => conf}/log4j.properties.template (100%) rename {flume/conf => conf}/matching_table.conf.template (100%) rename {flume/doc => doc}/design/OrionCKANSink.md (100%) rename {flume/doc => doc}/design/OrionCKANwqSink.md (100%) rename {flume/doc => doc}/design/OrionHDFSSink.md (100%) rename {flume/doc => doc}/design/OrionMySQLSink.md (100%) rename {flume/doc => doc}/design/OrionRestHander.md (100%) rename {flume/doc => doc}/design/OrionTestSink.md (100%) rename {flume/doc => doc}/design/architecture.md (100%) rename {flume/doc => doc}/design/ha.md (100%) rename {flume/doc => doc}/design/interceptors.md (100%) rename {flume/doc => doc}/design/naming_conventions.md (100%) rename {flume/doc => doc}/devel/add_new_sink.md (100%) rename {flume/doc => doc}/images/basic_architecture.jpg (100%) rename {flume/doc => doc}/images/basic_architecture.vsd (100%) rename {flume/doc => doc}/images/ha_architecture.jpg (100%) rename {flume/doc => doc}/images/ha_architecture.vsd (100%) rename {flume/doc => doc}/images/kerberos_auth.jpg (100%) rename {flume/doc => doc}/images/kerberos_auth.vsd (100%) rename {flume/doc => doc}/images/multiple_sinks_multiple_channels_architecture.jpg (100%) rename {flume/doc => doc}/images/multiple_sinks_multiple_channels_architecture.vsd (100%) rename {flume/doc => doc}/images/multiple_sinks_single_channel_architecture.jpg (100%) rename {flume/doc => doc}/images/multiple_sinks_single_channel_architecture.vsd (100%) rename {flume/doc => doc}/images/sequence_diagram_ckan.jpg (100%) rename {flume/doc => doc}/images/sequence_diagram_ckan.vsd (100%) rename {flume/doc => doc}/images/sequence_diagram_hdfs.jpg (100%) rename {flume/doc => doc}/images/sequence_diagram_hdfs.vsd (100%) rename {flume/doc => doc}/images/sequence_diagram_mysql.jpg (100%) rename {flume/doc => doc}/images/sequence_diagram_mysql.vsd (100%) rename {flume/doc => doc}/images/sequence_diagram_notification_handling_default.jpg (100%) rename {flume/doc => doc}/images/sequence_diagram_notification_handling_default.vsd (100%) rename {flume/doc => doc}/installation/chef_install.md (100%) rename {flume/doc => doc}/installation/rpm_install.md (100%) rename {flume/doc => doc}/installation/src_install.md (100%) rename {flume/doc => doc}/operation/alarms.md (100%) rename {flume/doc => doc}/operation/cygnus_0.1_to_0.2_migration.md (100%) rename {flume/doc => doc}/operation/cygnus_0.1_to_0.3_(or_higher)_migration.md (100%) rename {flume/doc => doc}/operation/hdfs_kerberos_authentication.md (100%) rename {flume/doc => doc}/operation/ngsi2cosmos_migration.md (100%) rename {flume/doc => doc}/operation/performance_tuning_tips.md (100%) rename {flume/doc => doc}/quick_start_guide.md (100%) delete mode 100644 flume/README.md rename {flume/neore => neore}/README.md (100%) rename {flume/neore => neore}/rpm/SOURCES/cron.d/cleanup_old_cygnus_logfiles (100%) rename {flume/neore => neore}/rpm/SOURCES/init.d/cygnus (100%) rename {flume/neore => neore}/rpm/SOURCES/logrotate.d/logrotate-cygnus-daily (100%) rename {flume/neore => neore}/rpm/SPECS/cygnus.spec (100%) rename {flume/neore => neore}/rpm/get_version_string.sh (100%) rename {flume/neore => neore}/scripts/check_rpm.sh (100%) rename {flume/neore => neore}/scripts/colors_shell.sh (100%) rename {flume/neore => neore}/scripts/package.sh (100%) rename flume/pom.xml => pom.xml (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANBackend.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANBackendImpl.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANCache.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANRequester.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANResponse.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hdfs/HDFSBackend.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hdfs/HDFSBackendImpl.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hive/HiveBackend.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/mysql/MySQLBackend.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/channels/CygnusChannel.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/channels/CygnusFileChannel.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/channels/CygnusMemoryChannel.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/channelselectors/RoundRobinChannelSelector.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/containers/NotifyContextRequest.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/containers/NotifyContextRequestSAXHandler.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusBadConfiguration.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusBadContextData.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusPersistenceError.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusRuntimeError.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/handlers/OrionRestHandler.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/http/HttpClientFactory.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/http/JettyServer.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/interceptors/DestinationExtractor.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/log/CygnusLogger.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/management/ManagementInterface.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/nodes/CygnusApplication.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionCKANSink.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionHDFSSink.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionMySQLSink.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionSink.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionTestSink.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/utils/Constants.java (100%) rename {flume/src => src}/main/java/es/tid/fiware/fiwareconnectors/cygnus/utils/Utils.java (100%) rename {flume/src => src}/main/resources/cygnus-flume-ng (100%) rename {flume/src => src}/main/resources/pom.properties (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANBackendImplTest.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANCacheTest.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANRequesterTest.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANResponseTest.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hdfs/HDFSBackendImplTest.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/mysql/MySQLBackendTest.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/channelselectors/RoundRobinChannelSelectorTest.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/containers/NotifyContextRequestTest.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/handlers/OrionRestHandlerTest.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/http/JettyServerTest.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/interceptors/DestinationExtractorTest.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/management/ManagementInterfaceTest.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionCKANSinkTest.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionHDFSSinkTest.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionMySQLSinkTest.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/utils/TestConstants.java (100%) rename {flume/src => src}/test/java/es/tid/fiware/fiwareconnectors/cygnus/utils/TestUtils.java (100%) rename flume/telefonica_checkstyle.xml => telefonica_checkstyle.xml (100%) diff --git a/flume/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE similarity index 100% rename from flume/CHANGES_NEXT_RELEASE rename to CHANGES_NEXT_RELEASE diff --git a/README.md b/README.md index 5405f72d7..f9affab40 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,565 @@ -# FI-WARE Connectors +# Cygnus connector -The FI-WARE Connectors repository is meant to include all the pieces of software connecting FI-WARE GEs among them or with other external actors (users, legacy software, etc.). +This connector is a (conceptual) derivative work of [ngsi2cosmos](https://github.com/telefonicaid/fiware-livedemoapp/tree/master/package/ngsi2cosmos), and implements a Flume-based connector for context data coming from Orion Context Broker and aimed to be stored in a specific persistent storage, such as HDFS, CKAN or MySQL. -## Structure of the repository +## Design -This repository is structured as follows: -* ```flume```, a.k.a. Cygnus, contains all the necessary for connecting Orion Context Broker with Cosmos Big Data through Flume. -* ```sftp``` contains the code for a SFTP-based server which directly talks with HDFS. -* ```resources``` contains additional miscellaneous pieces of code such as basic client templates, example configuration files about any connector, running scripts, etc. +All the details about Flume can be found at [flume.apache.org](http://flume.apache.org/), but, as a reminder, some concepts will be explained here: + +* A Flume source is an agent gathering event data from the real source (Twitter stream, a notification system, etc.), either by polling the source or listening for incoming pushes of data. Gathered data is sent to a Flume channel. +* A Flume channel is a passive store (implemented by means of a file, memory, etc.) that holds the event until it is consumed by the Flume sink. +* A Flume sink connects with the final destination of the data (a local file, HDFS, a database, etc.), taking events from the channel and consuming them (processing and/or persisting it). + +There exists a wide collection of already developed sources, channels and sinks. The Flume-based connector, also called Cygnus, development extends that collection by adding: + +* **OrionRestHandler**. A custom HTTP source handler for the default HTTP source. The existing HTTP source behaviour can be governed depending on the request handler associated to it in the configuration. In this case, the custom handler takes care of the method, the target and the headers (specially the Content-Type one) within the request, cheking everything is according to the expected [request format](https://forge.fi-ware.org/plugins/mediawiki/wiki/fiware/index.php/Publish/Subscribe_Broker_-_Orion_Context_Broker_-_User_and_Programmers_Guide#ONCHANGE). This allows for a certain degree of control on the incoming data. The header inspection step allows for a content type identification as well by sending, together with the data, the Content-Type header. +* **OrionHDFSSink**. A custom sink that persists Orion content data in a HDFS deployment. There already exists a native Flume HDFS sink persisting each event in a new file, but this is not suitable for Cygnus. Within Cygnus, the data coming from Orion must be persisted in the Cosmos HDFS in the form of files (a file per entity) containing Json-like lines about the values such entity's attributes have had along time. Several HDFS backends can be used for the data persistence (WebHDFS, HttpFS, Infinity), all of them based on the native WebHDFS REST API from Hadoop. +* **OrionCKANSink**. A custom sink that persists Orion context data in CKAN server instances (see http://docs.ckan.org/en/latest/). +* **OrionMySQLSink**. A custom sink for persisting Orion context data in a MySQL server. Each user owns a database, and each entity is mapped to a table within that database. Tables contain rows about the values such entity's attributes have had along time. + +All these new components (OrionRestHandler, OrionHDFSSink, etc) are combined with other native ones included in Flume itself (e.g. HttpSource), with the purpose of implementing the following data flow: + +1. On behalf of Cygnus, subscribe to Orion for certain context information. +2. Receive from Orion notifications about new update context data; this notification will be handled by the native HttpSource together with the custom OrionRestHandler. +3. Translate the notification into the Flume event format, and put them into the different sink channels (native memory ones). +4. For each enabled custom sink (OrionHDFSSink, OrionCKANSink, OrionMySQLSink), get the notifications from the sink channel and persist the data in the appropriate format. + +## Functionality explained (Json notification example) + +Let's consider the following notification in Json format coming from an Orion Context Broker instance: + + POST http://localhost:1028/notify + Content-Length: 492 + User-Agent: orion/0.9.0 + Host: localhost:1028 + Accept: application/xml, application/json + Content-Type: application/json + Fiware-Service: my-company-name + Fiware-ServicePath: /workingrooms/floor4 + + { + "subscriptionId" : "51c0ac9ed714fb3b37d7d5a8", + "originator" : "localhost", + "contextResponses" : [ + { + "contextElement" : { + "attributes" : [ + { + "name" : "temperature", + "type" : "centigrade", + "value" : "26.5", + "metadatas": [ + { + "name": "ID", + "type": "string", + "value": "ground" + } + ] + } + ], + "type" : "Room", + "isPattern" : "false", + "id" : "Room1" + }, + "statusCode" : { + "code" : "200", + "reasonPhrase" : "OK" + } + } + ] + } + +Such a notification is sent by Orion to the default Flume HTTP source, which relies on the developed OrionRestHandler for checking its validity (that it is a POST request, that the target is 'notify' and that the headers are OK), detecting the content type (that it is in Json format), extracting the data (the Json part) and finally creating a Flume event to be put in the channel: + + event={ + body=json_data, + headers={ + content-type=application/json, + fiware-service=my_company_name, + fiware-servicepath=workingrooms_floor4, + timestamp=1402409899391, + transactionId=asdfasdfsdfa, + ttl=10, + destination=Room1-Room + } + } + +NOTE: The above is an object representation, not Json data nor any other data format. + +Let's have a look on the Flume event headers: + +* The content-type header is a replica of the HTTP header. It is needed for the different sinks to know how to parse the event body. In this case it is JSON. +* Note that Orion can include a `Fiware-Service` HTTP header specifying the tenant/organization associated to the notification, which is added to the event headers as well (as `fiware-service`). Since version 0.3, Cygnus is able to support this header, although the actual processing of such tenant/organization depends on the particular sink. If the notification doesn't include this header, then Cygnus will use the default service specified in the `default_service` configuration property. Please observe that the notified `fiware-service` is transformed following the rules described at [`doc/design/naming_conventions.md`](doc/design/naming_conventions.md). +* Orion can notify another HTTP header, `Fiware-ServicePath` specifying a subservice within a tenant/organization, which is added to the event headers as well (as `fiware-servicepath`). Since version 0.6, Cygnus is able to support this header, although the actual processing of such subservice depends on the particular sink. If the notification doesn't include this header, then Cygnus will use the default service path specified in the `default_service_path` configuration property. Please observe that the notified `fiware-servicePath` is transformed following the rules described at [`doc/design/naming_conventions.md`](doc/design/naming_conventions.md). +* The notification reception time is included in the list of headers (as timestamp) for timestamping purposes in the different sinks. It is added by a native interceptor. See the [doc/design/interceptors.md](doc/design/interceptors.md) document for more details. +* The transactionId identifies a complete Cygnus transaction, starting at the source when the context data is notified, and finishing in the sink, where such data is finally persisted. +* The time-to-live (or ttl) specifies the number of re-injection retries in the channel when something goes wrong while persisting the data. This re-injection mechanism is part of the reliability features of Flume. -1 means inifinite retries. +* The destination headers is used to identify the persistence element within the used storage, i.e. a file in HDFS, a MySQL table or a CKAN resource. This is added by a custom interceptor called `DestinationExtractor` added to the Flume's suite. See the doc/design/interceptors document for more details. + +Finally, the channel is a simple MemoryChannel behaving as a FIFO queue, and from where the different sinks extract the events in order to persist them; let's see how: + +### OrionHDFSSink persistence + +This sink persists the data in files, one per each entity, following this entity descriptor format: + + =-.txt + +These files are stored under this HDFS path: + + hdfs:///user/////.txt + +Usernames allow for specific private HDFS data spaces, and in the current version, it is given by the `cosmos_default_username` parameter that can be found in the configuration. Both the `service` and `servicePath` directories are given by Orion as headers in the notification (`Fiware-Service` and `Fiware-ServicePath` respectively) and sent to the sinks through the Flume event headers (`fiware-service` and `fiware-servicepath` respectively). + +More details regarding the naming conventions can be found at [doc/design/naming_convetions.md](doc/design/naming_convetions.md). + +Within files, Json documents are written following one of these two schemas: + +* Fixed 8-field lines: `recvTimeTs`, `recvTime`, `entityId`, `entityType`, `attrName`, `attrType`, `attrValue` and `attrMd`. Regarding `attrValue`, in its simplest form, this value is just a string, but since Orion 0.11.0 it can be Json object or Json array. Regarding `attrMd`, it contains a string serialization of the metadata array for the attribute in Json (if the attribute hasn't metadata, an empty array `[]` is inserted). +* Two fields per each entity's attribute (one for the value and other for the metadata), plus an additional field about the reception time of the data (`recvTime`). Regarding this kind of persistence, the notifications must ensure a value per each attribute is notified. + +In both cases, the files are created at execution time if the file doesn't exist previously to the line insertion. The behaviour of the connector regarding the internal representation of the data is governed through a configuration parameter, `attr_persistence`, whose values can be `row` or `column`. + +Thus, by receiving a notification like the one above, being the persistence mode `row` and a `default_user` as the default Cosmos username, then the file named `hdfs:///user/default_user/mycompanyname/workingrooms/floor4/Room1-Room/Room1-Room.txt` (it is created if not existing) will contain a new line such as: + + {"recvTimeTs":"13453464536", "recvTime":"2014-02-27T14:46:21", "entityId":"Room1", "entityType":"Room", "attrName":"temperature", "attrType":"centigrade", "attrValue":"26.5", "attrMd":[{name:ID, type:string, value:ground}]} + +On the contrary, being the persistence mode `column`, the file named `hdfs:///user/default_user/mycompanyname/workingrooms/floor4/Room1-Room/Room1-Room.txt` (it is created if not existing) will contain a new line such as: + + {"recvTime":"2014-02-27T14:46:21", "temperature":"26.5", "temperature_md":[{"name":"ID", "type":"string", "value":"ground"}]} + +A special particularity regarding HDFS persisted data is the posssibility to exploit such data through Hive, a SQL-like querying system. OrionHDFSSink automatically creates a Hive table (similar to a SQL table) for each persisted entity in the default database, being the name for such tables: + + ____[row|column] + +Following with the example, by receiving a notification like the one above, and being the persistence mode `row`, the table named `default_user_mycompanyname_workingrooms_floor4_room1_Room_row` will contain a new row such as: + + | recvTimeTs | recvTime | entityId | entityType | attrName | attrType | attrValue | attrMd | + |--------------|---------------------|----------|------------|-------------|------------|-----------|----------------------------------------------------| + | 13453464536 | 2014-02-27T14:46:21 | Room1 | Room | temperature | centigrade | 26.5 | [{"name":"ID", "type":"string", "value":"ground"}] | + +On the contrary, being the persistence mode `column`, the table named `default_user_mycompanyname_workingrooms_floor4_room1_Room_column` will contain a new row such as: + + | recvTime | temperature | temperature_md | + |---------------------|-------------|----------------------------------------------------| + | 2014-02-27T14:46:21 | 26.5 | [{"name":"ID", "type":"string", "value":"ground"}] | + +### OrionCKANSink persistence + +This sink persists the data in a [datastore](see http://docs.ckan.org/en/latest/maintaining/datastore.html) in CKAN. Datastores are associated to CKAN resources and as CKAN resources we use the entityId-entityType string concatenation. All CKAN resource IDs belong to the same dataset (also referred as package in CKAN terms), whose name is specified by the notified `Fiware-ServicePath` header (or by the `default_service_path` property -prefixed by organization name- in the CKAN sink configuration, if such a header is not notified). Datasets belong to single organization, whose name is specified by the notified `Fiware-Service` header (or by the `default_service` property if it is not notified). + +More details regarding the naming conventions can be found at [doc/design/naming_convetions.md](doc/design/naming_convetions.md). + +Each datastore, we can find two options: + +* Fixed 6-field lines: `recvTimeTs`, `recvTime`, `attrName`, `attrType`, `attrValue` and `attrMd`. Regarding `attrValue`, in its simplest form, this value is just a string, but since Orion 0.11.0 it can be JSON object or JSON array. Regarding `attrMd`, it contains a string serialization of the metadata array for the attribute in JSON (if the attribute hasn't metadata, `null` is inserted). +* Two columns per each entity's attribute (one for the value and other for the metadata), plus an additional field about the reception time of the data (`recvTime`). Regarding this kind of persistence, the notifications must ensure a value per each attribute is notified. + +The behaviour of the connector regarding the internal representation of the data is governed through a configuration parameter, `attr_persistence`, whose values can be `row` or `column`. + +Thus, by receiving a notification like the one above, and being the persistence mode `row`, the resource `room1-Room` (it is created if not existing), will containt the following row in its datastore: + + | _id | recvTimeTs | recvTime | attrName | attrType | attrValue | attrMd | + |-----|--------------|---------------------|-----.-------|------------|-----------|-----------------------------------------------------| + | i | 13453464536 | 2014-02-27T14:46:21 | temperature | centigrade | 26.5 | [{"name":"ID", "type":"string", "value":"ground"}] | + +where `i` depends on the number of rows previously inserted. + +On the contrary, being the persistence mode `column`, the resource `Room1-Room` (it and its datastore must be created in advance) will contain a new row such as shown below. In this case, an extra column ended with `_md` is added for the metadata. + + | _id | recvTime | temperature | temperature_md | + |--------------------------|-------------|----------------------------------------------------| + | i |2014-02-27T14:46:21 | 26.5 | [{"name":"ID", "type":"string", "value":"ground"}] | + +where `i` depends on the number of rows previously inserted. + +In both cases, `row` or `column`, the CKAN organization will be `mycompanyname` and the dataset containing the resource will be `workingrooms_floor4`. + +The information stored in the datastore can be accesses as any other CKAN information, e.g. through the web frontend or using the query API, e.g; + + curl -s -S "http://${CKAN_HOST}/api/3/action/datastore_search?resource_id=${RESOURCE_ID} + +Each organization/tenant is associated to a CKAN organization. + +### OrionMySQLSink persistence + +Similarly to OrionHDFSSink, a table is considered for each entity in order to store its notified context data, being the name for these tables the following entity descriptor: + + =__ + +These tables are stored in databases, one per service, enabling a private data space such as: + + jdbc:mysql:/// + +Both the `service` and `servicePath` names are given by Orion as headers in the notification (`Fiware-Service` and `Fiware-ServicePath` respectively) and sent to the sinks through the Flume event headers (`fiware-service` and `fiware-servicepath` respectively). + +More details regarding the naming conventions can be found at [doc/design/naming_convetions.md](doc/design/naming_convetions.md). + +Within tables, we can find two options: + +* Fixed 8-field rows, as usual: `recvTimeTs`, `recvTime`, `entityId`, `entityType`, `attrName`, `attrType`, `attrValue` and `attrMd`. These tables (and the databases) are created at execution time if the table doesn't exist previously to the row insertion. Regarding `attrValue`, in its simplest form, this value is just a string, but since Orion 0.11.0 it can be Json object or Json array. Regarding `attrMd`, it contains a string serialization of the metadata array for the attribute in Json (if the attribute hasn't metadata, an empty array `[]` is inserted), +* Two columns per each entity's attribute (one for the value and other for the metadata), plus an addition column about the reception time of the data (`recv_time`). This kind of tables (and the databases) must be provisioned previously to the execution of Cygnus, because each entity may have a different number of attributes, and the notifications must ensure a value per each attribute is notified. + +The behaviour of the connector regarding the internal representation of the data is governed through a configuration parameter, `attr_persistence`, whose values can be `row` or `column`. + +Thus, by receiving a notification like the one above, and being the persistence mode `row`, the table named `workingrooms_floor4_room1_Room` (it is created if not existing) will contain a new row such as: + + | recvTimeTs | recvTime | entityId | entityType | attrName | attrType | attrValue | attrMd | + |--------------|---------------------|----------|------------|-------------|------------|-----------|----------------------------------------------------| + | 13453464536 | 2014-02-27T14:46:21 | Room1 | Room | temperature | centigrade | 26.5 | [{"name":"ID", "type":"string", "value":"ground"}] | + +On the contrary, being the persistence mode `column`, the table named `workingrooms_floor4_room1_Room` (it must be created in advance) will contain a new row such as: + + | recvTime | temperature | temperature_md | + |---------------------|-------------|----------------------------------------------------| + | 2014-02-27T14:46:21 | 26.5 | [{"name":"ID", "type":"string", "value":"ground"}] | + +Each organization/tenant is associated to a different database. + +## Functionality explained (XML notification example) + +Cygnus also works with [XML-based notifications](https://forge.fi-ware.eu/plugins/mediawiki/wiki/fiware/index.php/Publish/Subscribe_Broker_-_Orion_Context_Broker_-_User_and_Programmers_Guide#ONCHANGE) sent to the connector. The only difference is the event is created by specifying the content type will be XML (in order the notification parser notices it): + + event={ + body=json_data, + headers={ + content-type=application/xml, + fiware-service=my_company_name, + fiware-servicepath=workingrooms_floor4, + timestamp=1402409899391, + transactionId=asdfasdfsdfa, + ttl=10, + destination=Room1-Room + } + } + +The key point is the behaviour remains the same than in the Json example: the same file/datastores/tables will be created, and the same data will be persisted within it. + +## Installing Cygnus +###RPM install (recommended) +Simply configure the FIWARE repository if not yet configured and use your applications manager in order to install the latest version of Cygnus (CentOS/RedHat example): + + $ cat > /etc/yum.repos.d/fiware.repo <.conf` file addressing all those non Flume parameters, such as the Flume agent name, the specific log file for this instance, the administration port, etc. This configuration file is not necessary if Cygnus is run as a standlalone application (see later), bt it is mandatory if run as a service (see later). +* An `agent_.conf` file addressing all those Flume parameters, i.e. how to configure the different sources, channels, sinks, etc. that compose the Flume agent behind the Cygnus instance. always mandatory. + +Please observe there may exist several Cygnus instances identified by ``, which must be the same for both configuration files regarding the same Cygnus instance. + +###`cygnus_instance_.conf` + +The file `cygnus_instance_.conf` can be instantiated from a template given in the Cygnus repository, `conf/cygnus_instance.conf.template`. + +``` +# The OS user that will be running Cygnus. Note this must be `root` if you want to run cygnus in a privileged port (<1024), either the admin port or the port in which Cygnus receives Orion notifications +CYGNUS_USER=cygnus +# Which is the config folder +CONFIG_FOLDER=/usr/cygnus/conf +# Which is the config file +CONFIG_FILE=/usr/cygnus/conf/agent_.conf +# Name of the agent. The name of the agent is not trivial, since it is the base for the Flume parameters naming conventions, e.g. it appears in .sources.http-source.channels=... +AGENT_NAME=cygnusagent +# Name of the logfile located at /var/log/cygnus. It is important to put the extension '.log' in order to the log rotation works properly +LOGFILE_NAME=cygnus.log +# Administration port. Must be unique per instance +ADMIN_PORT=8081 +# Polling interval (seconds) for the configuration reloading +POLLING_INTERVAL=30 +``` + +###`agent_.conf` +A typical configuration when using the `HTTPSource`, the `OrionRestHandler`, the `MemoryChannel` and any of the available sinks is shown below. More advanced configurations can be found at [`doc/operation/performance_tuning_tips.md`](doc/operation/performance_tuning_tips.md). + +Kerberos authentication enabling in HDFS is described at [`doc/operation/hdfs_kerberos_authentication.md`](doc/operation/hdfs_kerberos_authentication.md). If your HDFS is not using such an authentication method, just set `cygnusagent.sinks.hdfs-sink.krb5_auth` to `false` and forget the rest of the Kerberos part. + +The file `agent_.conf` can be instantiated from a template given in the Cygnus repository, `conf/agent.conf.template`. + +```Java +#============================================= +# To be put in APACHE_FLUME_HOME/conf/cygnus.conf +# +# General configuration template explaining how to setup a sink of each of the available types (HDFS, CKAN, MySQL). + +#============================================= +# The next tree fields set the sources, sinks and channels used by Cygnus. You could use different names than the +# ones suggested below, but in that case make sure you keep coherence in properties names along the configuration file. +# Regarding sinks, you can use multiple types at the same time; the only requirement is to provide a channel for each +# one of them (this example shows how to configure 3 sink types at the same time). Even, you can define more than one +# sink of the same type and sharing the channel in order to improve the performance (this is like having +# multi-threading). +cygnusagent.sources = http-source +cygnusagent.sinks = hdfs-sink mysql-sink ckan-sink +cygnusagent.channels = hdfs-channel mysql-channel ckan-channel + +#============================================= +# source configuration +# channel name where to write the notification events +cygnusagent.sources.http-source.channels = hdfs-channel mysql-channel ckan-channel +# source class, must not be changed +cygnusagent.sources.http-source.type = org.apache.flume.source.http.HTTPSource +# listening port the Flume source will use for receiving incoming notifications +cygnusagent.sources.http-source.port = 5050 +# Flume handler that will parse the notifications, must not be changed +cygnusagent.sources.http-source.handler = es.tid.fiware.fiwareconnectors.cygnus.handlers.OrionRestHandler +# URL target +cygnusagent.sources.http-source.handler.notification_target = /notify +# Default service (service semantic depends on the persistence sink) +cygnusagent.sources.http-source.handler.default_service = def_serv +# Default service path (service path semantic depends on the persistence sink) +cygnusagent.sources.http-source.handler.default_service_path = def_servpath +# Number of channel re-injection retries before a Flume event is definitely discarded (-1 means infinite retries) +cygnusagent.sources.http-source.handler.events_ttl = 10 +# Source interceptors, do not change +cygnusagent.sources.http-source.interceptors = ts de +# Interceptor type, do not change +cygnusagent.sources.http-source.interceptors.ts.type = timestamp +# Destination extractor interceptor, do not change +cygnusagent.sources.http-source.interceptors.de.type = es.tid.fiware.fiwareconnectors.cygnus.interceptors.DestinationExtractor$Builder +# Matching table for the destination extractor interceptor, put the right absolute path to the file if necessary +# See the doc/design/interceptors document for more details +cygnusagent.sources.http-source.interceptors.de.matching_table = /usr/cygnus/conf/matching_table.conf + +# ============================================ +# OrionHDFSSink configuration +# channel name from where to read notification events +cygnusagent.sinks.hdfs-sink.channel = hdfs-channel +# sink class, must not be changed +cygnusagent.sinks.hdfs-sink.type = es.tid.fiware.fiwareconnectors.cygnus.sinks.OrionHDFSSink +# Comma-separated list of FQDN/IP address regarding the Cosmos Namenode endpoints +# If you are using Kerberos authentication, then the usage of FQDNs instead of IP addresses is mandatory +cygnusagent.sinks.hdfs-sink.cosmos_host = x1.y1.z1.w1,x2.y2.z2.w2 +# port of the Cosmos service listening for persistence operations; 14000 for httpfs, 50070 for webhdfs and free choice for inifinty +cygnusagent.sinks.hdfs-sink.cosmos_port = 14000 +# default username allowed to write in HDFS +cygnusagent.sinks.hdfs-sink.cosmos_default_username = cosmos_username +# default password for the default username +cygnusagent.sinks.hdfs-sink.cosmos_default_password = xxxxxxxxxxxxx +# HDFS backend type (webhdfs, httpfs or infinity) +cygnusagent.sinks.hdfs-sink.hdfs_api = httpfs +# how the attributes are stored, either per row either per column (row, column) +cygnusagent.sinks.hdfs-sink.attr_persistence = column +# Hive FQDN/IP address of the Hive server +cygnusagent.sinks.hdfs-sink.hive_host = x.y.z.w +# Hive port for Hive external table provisioning +cygnusagent.sinks.hdfs-sink.hive_port = 10000 +# Kerberos-based authentication enabling +cygnusagent.sinks.hdfs-sink.krb5_auth = false +# Kerberos username +cygnusagent.sinks.hdfs-sink.krb5_auth.krb5_user = krb5_username +# Kerberos password +cygnusagent.sinks.hdfs-sink.krb5_auth.krb5_password = xxxxxxxxxxxxx +# Kerberos login file +cygnusagent.sinks.hdfs-sink.krb5_auth.krb5_login_conf_file = /usr/cygnus/conf/krb5_login.conf +# Kerberos configuration file +cygnusagent.sinks.hdfs-sink.krb5_auth.krb5_conf_file = /usr/cygnus/conf/krb5.conf + +# ============================================ +# OrionCKANSink configuration +# channel name from where to read notification events +cygnusagent.sinks.ckan-sink.channel = ckan-channel +# sink class, must not be changed +cygnusagent.sinks.ckan-sink.type = es.tid.fiware.fiwareconnectors.cygnus.sinks.OrionCKANSink +# the CKAN API key to use +cygnusagent.sinks.ckan-sink.api_key = ckanapikey +# the FQDN/IP address for the CKAN API endpoint +cygnusagent.sinks.ckan-sink.ckan_host = x.y.z.w +# the port for the CKAN API endpoint +cygnusagent.sinks.ckan-sink.ckan_port = 80 +# Orion URL used to compose the resource URL with the convenience operation URL to query it +cygnusagent.sinks.ckan-sink.orion_url = http://localhost:1026 +# how the attributes are stored, either per row either per column (row, column) +cygnusagent.sinks.ckan-sink.attr_persistence = row +# enable SSL for secure Http transportation; 'true' or 'false' +cygnusagent.sinks.ckan-sink.ssl = false + +# ============================================ +# OrionMySQLSink configuration +# channel name from where to read notification events +cygnusagent.sinks.mysql-sink.channel = mysql-channel +# sink class, must not be changed +cygnusagent.sinks.mysql-sink.type = es.tid.fiware.fiwareconnectors.cygnus.sinks.OrionMySQLSink +# the FQDN/IP address where the MySQL server runs +cygnusagent.sinks.mysql-sink.mysql_host = x.y.z.w +# the port where the MySQL server listes for incomming connections +cygnusagent.sinks.mysql-sink.mysql_port = 3306 +# a valid user in the MySQL server +cygnusagent.sinks.mysql-sink.mysql_username = root +# password for the user above +cygnusagent.sinks.mysql-sink.mysql_password = xxxxxxxxxxxx +# how the attributes are stored, either per row either per column (row, column) +cygnusagent.sinks.mysql-sink.attr_persistence = column + +#============================================= +# hdfs-channel configuration +# channel type (must not be changed) +cygnusagent.channels.hdfs-channel.type = memory +# capacity of the channel +cygnusagent.channels.hdfs-channel.capacity = 1000 +# amount of bytes that can be sent per transaction +cygnusagent.channels.hdfs-channel.transactionCapacity = 100 + +#============================================= +# ckan-channel configuration +# channel type (must not be changed) +cygnusagent.channels.ckan-channel.type = memory +# capacity of the channel +cygnusagent.channels.ckan-channel.capacity = 1000 +# amount of bytes that can be sent per transaction +cygnusagent.channels.ckan-channel.transactionCapacity = 100 + +#============================================= +# mysql-channel configuration +# channel type (must not be changed) +cygnusagent.channels.mysql-channel.type = memory +# capacity of the channel +cygnusagent.channels.mysql-channel.capacity = 1000 +# amount of bytes that can be sent per transaction +cygnusagent.channels.mysql-channel.transactionCapacity = 100 +``` + +## Running as a service (recommended) +NOTE: Cygnus can only be run as a service if you installed it through the RPM. + +Once the `cygnus_instance_.conf` and `agent_.conf` files are properly configured, just use the `service` command to start, restart, stop or get the status (as a sudoer): + + $ sudo service cygnus status + + $ sudo service cygnus start + + $ sudo service cygnus restart + + $ sudo service cygnus stop + +Previous commands afefcts to **all** of Cygnus instances configured. If only one instance is wanted to be managed by the service script then the instance identifier after de the action must be specified: + + $ sudo service cygnus status + + $ sudo service cygnus start + + $ sudo service cygnus restart + + $ sudo service cygnus stop + +Where `` is the suffix at the end of the `cygnus_instace_.conf` or `agent_.conf` files you used to configure the instance. + +## Running as standalone application (advanced) + +NOTE: If you installed Cygnus through the RPM, APACHE\_FLUME\_HOME is `/usr/cygnus/`. If not, it is a directory of your choice. + +Cygnus implements its own startup script, `cygnus-flume-ng` which replaces the standard `flume-ng` one, which in the end runs a custom `es.tid.fiware.fiwareconnectors.cygnus.nodes.CygnusApplication` instead of a standard `org.apache.flume.node.Application`. + +In foreground (with logging): + + $ APACHE_FLUME_HOME/bin/cygnus-flume-ng agent --conf APACHE_FLUME_HOME/conf -f APACHE_FLUME_HOME/conf/cygnus.conf -n cygnusagent -Dflume.root.logger=INFO,console [-p ] [-t ] + +In background: + + $ nohup APACHE_FLUME_HOME/bin/cygnus-flume-ng agent --conf APACHE_FLUME_HOME/conf -f APACHE_FLUME_HOME/conf/cygnus.conf -n cygnusagent -Dflume.root.logger=INFO,LOGFILE [-p ] [-t ] & + +The parameters used in these commands are: + +* `agent`. This is the type of application to be run by the `cygnus-flume-ng` script. +* `--conf`. Points to the Apache Flume configuration folder. +* `-f` (or `--conf-file`). This is the agent configuration (`agent_.conf`) file. Please observe when running in this mode no `cygnus_instance_.conf` file is required. +* `-n` (or `--name`). The name of the Flume agent to be run. +* `-Dflume.root.logger`. Changes the logging level and the logging appender for log4j. +* `-p` (or `--mgmt-if-port`). Configures the listening port for the Management Interface. If not configured, the default value is used, `8081`. +* `-t` (or `--polling-interval`). Configures the polling interval (seconds) when the configuration is periodically reloaded. If not configured, the default value is used, `30`. + +## Orion subscription + +Once the connector is running, it is necessary to tell Orion Context Broker about it, in order Orion can send context data notifications to the connector. This can be done on behalf of the connector by performing the following curl command: + + $ (curl localhost:1026/NGSI10/subscribeContext -s -S --header 'Content-Type: application/xml' -d @- | xmllint --format -) < + + + + Room1 + + + + temperature + + + http://host_running_cygnus:5050/notify + P1M + + + ONCHANGE + + pressure + + + + PT5S + + EOF + +Its equivalent in Json format can be seen [here](https://forge.fi-ware.eu/plugins/mediawiki/wiki/fiware/index.php/Publish/Subscribe_Broker_-_Orion_Context_Broker_-_User_and_Programmers_Guide#ONCHANGE). + +## Logs + +###log4j configuration + +Cygnus uses the log4j facilities added by Flume for logging purposes. You can maintain the default `APACHE_FLUME_HOME/conf/log4j.properties` file, where a console and a file appender are defined (in addition, the console is used by default), or customize it by adding new appenders. Typically, you will have several instances of Cygnus running; they will be listening on different TCP ports for incoming notifyContextRequest and you'll probably want to have differente log files for them. E.g., if you have two Flume processes listening on TCP/1028 and TCP/1029 ports, then you can add the following lines to the `log4j.properties` file: + + log4j.appender.cygnus1028=org.apache.log4j.RollingFileAppender + log4j.appender.cygnus1028.MaxFileSize=100MB + log4j.appender.cygnus1028.MaxBackupIndex=10 + log4j.appender.cygnus1028.File=${flume.log.dir}/cygnus.1028.log + log4j.appender.cygnus1028.layout=org.apache.log4j.PatternLayout + log4j.appender.cygnus1028.layout.ConversionPattern=time=%d{yyyy-MM-dd}T%d{HH:mm:ss.SSSzzz} | lvl=%p | trans=%X{transactionId} | function=%M | comp=Cygnus | msg=%C[%L] : %m%n + + log4j.appender.cygnus1029=org.apache.log4j.RollingFileAppender + log4j.appender.cygnus1029.MaxFileSize=100MB + log4j.appender.cygnus1029.MaxBackupIndex=10 + log4j.appender.cygnus1029.File=${flume.log.dir}/cygnus.1029.log + log4j.appender.cygnus1029.layout=org.apache.log4j.PatternLayout + log4j.appender.cygnus1029.layout.ConversionPattern=time=%d{yyyy-MM-dd}T%d{HH:mm:ss.SSSzzz} | lvl=%p | trans=%X{transactionId} | function=%M | comp=Cygnus | msg=%C[%L] : %m%n + +Regarding the log4j Conversion Pattern: + +* `time` makes reference to a timestamp following the [RFC3339](http://tools.ietf.org/html/rfc3339). +* `lvl`means logging level, and matches the traditional log4j levels: `INFO`, `WARN`, `ERROR`, `FATAL` and `DEBUG`. +* `trans` is a transaction identifier, i.e. an identifier that is printed in all the traces related to the same Orion notification. The format is `--<10_digits_transaction_count>`. Its generation logic ensures that every transaction identifier is unique, also for Cygnus instances running in different VMs, except if they are started in the exactly same millisecond (highly unprobable). +* `function` identifies the function/method within the class printing the log. +* `comp` is always `Cygnus`. +* `msg` is a custom message that has always the same format: `[] : `. + +Once the log4j has been properly configured, you only have to add to the Flume command line the following parameter, which overwrites the default configutation (`flume.root.logger=INFO,LOGFILE`): + + -Dflume.root.logger=,cygnus..log + +In addition, you have a complete `log4j.properties` template in `conf/log4j.properties.template`, once you clone the Cygnus repository. + +### Message types + +Check [doc/operation/alarms.md](doc/operation/alarms.md) for a detailed list of message types. + +## Management interface + +From Cygnus 0.5 there is a REST-based management interface for administration purposes. Current available operations are: + +Get the version of the running software, including the last Git commit: + + GET http://host:management_port/version + + {"version":"0.5_SNAPSHOT.8a6c07054da894fc37ef30480cb091333e2fccfa"} ## Contact -* Fermín Galán Márquez (fermin at tid dot es). -* Francisco Romero Bueno (frb at tid dot es). +* Fermín Galán Márquez (fermin.galanmarquez@telefonica.com). +* Francisco Romero Bueno (francisco.romerobueno@telefonica.com). diff --git a/flume/conf/README.md b/conf/README.md similarity index 100% rename from flume/conf/README.md rename to conf/README.md diff --git a/flume/conf/agent.conf.template b/conf/agent.conf.template similarity index 100% rename from flume/conf/agent.conf.template rename to conf/agent.conf.template diff --git a/flume/conf/cygnus_instance.conf.template b/conf/cygnus_instance.conf.template similarity index 100% rename from flume/conf/cygnus_instance.conf.template rename to conf/cygnus_instance.conf.template diff --git a/flume/conf/krb5.conf.template b/conf/krb5.conf.template similarity index 100% rename from flume/conf/krb5.conf.template rename to conf/krb5.conf.template diff --git a/flume/conf/krb5_login.conf b/conf/krb5_login.conf similarity index 100% rename from flume/conf/krb5_login.conf rename to conf/krb5_login.conf diff --git a/flume/conf/log4j.properties.template b/conf/log4j.properties.template similarity index 100% rename from flume/conf/log4j.properties.template rename to conf/log4j.properties.template diff --git a/flume/conf/matching_table.conf.template b/conf/matching_table.conf.template similarity index 100% rename from flume/conf/matching_table.conf.template rename to conf/matching_table.conf.template diff --git a/flume/doc/design/OrionCKANSink.md b/doc/design/OrionCKANSink.md similarity index 100% rename from flume/doc/design/OrionCKANSink.md rename to doc/design/OrionCKANSink.md diff --git a/flume/doc/design/OrionCKANwqSink.md b/doc/design/OrionCKANwqSink.md similarity index 100% rename from flume/doc/design/OrionCKANwqSink.md rename to doc/design/OrionCKANwqSink.md diff --git a/flume/doc/design/OrionHDFSSink.md b/doc/design/OrionHDFSSink.md similarity index 100% rename from flume/doc/design/OrionHDFSSink.md rename to doc/design/OrionHDFSSink.md diff --git a/flume/doc/design/OrionMySQLSink.md b/doc/design/OrionMySQLSink.md similarity index 100% rename from flume/doc/design/OrionMySQLSink.md rename to doc/design/OrionMySQLSink.md diff --git a/flume/doc/design/OrionRestHander.md b/doc/design/OrionRestHander.md similarity index 100% rename from flume/doc/design/OrionRestHander.md rename to doc/design/OrionRestHander.md diff --git a/flume/doc/design/OrionTestSink.md b/doc/design/OrionTestSink.md similarity index 100% rename from flume/doc/design/OrionTestSink.md rename to doc/design/OrionTestSink.md diff --git a/flume/doc/design/architecture.md b/doc/design/architecture.md similarity index 100% rename from flume/doc/design/architecture.md rename to doc/design/architecture.md diff --git a/flume/doc/design/ha.md b/doc/design/ha.md similarity index 100% rename from flume/doc/design/ha.md rename to doc/design/ha.md diff --git a/flume/doc/design/interceptors.md b/doc/design/interceptors.md similarity index 100% rename from flume/doc/design/interceptors.md rename to doc/design/interceptors.md diff --git a/flume/doc/design/naming_conventions.md b/doc/design/naming_conventions.md similarity index 100% rename from flume/doc/design/naming_conventions.md rename to doc/design/naming_conventions.md diff --git a/flume/doc/devel/add_new_sink.md b/doc/devel/add_new_sink.md similarity index 100% rename from flume/doc/devel/add_new_sink.md rename to doc/devel/add_new_sink.md diff --git a/flume/doc/images/basic_architecture.jpg b/doc/images/basic_architecture.jpg similarity index 100% rename from flume/doc/images/basic_architecture.jpg rename to doc/images/basic_architecture.jpg diff --git a/flume/doc/images/basic_architecture.vsd b/doc/images/basic_architecture.vsd similarity index 100% rename from flume/doc/images/basic_architecture.vsd rename to doc/images/basic_architecture.vsd diff --git a/flume/doc/images/ha_architecture.jpg b/doc/images/ha_architecture.jpg similarity index 100% rename from flume/doc/images/ha_architecture.jpg rename to doc/images/ha_architecture.jpg diff --git a/flume/doc/images/ha_architecture.vsd b/doc/images/ha_architecture.vsd similarity index 100% rename from flume/doc/images/ha_architecture.vsd rename to doc/images/ha_architecture.vsd diff --git a/flume/doc/images/kerberos_auth.jpg b/doc/images/kerberos_auth.jpg similarity index 100% rename from flume/doc/images/kerberos_auth.jpg rename to doc/images/kerberos_auth.jpg diff --git a/flume/doc/images/kerberos_auth.vsd b/doc/images/kerberos_auth.vsd similarity index 100% rename from flume/doc/images/kerberos_auth.vsd rename to doc/images/kerberos_auth.vsd diff --git a/flume/doc/images/multiple_sinks_multiple_channels_architecture.jpg b/doc/images/multiple_sinks_multiple_channels_architecture.jpg similarity index 100% rename from flume/doc/images/multiple_sinks_multiple_channels_architecture.jpg rename to doc/images/multiple_sinks_multiple_channels_architecture.jpg diff --git a/flume/doc/images/multiple_sinks_multiple_channels_architecture.vsd b/doc/images/multiple_sinks_multiple_channels_architecture.vsd similarity index 100% rename from flume/doc/images/multiple_sinks_multiple_channels_architecture.vsd rename to doc/images/multiple_sinks_multiple_channels_architecture.vsd diff --git a/flume/doc/images/multiple_sinks_single_channel_architecture.jpg b/doc/images/multiple_sinks_single_channel_architecture.jpg similarity index 100% rename from flume/doc/images/multiple_sinks_single_channel_architecture.jpg rename to doc/images/multiple_sinks_single_channel_architecture.jpg diff --git a/flume/doc/images/multiple_sinks_single_channel_architecture.vsd b/doc/images/multiple_sinks_single_channel_architecture.vsd similarity index 100% rename from flume/doc/images/multiple_sinks_single_channel_architecture.vsd rename to doc/images/multiple_sinks_single_channel_architecture.vsd diff --git a/flume/doc/images/sequence_diagram_ckan.jpg b/doc/images/sequence_diagram_ckan.jpg similarity index 100% rename from flume/doc/images/sequence_diagram_ckan.jpg rename to doc/images/sequence_diagram_ckan.jpg diff --git a/flume/doc/images/sequence_diagram_ckan.vsd b/doc/images/sequence_diagram_ckan.vsd similarity index 100% rename from flume/doc/images/sequence_diagram_ckan.vsd rename to doc/images/sequence_diagram_ckan.vsd diff --git a/flume/doc/images/sequence_diagram_hdfs.jpg b/doc/images/sequence_diagram_hdfs.jpg similarity index 100% rename from flume/doc/images/sequence_diagram_hdfs.jpg rename to doc/images/sequence_diagram_hdfs.jpg diff --git a/flume/doc/images/sequence_diagram_hdfs.vsd b/doc/images/sequence_diagram_hdfs.vsd similarity index 100% rename from flume/doc/images/sequence_diagram_hdfs.vsd rename to doc/images/sequence_diagram_hdfs.vsd diff --git a/flume/doc/images/sequence_diagram_mysql.jpg b/doc/images/sequence_diagram_mysql.jpg similarity index 100% rename from flume/doc/images/sequence_diagram_mysql.jpg rename to doc/images/sequence_diagram_mysql.jpg diff --git a/flume/doc/images/sequence_diagram_mysql.vsd b/doc/images/sequence_diagram_mysql.vsd similarity index 100% rename from flume/doc/images/sequence_diagram_mysql.vsd rename to doc/images/sequence_diagram_mysql.vsd diff --git a/flume/doc/images/sequence_diagram_notification_handling_default.jpg b/doc/images/sequence_diagram_notification_handling_default.jpg similarity index 100% rename from flume/doc/images/sequence_diagram_notification_handling_default.jpg rename to doc/images/sequence_diagram_notification_handling_default.jpg diff --git a/flume/doc/images/sequence_diagram_notification_handling_default.vsd b/doc/images/sequence_diagram_notification_handling_default.vsd similarity index 100% rename from flume/doc/images/sequence_diagram_notification_handling_default.vsd rename to doc/images/sequence_diagram_notification_handling_default.vsd diff --git a/flume/doc/installation/chef_install.md b/doc/installation/chef_install.md similarity index 100% rename from flume/doc/installation/chef_install.md rename to doc/installation/chef_install.md diff --git a/flume/doc/installation/rpm_install.md b/doc/installation/rpm_install.md similarity index 100% rename from flume/doc/installation/rpm_install.md rename to doc/installation/rpm_install.md diff --git a/flume/doc/installation/src_install.md b/doc/installation/src_install.md similarity index 100% rename from flume/doc/installation/src_install.md rename to doc/installation/src_install.md diff --git a/flume/doc/operation/alarms.md b/doc/operation/alarms.md similarity index 100% rename from flume/doc/operation/alarms.md rename to doc/operation/alarms.md diff --git a/flume/doc/operation/cygnus_0.1_to_0.2_migration.md b/doc/operation/cygnus_0.1_to_0.2_migration.md similarity index 100% rename from flume/doc/operation/cygnus_0.1_to_0.2_migration.md rename to doc/operation/cygnus_0.1_to_0.2_migration.md diff --git a/flume/doc/operation/cygnus_0.1_to_0.3_(or_higher)_migration.md b/doc/operation/cygnus_0.1_to_0.3_(or_higher)_migration.md similarity index 100% rename from flume/doc/operation/cygnus_0.1_to_0.3_(or_higher)_migration.md rename to doc/operation/cygnus_0.1_to_0.3_(or_higher)_migration.md diff --git a/flume/doc/operation/hdfs_kerberos_authentication.md b/doc/operation/hdfs_kerberos_authentication.md similarity index 100% rename from flume/doc/operation/hdfs_kerberos_authentication.md rename to doc/operation/hdfs_kerberos_authentication.md diff --git a/flume/doc/operation/ngsi2cosmos_migration.md b/doc/operation/ngsi2cosmos_migration.md similarity index 100% rename from flume/doc/operation/ngsi2cosmos_migration.md rename to doc/operation/ngsi2cosmos_migration.md diff --git a/flume/doc/operation/performance_tuning_tips.md b/doc/operation/performance_tuning_tips.md similarity index 100% rename from flume/doc/operation/performance_tuning_tips.md rename to doc/operation/performance_tuning_tips.md diff --git a/flume/doc/quick_start_guide.md b/doc/quick_start_guide.md similarity index 100% rename from flume/doc/quick_start_guide.md rename to doc/quick_start_guide.md diff --git a/flume/README.md b/flume/README.md deleted file mode 100644 index f9affab40..000000000 --- a/flume/README.md +++ /dev/null @@ -1,565 +0,0 @@ -# Cygnus connector - -This connector is a (conceptual) derivative work of [ngsi2cosmos](https://github.com/telefonicaid/fiware-livedemoapp/tree/master/package/ngsi2cosmos), and implements a Flume-based connector for context data coming from Orion Context Broker and aimed to be stored in a specific persistent storage, such as HDFS, CKAN or MySQL. - -## Design - -All the details about Flume can be found at [flume.apache.org](http://flume.apache.org/), but, as a reminder, some concepts will be explained here: - -* A Flume source is an agent gathering event data from the real source (Twitter stream, a notification system, etc.), either by polling the source or listening for incoming pushes of data. Gathered data is sent to a Flume channel. -* A Flume channel is a passive store (implemented by means of a file, memory, etc.) that holds the event until it is consumed by the Flume sink. -* A Flume sink connects with the final destination of the data (a local file, HDFS, a database, etc.), taking events from the channel and consuming them (processing and/or persisting it). - -There exists a wide collection of already developed sources, channels and sinks. The Flume-based connector, also called Cygnus, development extends that collection by adding: - -* **OrionRestHandler**. A custom HTTP source handler for the default HTTP source. The existing HTTP source behaviour can be governed depending on the request handler associated to it in the configuration. In this case, the custom handler takes care of the method, the target and the headers (specially the Content-Type one) within the request, cheking everything is according to the expected [request format](https://forge.fi-ware.org/plugins/mediawiki/wiki/fiware/index.php/Publish/Subscribe_Broker_-_Orion_Context_Broker_-_User_and_Programmers_Guide#ONCHANGE). This allows for a certain degree of control on the incoming data. The header inspection step allows for a content type identification as well by sending, together with the data, the Content-Type header. -* **OrionHDFSSink**. A custom sink that persists Orion content data in a HDFS deployment. There already exists a native Flume HDFS sink persisting each event in a new file, but this is not suitable for Cygnus. Within Cygnus, the data coming from Orion must be persisted in the Cosmos HDFS in the form of files (a file per entity) containing Json-like lines about the values such entity's attributes have had along time. Several HDFS backends can be used for the data persistence (WebHDFS, HttpFS, Infinity), all of them based on the native WebHDFS REST API from Hadoop. -* **OrionCKANSink**. A custom sink that persists Orion context data in CKAN server instances (see http://docs.ckan.org/en/latest/). -* **OrionMySQLSink**. A custom sink for persisting Orion context data in a MySQL server. Each user owns a database, and each entity is mapped to a table within that database. Tables contain rows about the values such entity's attributes have had along time. - -All these new components (OrionRestHandler, OrionHDFSSink, etc) are combined with other native ones included in Flume itself (e.g. HttpSource), with the purpose of implementing the following data flow: - -1. On behalf of Cygnus, subscribe to Orion for certain context information. -2. Receive from Orion notifications about new update context data; this notification will be handled by the native HttpSource together with the custom OrionRestHandler. -3. Translate the notification into the Flume event format, and put them into the different sink channels (native memory ones). -4. For each enabled custom sink (OrionHDFSSink, OrionCKANSink, OrionMySQLSink), get the notifications from the sink channel and persist the data in the appropriate format. - -## Functionality explained (Json notification example) - -Let's consider the following notification in Json format coming from an Orion Context Broker instance: - - POST http://localhost:1028/notify - Content-Length: 492 - User-Agent: orion/0.9.0 - Host: localhost:1028 - Accept: application/xml, application/json - Content-Type: application/json - Fiware-Service: my-company-name - Fiware-ServicePath: /workingrooms/floor4 - - { - "subscriptionId" : "51c0ac9ed714fb3b37d7d5a8", - "originator" : "localhost", - "contextResponses" : [ - { - "contextElement" : { - "attributes" : [ - { - "name" : "temperature", - "type" : "centigrade", - "value" : "26.5", - "metadatas": [ - { - "name": "ID", - "type": "string", - "value": "ground" - } - ] - } - ], - "type" : "Room", - "isPattern" : "false", - "id" : "Room1" - }, - "statusCode" : { - "code" : "200", - "reasonPhrase" : "OK" - } - } - ] - } - -Such a notification is sent by Orion to the default Flume HTTP source, which relies on the developed OrionRestHandler for checking its validity (that it is a POST request, that the target is 'notify' and that the headers are OK), detecting the content type (that it is in Json format), extracting the data (the Json part) and finally creating a Flume event to be put in the channel: - - event={ - body=json_data, - headers={ - content-type=application/json, - fiware-service=my_company_name, - fiware-servicepath=workingrooms_floor4, - timestamp=1402409899391, - transactionId=asdfasdfsdfa, - ttl=10, - destination=Room1-Room - } - } - -NOTE: The above is an object representation, not Json data nor any other data format. - -Let's have a look on the Flume event headers: - -* The content-type header is a replica of the HTTP header. It is needed for the different sinks to know how to parse the event body. In this case it is JSON. -* Note that Orion can include a `Fiware-Service` HTTP header specifying the tenant/organization associated to the notification, which is added to the event headers as well (as `fiware-service`). Since version 0.3, Cygnus is able to support this header, although the actual processing of such tenant/organization depends on the particular sink. If the notification doesn't include this header, then Cygnus will use the default service specified in the `default_service` configuration property. Please observe that the notified `fiware-service` is transformed following the rules described at [`doc/design/naming_conventions.md`](doc/design/naming_conventions.md). -* Orion can notify another HTTP header, `Fiware-ServicePath` specifying a subservice within a tenant/organization, which is added to the event headers as well (as `fiware-servicepath`). Since version 0.6, Cygnus is able to support this header, although the actual processing of such subservice depends on the particular sink. If the notification doesn't include this header, then Cygnus will use the default service path specified in the `default_service_path` configuration property. Please observe that the notified `fiware-servicePath` is transformed following the rules described at [`doc/design/naming_conventions.md`](doc/design/naming_conventions.md). -* The notification reception time is included in the list of headers (as timestamp) for timestamping purposes in the different sinks. It is added by a native interceptor. See the [doc/design/interceptors.md](doc/design/interceptors.md) document for more details. -* The transactionId identifies a complete Cygnus transaction, starting at the source when the context data is notified, and finishing in the sink, where such data is finally persisted. -* The time-to-live (or ttl) specifies the number of re-injection retries in the channel when something goes wrong while persisting the data. This re-injection mechanism is part of the reliability features of Flume. -1 means inifinite retries. -* The destination headers is used to identify the persistence element within the used storage, i.e. a file in HDFS, a MySQL table or a CKAN resource. This is added by a custom interceptor called `DestinationExtractor` added to the Flume's suite. See the doc/design/interceptors document for more details. - -Finally, the channel is a simple MemoryChannel behaving as a FIFO queue, and from where the different sinks extract the events in order to persist them; let's see how: - -### OrionHDFSSink persistence - -This sink persists the data in files, one per each entity, following this entity descriptor format: - - =-.txt - -These files are stored under this HDFS path: - - hdfs:///user/////.txt - -Usernames allow for specific private HDFS data spaces, and in the current version, it is given by the `cosmos_default_username` parameter that can be found in the configuration. Both the `service` and `servicePath` directories are given by Orion as headers in the notification (`Fiware-Service` and `Fiware-ServicePath` respectively) and sent to the sinks through the Flume event headers (`fiware-service` and `fiware-servicepath` respectively). - -More details regarding the naming conventions can be found at [doc/design/naming_convetions.md](doc/design/naming_convetions.md). - -Within files, Json documents are written following one of these two schemas: - -* Fixed 8-field lines: `recvTimeTs`, `recvTime`, `entityId`, `entityType`, `attrName`, `attrType`, `attrValue` and `attrMd`. Regarding `attrValue`, in its simplest form, this value is just a string, but since Orion 0.11.0 it can be Json object or Json array. Regarding `attrMd`, it contains a string serialization of the metadata array for the attribute in Json (if the attribute hasn't metadata, an empty array `[]` is inserted). -* Two fields per each entity's attribute (one for the value and other for the metadata), plus an additional field about the reception time of the data (`recvTime`). Regarding this kind of persistence, the notifications must ensure a value per each attribute is notified. - -In both cases, the files are created at execution time if the file doesn't exist previously to the line insertion. The behaviour of the connector regarding the internal representation of the data is governed through a configuration parameter, `attr_persistence`, whose values can be `row` or `column`. - -Thus, by receiving a notification like the one above, being the persistence mode `row` and a `default_user` as the default Cosmos username, then the file named `hdfs:///user/default_user/mycompanyname/workingrooms/floor4/Room1-Room/Room1-Room.txt` (it is created if not existing) will contain a new line such as: - - {"recvTimeTs":"13453464536", "recvTime":"2014-02-27T14:46:21", "entityId":"Room1", "entityType":"Room", "attrName":"temperature", "attrType":"centigrade", "attrValue":"26.5", "attrMd":[{name:ID, type:string, value:ground}]} - -On the contrary, being the persistence mode `column`, the file named `hdfs:///user/default_user/mycompanyname/workingrooms/floor4/Room1-Room/Room1-Room.txt` (it is created if not existing) will contain a new line such as: - - {"recvTime":"2014-02-27T14:46:21", "temperature":"26.5", "temperature_md":[{"name":"ID", "type":"string", "value":"ground"}]} - -A special particularity regarding HDFS persisted data is the posssibility to exploit such data through Hive, a SQL-like querying system. OrionHDFSSink automatically creates a Hive table (similar to a SQL table) for each persisted entity in the default database, being the name for such tables: - - ____[row|column] - -Following with the example, by receiving a notification like the one above, and being the persistence mode `row`, the table named `default_user_mycompanyname_workingrooms_floor4_room1_Room_row` will contain a new row such as: - - | recvTimeTs | recvTime | entityId | entityType | attrName | attrType | attrValue | attrMd | - |--------------|---------------------|----------|------------|-------------|------------|-----------|----------------------------------------------------| - | 13453464536 | 2014-02-27T14:46:21 | Room1 | Room | temperature | centigrade | 26.5 | [{"name":"ID", "type":"string", "value":"ground"}] | - -On the contrary, being the persistence mode `column`, the table named `default_user_mycompanyname_workingrooms_floor4_room1_Room_column` will contain a new row such as: - - | recvTime | temperature | temperature_md | - |---------------------|-------------|----------------------------------------------------| - | 2014-02-27T14:46:21 | 26.5 | [{"name":"ID", "type":"string", "value":"ground"}] | - -### OrionCKANSink persistence - -This sink persists the data in a [datastore](see http://docs.ckan.org/en/latest/maintaining/datastore.html) in CKAN. Datastores are associated to CKAN resources and as CKAN resources we use the entityId-entityType string concatenation. All CKAN resource IDs belong to the same dataset (also referred as package in CKAN terms), whose name is specified by the notified `Fiware-ServicePath` header (or by the `default_service_path` property -prefixed by organization name- in the CKAN sink configuration, if such a header is not notified). Datasets belong to single organization, whose name is specified by the notified `Fiware-Service` header (or by the `default_service` property if it is not notified). - -More details regarding the naming conventions can be found at [doc/design/naming_convetions.md](doc/design/naming_convetions.md). - -Each datastore, we can find two options: - -* Fixed 6-field lines: `recvTimeTs`, `recvTime`, `attrName`, `attrType`, `attrValue` and `attrMd`. Regarding `attrValue`, in its simplest form, this value is just a string, but since Orion 0.11.0 it can be JSON object or JSON array. Regarding `attrMd`, it contains a string serialization of the metadata array for the attribute in JSON (if the attribute hasn't metadata, `null` is inserted). -* Two columns per each entity's attribute (one for the value and other for the metadata), plus an additional field about the reception time of the data (`recvTime`). Regarding this kind of persistence, the notifications must ensure a value per each attribute is notified. - -The behaviour of the connector regarding the internal representation of the data is governed through a configuration parameter, `attr_persistence`, whose values can be `row` or `column`. - -Thus, by receiving a notification like the one above, and being the persistence mode `row`, the resource `room1-Room` (it is created if not existing), will containt the following row in its datastore: - - | _id | recvTimeTs | recvTime | attrName | attrType | attrValue | attrMd | - |-----|--------------|---------------------|-----.-------|------------|-----------|-----------------------------------------------------| - | i | 13453464536 | 2014-02-27T14:46:21 | temperature | centigrade | 26.5 | [{"name":"ID", "type":"string", "value":"ground"}] | - -where `i` depends on the number of rows previously inserted. - -On the contrary, being the persistence mode `column`, the resource `Room1-Room` (it and its datastore must be created in advance) will contain a new row such as shown below. In this case, an extra column ended with `_md` is added for the metadata. - - | _id | recvTime | temperature | temperature_md | - |--------------------------|-------------|----------------------------------------------------| - | i |2014-02-27T14:46:21 | 26.5 | [{"name":"ID", "type":"string", "value":"ground"}] | - -where `i` depends on the number of rows previously inserted. - -In both cases, `row` or `column`, the CKAN organization will be `mycompanyname` and the dataset containing the resource will be `workingrooms_floor4`. - -The information stored in the datastore can be accesses as any other CKAN information, e.g. through the web frontend or using the query API, e.g; - - curl -s -S "http://${CKAN_HOST}/api/3/action/datastore_search?resource_id=${RESOURCE_ID} - -Each organization/tenant is associated to a CKAN organization. - -### OrionMySQLSink persistence - -Similarly to OrionHDFSSink, a table is considered for each entity in order to store its notified context data, being the name for these tables the following entity descriptor: - - =__ - -These tables are stored in databases, one per service, enabling a private data space such as: - - jdbc:mysql:/// - -Both the `service` and `servicePath` names are given by Orion as headers in the notification (`Fiware-Service` and `Fiware-ServicePath` respectively) and sent to the sinks through the Flume event headers (`fiware-service` and `fiware-servicepath` respectively). - -More details regarding the naming conventions can be found at [doc/design/naming_convetions.md](doc/design/naming_convetions.md). - -Within tables, we can find two options: - -* Fixed 8-field rows, as usual: `recvTimeTs`, `recvTime`, `entityId`, `entityType`, `attrName`, `attrType`, `attrValue` and `attrMd`. These tables (and the databases) are created at execution time if the table doesn't exist previously to the row insertion. Regarding `attrValue`, in its simplest form, this value is just a string, but since Orion 0.11.0 it can be Json object or Json array. Regarding `attrMd`, it contains a string serialization of the metadata array for the attribute in Json (if the attribute hasn't metadata, an empty array `[]` is inserted), -* Two columns per each entity's attribute (one for the value and other for the metadata), plus an addition column about the reception time of the data (`recv_time`). This kind of tables (and the databases) must be provisioned previously to the execution of Cygnus, because each entity may have a different number of attributes, and the notifications must ensure a value per each attribute is notified. - -The behaviour of the connector regarding the internal representation of the data is governed through a configuration parameter, `attr_persistence`, whose values can be `row` or `column`. - -Thus, by receiving a notification like the one above, and being the persistence mode `row`, the table named `workingrooms_floor4_room1_Room` (it is created if not existing) will contain a new row such as: - - | recvTimeTs | recvTime | entityId | entityType | attrName | attrType | attrValue | attrMd | - |--------------|---------------------|----------|------------|-------------|------------|-----------|----------------------------------------------------| - | 13453464536 | 2014-02-27T14:46:21 | Room1 | Room | temperature | centigrade | 26.5 | [{"name":"ID", "type":"string", "value":"ground"}] | - -On the contrary, being the persistence mode `column`, the table named `workingrooms_floor4_room1_Room` (it must be created in advance) will contain a new row such as: - - | recvTime | temperature | temperature_md | - |---------------------|-------------|----------------------------------------------------| - | 2014-02-27T14:46:21 | 26.5 | [{"name":"ID", "type":"string", "value":"ground"}] | - -Each organization/tenant is associated to a different database. - -## Functionality explained (XML notification example) - -Cygnus also works with [XML-based notifications](https://forge.fi-ware.eu/plugins/mediawiki/wiki/fiware/index.php/Publish/Subscribe_Broker_-_Orion_Context_Broker_-_User_and_Programmers_Guide#ONCHANGE) sent to the connector. The only difference is the event is created by specifying the content type will be XML (in order the notification parser notices it): - - event={ - body=json_data, - headers={ - content-type=application/xml, - fiware-service=my_company_name, - fiware-servicepath=workingrooms_floor4, - timestamp=1402409899391, - transactionId=asdfasdfsdfa, - ttl=10, - destination=Room1-Room - } - } - -The key point is the behaviour remains the same than in the Json example: the same file/datastores/tables will be created, and the same data will be persisted within it. - -## Installing Cygnus -###RPM install (recommended) -Simply configure the FIWARE repository if not yet configured and use your applications manager in order to install the latest version of Cygnus (CentOS/RedHat example): - - $ cat > /etc/yum.repos.d/fiware.repo <.conf` file addressing all those non Flume parameters, such as the Flume agent name, the specific log file for this instance, the administration port, etc. This configuration file is not necessary if Cygnus is run as a standlalone application (see later), bt it is mandatory if run as a service (see later). -* An `agent_.conf` file addressing all those Flume parameters, i.e. how to configure the different sources, channels, sinks, etc. that compose the Flume agent behind the Cygnus instance. always mandatory. - -Please observe there may exist several Cygnus instances identified by ``, which must be the same for both configuration files regarding the same Cygnus instance. - -###`cygnus_instance_.conf` - -The file `cygnus_instance_.conf` can be instantiated from a template given in the Cygnus repository, `conf/cygnus_instance.conf.template`. - -``` -# The OS user that will be running Cygnus. Note this must be `root` if you want to run cygnus in a privileged port (<1024), either the admin port or the port in which Cygnus receives Orion notifications -CYGNUS_USER=cygnus -# Which is the config folder -CONFIG_FOLDER=/usr/cygnus/conf -# Which is the config file -CONFIG_FILE=/usr/cygnus/conf/agent_.conf -# Name of the agent. The name of the agent is not trivial, since it is the base for the Flume parameters naming conventions, e.g. it appears in .sources.http-source.channels=... -AGENT_NAME=cygnusagent -# Name of the logfile located at /var/log/cygnus. It is important to put the extension '.log' in order to the log rotation works properly -LOGFILE_NAME=cygnus.log -# Administration port. Must be unique per instance -ADMIN_PORT=8081 -# Polling interval (seconds) for the configuration reloading -POLLING_INTERVAL=30 -``` - -###`agent_.conf` -A typical configuration when using the `HTTPSource`, the `OrionRestHandler`, the `MemoryChannel` and any of the available sinks is shown below. More advanced configurations can be found at [`doc/operation/performance_tuning_tips.md`](doc/operation/performance_tuning_tips.md). - -Kerberos authentication enabling in HDFS is described at [`doc/operation/hdfs_kerberos_authentication.md`](doc/operation/hdfs_kerberos_authentication.md). If your HDFS is not using such an authentication method, just set `cygnusagent.sinks.hdfs-sink.krb5_auth` to `false` and forget the rest of the Kerberos part. - -The file `agent_.conf` can be instantiated from a template given in the Cygnus repository, `conf/agent.conf.template`. - -```Java -#============================================= -# To be put in APACHE_FLUME_HOME/conf/cygnus.conf -# -# General configuration template explaining how to setup a sink of each of the available types (HDFS, CKAN, MySQL). - -#============================================= -# The next tree fields set the sources, sinks and channels used by Cygnus. You could use different names than the -# ones suggested below, but in that case make sure you keep coherence in properties names along the configuration file. -# Regarding sinks, you can use multiple types at the same time; the only requirement is to provide a channel for each -# one of them (this example shows how to configure 3 sink types at the same time). Even, you can define more than one -# sink of the same type and sharing the channel in order to improve the performance (this is like having -# multi-threading). -cygnusagent.sources = http-source -cygnusagent.sinks = hdfs-sink mysql-sink ckan-sink -cygnusagent.channels = hdfs-channel mysql-channel ckan-channel - -#============================================= -# source configuration -# channel name where to write the notification events -cygnusagent.sources.http-source.channels = hdfs-channel mysql-channel ckan-channel -# source class, must not be changed -cygnusagent.sources.http-source.type = org.apache.flume.source.http.HTTPSource -# listening port the Flume source will use for receiving incoming notifications -cygnusagent.sources.http-source.port = 5050 -# Flume handler that will parse the notifications, must not be changed -cygnusagent.sources.http-source.handler = es.tid.fiware.fiwareconnectors.cygnus.handlers.OrionRestHandler -# URL target -cygnusagent.sources.http-source.handler.notification_target = /notify -# Default service (service semantic depends on the persistence sink) -cygnusagent.sources.http-source.handler.default_service = def_serv -# Default service path (service path semantic depends on the persistence sink) -cygnusagent.sources.http-source.handler.default_service_path = def_servpath -# Number of channel re-injection retries before a Flume event is definitely discarded (-1 means infinite retries) -cygnusagent.sources.http-source.handler.events_ttl = 10 -# Source interceptors, do not change -cygnusagent.sources.http-source.interceptors = ts de -# Interceptor type, do not change -cygnusagent.sources.http-source.interceptors.ts.type = timestamp -# Destination extractor interceptor, do not change -cygnusagent.sources.http-source.interceptors.de.type = es.tid.fiware.fiwareconnectors.cygnus.interceptors.DestinationExtractor$Builder -# Matching table for the destination extractor interceptor, put the right absolute path to the file if necessary -# See the doc/design/interceptors document for more details -cygnusagent.sources.http-source.interceptors.de.matching_table = /usr/cygnus/conf/matching_table.conf - -# ============================================ -# OrionHDFSSink configuration -# channel name from where to read notification events -cygnusagent.sinks.hdfs-sink.channel = hdfs-channel -# sink class, must not be changed -cygnusagent.sinks.hdfs-sink.type = es.tid.fiware.fiwareconnectors.cygnus.sinks.OrionHDFSSink -# Comma-separated list of FQDN/IP address regarding the Cosmos Namenode endpoints -# If you are using Kerberos authentication, then the usage of FQDNs instead of IP addresses is mandatory -cygnusagent.sinks.hdfs-sink.cosmos_host = x1.y1.z1.w1,x2.y2.z2.w2 -# port of the Cosmos service listening for persistence operations; 14000 for httpfs, 50070 for webhdfs and free choice for inifinty -cygnusagent.sinks.hdfs-sink.cosmos_port = 14000 -# default username allowed to write in HDFS -cygnusagent.sinks.hdfs-sink.cosmos_default_username = cosmos_username -# default password for the default username -cygnusagent.sinks.hdfs-sink.cosmos_default_password = xxxxxxxxxxxxx -# HDFS backend type (webhdfs, httpfs or infinity) -cygnusagent.sinks.hdfs-sink.hdfs_api = httpfs -# how the attributes are stored, either per row either per column (row, column) -cygnusagent.sinks.hdfs-sink.attr_persistence = column -# Hive FQDN/IP address of the Hive server -cygnusagent.sinks.hdfs-sink.hive_host = x.y.z.w -# Hive port for Hive external table provisioning -cygnusagent.sinks.hdfs-sink.hive_port = 10000 -# Kerberos-based authentication enabling -cygnusagent.sinks.hdfs-sink.krb5_auth = false -# Kerberos username -cygnusagent.sinks.hdfs-sink.krb5_auth.krb5_user = krb5_username -# Kerberos password -cygnusagent.sinks.hdfs-sink.krb5_auth.krb5_password = xxxxxxxxxxxxx -# Kerberos login file -cygnusagent.sinks.hdfs-sink.krb5_auth.krb5_login_conf_file = /usr/cygnus/conf/krb5_login.conf -# Kerberos configuration file -cygnusagent.sinks.hdfs-sink.krb5_auth.krb5_conf_file = /usr/cygnus/conf/krb5.conf - -# ============================================ -# OrionCKANSink configuration -# channel name from where to read notification events -cygnusagent.sinks.ckan-sink.channel = ckan-channel -# sink class, must not be changed -cygnusagent.sinks.ckan-sink.type = es.tid.fiware.fiwareconnectors.cygnus.sinks.OrionCKANSink -# the CKAN API key to use -cygnusagent.sinks.ckan-sink.api_key = ckanapikey -# the FQDN/IP address for the CKAN API endpoint -cygnusagent.sinks.ckan-sink.ckan_host = x.y.z.w -# the port for the CKAN API endpoint -cygnusagent.sinks.ckan-sink.ckan_port = 80 -# Orion URL used to compose the resource URL with the convenience operation URL to query it -cygnusagent.sinks.ckan-sink.orion_url = http://localhost:1026 -# how the attributes are stored, either per row either per column (row, column) -cygnusagent.sinks.ckan-sink.attr_persistence = row -# enable SSL for secure Http transportation; 'true' or 'false' -cygnusagent.sinks.ckan-sink.ssl = false - -# ============================================ -# OrionMySQLSink configuration -# channel name from where to read notification events -cygnusagent.sinks.mysql-sink.channel = mysql-channel -# sink class, must not be changed -cygnusagent.sinks.mysql-sink.type = es.tid.fiware.fiwareconnectors.cygnus.sinks.OrionMySQLSink -# the FQDN/IP address where the MySQL server runs -cygnusagent.sinks.mysql-sink.mysql_host = x.y.z.w -# the port where the MySQL server listes for incomming connections -cygnusagent.sinks.mysql-sink.mysql_port = 3306 -# a valid user in the MySQL server -cygnusagent.sinks.mysql-sink.mysql_username = root -# password for the user above -cygnusagent.sinks.mysql-sink.mysql_password = xxxxxxxxxxxx -# how the attributes are stored, either per row either per column (row, column) -cygnusagent.sinks.mysql-sink.attr_persistence = column - -#============================================= -# hdfs-channel configuration -# channel type (must not be changed) -cygnusagent.channels.hdfs-channel.type = memory -# capacity of the channel -cygnusagent.channels.hdfs-channel.capacity = 1000 -# amount of bytes that can be sent per transaction -cygnusagent.channels.hdfs-channel.transactionCapacity = 100 - -#============================================= -# ckan-channel configuration -# channel type (must not be changed) -cygnusagent.channels.ckan-channel.type = memory -# capacity of the channel -cygnusagent.channels.ckan-channel.capacity = 1000 -# amount of bytes that can be sent per transaction -cygnusagent.channels.ckan-channel.transactionCapacity = 100 - -#============================================= -# mysql-channel configuration -# channel type (must not be changed) -cygnusagent.channels.mysql-channel.type = memory -# capacity of the channel -cygnusagent.channels.mysql-channel.capacity = 1000 -# amount of bytes that can be sent per transaction -cygnusagent.channels.mysql-channel.transactionCapacity = 100 -``` - -## Running as a service (recommended) -NOTE: Cygnus can only be run as a service if you installed it through the RPM. - -Once the `cygnus_instance_.conf` and `agent_.conf` files are properly configured, just use the `service` command to start, restart, stop or get the status (as a sudoer): - - $ sudo service cygnus status - - $ sudo service cygnus start - - $ sudo service cygnus restart - - $ sudo service cygnus stop - -Previous commands afefcts to **all** of Cygnus instances configured. If only one instance is wanted to be managed by the service script then the instance identifier after de the action must be specified: - - $ sudo service cygnus status - - $ sudo service cygnus start - - $ sudo service cygnus restart - - $ sudo service cygnus stop - -Where `` is the suffix at the end of the `cygnus_instace_.conf` or `agent_.conf` files you used to configure the instance. - -## Running as standalone application (advanced) - -NOTE: If you installed Cygnus through the RPM, APACHE\_FLUME\_HOME is `/usr/cygnus/`. If not, it is a directory of your choice. - -Cygnus implements its own startup script, `cygnus-flume-ng` which replaces the standard `flume-ng` one, which in the end runs a custom `es.tid.fiware.fiwareconnectors.cygnus.nodes.CygnusApplication` instead of a standard `org.apache.flume.node.Application`. - -In foreground (with logging): - - $ APACHE_FLUME_HOME/bin/cygnus-flume-ng agent --conf APACHE_FLUME_HOME/conf -f APACHE_FLUME_HOME/conf/cygnus.conf -n cygnusagent -Dflume.root.logger=INFO,console [-p ] [-t ] - -In background: - - $ nohup APACHE_FLUME_HOME/bin/cygnus-flume-ng agent --conf APACHE_FLUME_HOME/conf -f APACHE_FLUME_HOME/conf/cygnus.conf -n cygnusagent -Dflume.root.logger=INFO,LOGFILE [-p ] [-t ] & - -The parameters used in these commands are: - -* `agent`. This is the type of application to be run by the `cygnus-flume-ng` script. -* `--conf`. Points to the Apache Flume configuration folder. -* `-f` (or `--conf-file`). This is the agent configuration (`agent_.conf`) file. Please observe when running in this mode no `cygnus_instance_.conf` file is required. -* `-n` (or `--name`). The name of the Flume agent to be run. -* `-Dflume.root.logger`. Changes the logging level and the logging appender for log4j. -* `-p` (or `--mgmt-if-port`). Configures the listening port for the Management Interface. If not configured, the default value is used, `8081`. -* `-t` (or `--polling-interval`). Configures the polling interval (seconds) when the configuration is periodically reloaded. If not configured, the default value is used, `30`. - -## Orion subscription - -Once the connector is running, it is necessary to tell Orion Context Broker about it, in order Orion can send context data notifications to the connector. This can be done on behalf of the connector by performing the following curl command: - - $ (curl localhost:1026/NGSI10/subscribeContext -s -S --header 'Content-Type: application/xml' -d @- | xmllint --format -) < - - - - Room1 - - - - temperature - - - http://host_running_cygnus:5050/notify - P1M - - - ONCHANGE - - pressure - - - - PT5S - - EOF - -Its equivalent in Json format can be seen [here](https://forge.fi-ware.eu/plugins/mediawiki/wiki/fiware/index.php/Publish/Subscribe_Broker_-_Orion_Context_Broker_-_User_and_Programmers_Guide#ONCHANGE). - -## Logs - -###log4j configuration - -Cygnus uses the log4j facilities added by Flume for logging purposes. You can maintain the default `APACHE_FLUME_HOME/conf/log4j.properties` file, where a console and a file appender are defined (in addition, the console is used by default), or customize it by adding new appenders. Typically, you will have several instances of Cygnus running; they will be listening on different TCP ports for incoming notifyContextRequest and you'll probably want to have differente log files for them. E.g., if you have two Flume processes listening on TCP/1028 and TCP/1029 ports, then you can add the following lines to the `log4j.properties` file: - - log4j.appender.cygnus1028=org.apache.log4j.RollingFileAppender - log4j.appender.cygnus1028.MaxFileSize=100MB - log4j.appender.cygnus1028.MaxBackupIndex=10 - log4j.appender.cygnus1028.File=${flume.log.dir}/cygnus.1028.log - log4j.appender.cygnus1028.layout=org.apache.log4j.PatternLayout - log4j.appender.cygnus1028.layout.ConversionPattern=time=%d{yyyy-MM-dd}T%d{HH:mm:ss.SSSzzz} | lvl=%p | trans=%X{transactionId} | function=%M | comp=Cygnus | msg=%C[%L] : %m%n - - log4j.appender.cygnus1029=org.apache.log4j.RollingFileAppender - log4j.appender.cygnus1029.MaxFileSize=100MB - log4j.appender.cygnus1029.MaxBackupIndex=10 - log4j.appender.cygnus1029.File=${flume.log.dir}/cygnus.1029.log - log4j.appender.cygnus1029.layout=org.apache.log4j.PatternLayout - log4j.appender.cygnus1029.layout.ConversionPattern=time=%d{yyyy-MM-dd}T%d{HH:mm:ss.SSSzzz} | lvl=%p | trans=%X{transactionId} | function=%M | comp=Cygnus | msg=%C[%L] : %m%n - -Regarding the log4j Conversion Pattern: - -* `time` makes reference to a timestamp following the [RFC3339](http://tools.ietf.org/html/rfc3339). -* `lvl`means logging level, and matches the traditional log4j levels: `INFO`, `WARN`, `ERROR`, `FATAL` and `DEBUG`. -* `trans` is a transaction identifier, i.e. an identifier that is printed in all the traces related to the same Orion notification. The format is `--<10_digits_transaction_count>`. Its generation logic ensures that every transaction identifier is unique, also for Cygnus instances running in different VMs, except if they are started in the exactly same millisecond (highly unprobable). -* `function` identifies the function/method within the class printing the log. -* `comp` is always `Cygnus`. -* `msg` is a custom message that has always the same format: `[] : `. - -Once the log4j has been properly configured, you only have to add to the Flume command line the following parameter, which overwrites the default configutation (`flume.root.logger=INFO,LOGFILE`): - - -Dflume.root.logger=,cygnus..log - -In addition, you have a complete `log4j.properties` template in `conf/log4j.properties.template`, once you clone the Cygnus repository. - -### Message types - -Check [doc/operation/alarms.md](doc/operation/alarms.md) for a detailed list of message types. - -## Management interface - -From Cygnus 0.5 there is a REST-based management interface for administration purposes. Current available operations are: - -Get the version of the running software, including the last Git commit: - - GET http://host:management_port/version - - {"version":"0.5_SNAPSHOT.8a6c07054da894fc37ef30480cb091333e2fccfa"} - -## Contact - -* Fermín Galán Márquez (fermin.galanmarquez@telefonica.com). -* Francisco Romero Bueno (francisco.romerobueno@telefonica.com). diff --git a/flume/neore/README.md b/neore/README.md similarity index 100% rename from flume/neore/README.md rename to neore/README.md diff --git a/flume/neore/rpm/SOURCES/cron.d/cleanup_old_cygnus_logfiles b/neore/rpm/SOURCES/cron.d/cleanup_old_cygnus_logfiles similarity index 100% rename from flume/neore/rpm/SOURCES/cron.d/cleanup_old_cygnus_logfiles rename to neore/rpm/SOURCES/cron.d/cleanup_old_cygnus_logfiles diff --git a/flume/neore/rpm/SOURCES/init.d/cygnus b/neore/rpm/SOURCES/init.d/cygnus similarity index 100% rename from flume/neore/rpm/SOURCES/init.d/cygnus rename to neore/rpm/SOURCES/init.d/cygnus diff --git a/flume/neore/rpm/SOURCES/logrotate.d/logrotate-cygnus-daily b/neore/rpm/SOURCES/logrotate.d/logrotate-cygnus-daily similarity index 100% rename from flume/neore/rpm/SOURCES/logrotate.d/logrotate-cygnus-daily rename to neore/rpm/SOURCES/logrotate.d/logrotate-cygnus-daily diff --git a/flume/neore/rpm/SPECS/cygnus.spec b/neore/rpm/SPECS/cygnus.spec similarity index 100% rename from flume/neore/rpm/SPECS/cygnus.spec rename to neore/rpm/SPECS/cygnus.spec diff --git a/flume/neore/rpm/get_version_string.sh b/neore/rpm/get_version_string.sh similarity index 100% rename from flume/neore/rpm/get_version_string.sh rename to neore/rpm/get_version_string.sh diff --git a/flume/neore/scripts/check_rpm.sh b/neore/scripts/check_rpm.sh similarity index 100% rename from flume/neore/scripts/check_rpm.sh rename to neore/scripts/check_rpm.sh diff --git a/flume/neore/scripts/colors_shell.sh b/neore/scripts/colors_shell.sh similarity index 100% rename from flume/neore/scripts/colors_shell.sh rename to neore/scripts/colors_shell.sh diff --git a/flume/neore/scripts/package.sh b/neore/scripts/package.sh similarity index 100% rename from flume/neore/scripts/package.sh rename to neore/scripts/package.sh diff --git a/flume/pom.xml b/pom.xml similarity index 100% rename from flume/pom.xml rename to pom.xml diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANBackend.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANBackend.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANBackend.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANBackend.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANBackendImpl.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANBackendImpl.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANBackendImpl.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANBackendImpl.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANCache.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANCache.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANCache.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANCache.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANRequester.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANRequester.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANRequester.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANRequester.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANResponse.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANResponse.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANResponse.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANResponse.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hdfs/HDFSBackend.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hdfs/HDFSBackend.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hdfs/HDFSBackend.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hdfs/HDFSBackend.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hdfs/HDFSBackendImpl.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hdfs/HDFSBackendImpl.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hdfs/HDFSBackendImpl.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hdfs/HDFSBackendImpl.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hive/HiveBackend.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hive/HiveBackend.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hive/HiveBackend.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hive/HiveBackend.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/mysql/MySQLBackend.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/mysql/MySQLBackend.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/mysql/MySQLBackend.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/backends/mysql/MySQLBackend.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channels/CygnusChannel.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channels/CygnusChannel.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channels/CygnusChannel.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channels/CygnusChannel.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channels/CygnusFileChannel.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channels/CygnusFileChannel.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channels/CygnusFileChannel.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channels/CygnusFileChannel.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channels/CygnusMemoryChannel.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channels/CygnusMemoryChannel.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channels/CygnusMemoryChannel.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channels/CygnusMemoryChannel.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channelselectors/RoundRobinChannelSelector.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channelselectors/RoundRobinChannelSelector.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channelselectors/RoundRobinChannelSelector.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/channelselectors/RoundRobinChannelSelector.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/containers/NotifyContextRequest.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/containers/NotifyContextRequest.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/containers/NotifyContextRequest.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/containers/NotifyContextRequest.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/containers/NotifyContextRequestSAXHandler.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/containers/NotifyContextRequestSAXHandler.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/containers/NotifyContextRequestSAXHandler.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/containers/NotifyContextRequestSAXHandler.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusBadConfiguration.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusBadConfiguration.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusBadConfiguration.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusBadConfiguration.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusBadContextData.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusBadContextData.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusBadContextData.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusBadContextData.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusPersistenceError.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusPersistenceError.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusPersistenceError.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusPersistenceError.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusRuntimeError.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusRuntimeError.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusRuntimeError.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/errors/CygnusRuntimeError.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/handlers/OrionRestHandler.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/handlers/OrionRestHandler.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/handlers/OrionRestHandler.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/handlers/OrionRestHandler.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/http/HttpClientFactory.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/http/HttpClientFactory.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/http/HttpClientFactory.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/http/HttpClientFactory.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/http/JettyServer.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/http/JettyServer.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/http/JettyServer.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/http/JettyServer.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/interceptors/DestinationExtractor.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/interceptors/DestinationExtractor.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/interceptors/DestinationExtractor.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/interceptors/DestinationExtractor.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/log/CygnusLogger.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/log/CygnusLogger.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/log/CygnusLogger.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/log/CygnusLogger.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/management/ManagementInterface.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/management/ManagementInterface.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/management/ManagementInterface.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/management/ManagementInterface.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/nodes/CygnusApplication.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/nodes/CygnusApplication.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/nodes/CygnusApplication.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/nodes/CygnusApplication.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionCKANSink.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionCKANSink.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionCKANSink.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionCKANSink.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionHDFSSink.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionHDFSSink.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionHDFSSink.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionHDFSSink.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionMySQLSink.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionMySQLSink.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionMySQLSink.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionMySQLSink.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionSink.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionSink.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionSink.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionSink.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionTestSink.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionTestSink.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionTestSink.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionTestSink.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/utils/Constants.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/utils/Constants.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/utils/Constants.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/utils/Constants.java diff --git a/flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/utils/Utils.java b/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/utils/Utils.java similarity index 100% rename from flume/src/main/java/es/tid/fiware/fiwareconnectors/cygnus/utils/Utils.java rename to src/main/java/es/tid/fiware/fiwareconnectors/cygnus/utils/Utils.java diff --git a/flume/src/main/resources/cygnus-flume-ng b/src/main/resources/cygnus-flume-ng similarity index 100% rename from flume/src/main/resources/cygnus-flume-ng rename to src/main/resources/cygnus-flume-ng diff --git a/flume/src/main/resources/pom.properties b/src/main/resources/pom.properties similarity index 100% rename from flume/src/main/resources/pom.properties rename to src/main/resources/pom.properties diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANBackendImplTest.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANBackendImplTest.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANBackendImplTest.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANBackendImplTest.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANCacheTest.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANCacheTest.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANCacheTest.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANCacheTest.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANRequesterTest.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANRequesterTest.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANRequesterTest.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANRequesterTest.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANResponseTest.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANResponseTest.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANResponseTest.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/ckan/CKANResponseTest.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hdfs/HDFSBackendImplTest.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hdfs/HDFSBackendImplTest.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hdfs/HDFSBackendImplTest.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/hdfs/HDFSBackendImplTest.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/mysql/MySQLBackendTest.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/mysql/MySQLBackendTest.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/mysql/MySQLBackendTest.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/backends/mysql/MySQLBackendTest.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/channelselectors/RoundRobinChannelSelectorTest.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/channelselectors/RoundRobinChannelSelectorTest.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/channelselectors/RoundRobinChannelSelectorTest.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/channelselectors/RoundRobinChannelSelectorTest.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/containers/NotifyContextRequestTest.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/containers/NotifyContextRequestTest.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/containers/NotifyContextRequestTest.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/containers/NotifyContextRequestTest.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/handlers/OrionRestHandlerTest.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/handlers/OrionRestHandlerTest.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/handlers/OrionRestHandlerTest.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/handlers/OrionRestHandlerTest.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/http/JettyServerTest.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/http/JettyServerTest.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/http/JettyServerTest.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/http/JettyServerTest.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/interceptors/DestinationExtractorTest.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/interceptors/DestinationExtractorTest.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/interceptors/DestinationExtractorTest.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/interceptors/DestinationExtractorTest.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/management/ManagementInterfaceTest.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/management/ManagementInterfaceTest.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/management/ManagementInterfaceTest.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/management/ManagementInterfaceTest.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionCKANSinkTest.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionCKANSinkTest.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionCKANSinkTest.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionCKANSinkTest.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionHDFSSinkTest.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionHDFSSinkTest.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionHDFSSinkTest.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionHDFSSinkTest.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionMySQLSinkTest.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionMySQLSinkTest.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionMySQLSinkTest.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/sinks/OrionMySQLSinkTest.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/utils/TestConstants.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/utils/TestConstants.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/utils/TestConstants.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/utils/TestConstants.java diff --git a/flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/utils/TestUtils.java b/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/utils/TestUtils.java similarity index 100% rename from flume/src/test/java/es/tid/fiware/fiwareconnectors/cygnus/utils/TestUtils.java rename to src/test/java/es/tid/fiware/fiwareconnectors/cygnus/utils/TestUtils.java diff --git a/flume/telefonica_checkstyle.xml b/telefonica_checkstyle.xml similarity index 100% rename from flume/telefonica_checkstyle.xml rename to telefonica_checkstyle.xml