Initial commit

.dockerignore

@@ -0,0 +1,4 @@
+env
+.dockerignore
+Dockerfile
+Dockerfile.prod

.gitignore

@@ -0,0 +1,6 @@
+__pycache__
+env
+*.env
+.vscode
+*.pem
+venv

README.md

@@ -0,0 +1,19 @@
+To bring up service:
+
+Note: the environment variables file config.env must be at root folder and the ssl root cert (pem file) should be in folder /platform_in
+
+dev config (flask dev server):
+
+docker-compose up
+send POST requests to localhost:5000/TODO/v1/ with xml data
+should return success or failure response
+prod config (nginx+gunicorn)
+
+docker-compose -f docker-compose.prod.yml up
+2,3,4 same as previous but port for prod is 1337
+
+example POST data:
+
+```
+TODO
+```

docker-compose.prod.yml

@@ -0,0 +1,27 @@
+version: '3'
+
+services:
+
+  charging_in_api:
+    build:
+        context: ./platform_in/
+        dockerfile: Dockerfile.prod
+    command: gunicorn --bind 0.0.0.0:5002 manage:app
+    restart: always
+    #volumes:
+    #  - ./platform_in/:/project/
+    expose:
+      - 5002
+    env_file:
+      - app_config.env
+    environment:
+      - FLASK_APP=app/__init__.py
+      - FLASK_ENV=production
+      - APP_SETTINGS=app.config.ProductionConfig
+
+  nginx:
+    build: ./nginx
+    ports:
+      - 1337:80
+    depends_on:
+      - charging_in_api

docker-compose.yml

@@ -0,0 +1,19 @@
+version: '3'
+
+services:
+
+  charging_in_api:
+    build: ./platform_in/
+    command: python manage.py run -h 0.0.0.0
+    volumes:
+      - ./platform_in/:/project/
+    ports:
+      - 5002:5000
+    env_file:
+      - var.env
+    environment:
+      - FLASK_APP=app/__init__.py
+      - FLASK_ENV=development
+      - APP_SETTINGS=app.config.DevelopmentConfig
+
+

nginx/Dockerfile

@@ -0,0 +1,4 @@
+FROM nginx:1.17-alpine
+
+RUN rm /etc/nginx/conf.d/default.conf
+COPY nginx.conf /etc/nginx/conf.d

nginx/nginx.conf

@@ -0,0 +1,16 @@
+upstream ptf_incoming_api {
+    server charging_in_api:5002;
+}
+
+server {
+
+    listen 80;
+
+    location / {
+        proxy_pass http://ptf_incoming_api;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+    }
+
+}

platform_in/.gitignore

@@ -0,0 +1,6 @@
+env/
+.vscode
+*pyc
+migrations/
+pycache
+

platform_in/Dockerfile

@@ -0,0 +1,19 @@
+# pull official base image
+FROM python:3.8.0-alpine
+
+WORKDIR /project
+
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+COPY requirements.txt .
+
+RUN apk update  && \
+  apk add --no-cache postgresql-libs postgresql-dev ca-certificates gcc musl-dev libressl-dev libc-dev librdkafka librdkafka-dev python-dev netcat-openbsd bash && \
+  python3 -m pip install -r requirements.txt --no-cache-dir 
+
+COPY chain.pem .
+RUN cat chain.pem >> /usr/local/lib/python3.8/site-packages/certifi/cacert.pem
+
+COPY . /project/
+

platform_in/Dockerfile.prod

@@ -0,0 +1,38 @@
+# pull official base image
+FROM python:3.8.0-alpine
+
+# set working directory
+WORKDIR /project
+
+# set environment varibles
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+ENV FLASK_ENV production
+ENV APP_SETTINGS project.config.ProductionConfig
+
+# add and install requirements
+COPY requirements.txt .
+
+RUN apk update  && \
+  apk add --no-cache postgresql-libs postgresql-dev ca-certificates gcc musl-dev libressl-dev libc-dev librdkafka librdkafka-dev python-dev netcat-openbsd bash && \
+  python3 -m pip install -r requirements.txt --no-cache-dir
+
+COPY chain.pem .
+RUN cat chain.pem >> /usr/local/lib/python3.8/site-packages/certifi/cacert.pem
+
+COPY . /project/
+
+ # add and run as non-root user
+RUN addgroup -S app && adduser -S app -G app
+RUN chown -R app:app /project
+
+# # lint
+# RUN pip install --upgrade pip
+# RUN pip install flake8
+# COPY . .
+# RUN flake8 --ignore=E501,F401 .
+
+# change to the app user
+USER app
+CMD gunicorn --bind 0.0.0.0:5000 manage:app
+

platform_in/app/__init__.py

@@ -0,0 +1,119 @@
+from flask import Flask
+import os
+import sys
+from flask import jsonify
+from elasticapm.contrib.flask import ElasticAPM
+import logging
+from flask import jsonify, request
+import json
+from confluent_kafka import avro
+from confluent_kafka.avro import AvroProducer, Producer
+import xmltodict
+import certifi
+
+logging.basicConfig(level=logging.INFO)
+#elastic_apm = ElasticAPM()
+# print(app.config, file=sys.stderr)
+
+success_response_object = {"status":"success"}
+success_code = 202
+failure_response_object = {"status":"failure"}
+failure_code = 400
+
+def delivery_report(err, msg):
+    if err is not None:
+        logging.error(f"Message delivery failed: {err}")
+    else:
+        logging.debug(f"Message delivered to {msg.topic()} [{msg.partition()}]")
+
+def kafka_avro_produce(avroProducer, topic, data):
+
+    try:
+        avroProducer.produce(topic=topic, value=data)
+        logging.debug("avro produce")
+        avroProducer.poll(2)
+        if len(avroProducer) != 0:
+            return False
+    except BufferError:
+        logging.error("local buffer full", len(avroProducer))
+        return False
+    except Exception as e:
+        logging.error(e)
+        return False
+
+    return True
+
+def create_app(script_info=None):
+
+    # instantiate the app
+    app = Flask(__name__)
+
+    # set config
+    app_settings = os.getenv("APP_SETTINGS")
+    app.config.from_object(app_settings)
+
+    # set up extensions
+    #elastic_apm.init_app(app)
+
+    # value_schema = avro.load("avro/vehiclecharging.avsc")
+    # avroProducer = AvroProducer(
+    #     {
+    #         "bootstrap.servers": app.config["KAFKA_BROKERS"],
+    #         "security.protocol": app.config["SECURITY_PROTOCOL"],
+    #         "sasl.mechanism": app.config["SASL_MECHANISM"],
+    #         "sasl.username": app.config["SASL_UNAME"],
+    #         "sasl.password": app.config["SASL_PASSWORD"],
+    #         "ssl.ca.location": certifi.where(),
+    #         #"debug": "security,cgrp,fetch,topic,broker,protocol",
+    #         "on_delivery": delivery_report,
+    #         "schema.registry.url": app.config["SCHEMA_REGISTRY_URL"] ,
+    #     },
+    #     default_value_schema=value_schema,
+    # )
+
+    producer = Producer(
+        {
+            "bootstrap.servers": app.config["KAFKA_BROKERS"],
+            "security.protocol": app.config["SECURITY_PROTOCOL"],
+            "sasl.mechanism": app.config["SASL_MECHANISM"],
+            "sasl.username": app.config["SASL_UNAME"],
+            "sasl.password": app.config["SASL_PASSWORD"],
+            "ssl.ca.location": certifi.where(),
+            #"debug": "security,cgrp,fetch,topic,broker,protocol",
+            "on_delivery": delivery_report
+        }
+    )
+
+    # shell context for flask cli
+    @app.shell_context_processor
+    def ctx():
+        return {"app": app}
+
+    @app.route("/")
+    def hello_world():
+        return jsonify(health="ok")
+
+    @app.route("/ocpp/v16/observations", methods=["POST"])
+    def post_vehiclecharge_data():
+        try:
+            data = request.get_data()
+            #logging.info(f"post observation: {data}")
+
+            # topic =
+            # kafka_avro_produce(avroProducer, topic, data_dict)
+            producer.poll(0)
+
+        # Asynchronously produce a message, the delivery report callback
+        # will be triggered from poll() above, or flush() below, when the message has
+        # been successfully delivered or failed permanently.
+            producer.produce('test.finest.rawdata.vehiclecharging.ocpp', data, callback=delivery_report)
+
+            return success_response_object,success_code
+
+        except Exception as e:
+            producer.flush()
+            logging.error("post data error", e)
+            #elastic_apm.capture_exception()
+
+    return app
+

platform_in/app/config.py

@@ -0,0 +1,53 @@
+import os
+
+basedir = os.path.abspath(os.path.dirname(__file__))
+
+
+def get_env_variable(name):
+    try:
+        return os.environ[name]
+    except KeyError:
+        message = "Expected environment variable '{}' not set.".format(name)
+        raise Exception(message)
+
+
+
+class Config(object):
+
+    SECRET_KEY = os.environ.get("SECRET_KEY") or "super-secret-key"
+    DEBUG = True
+    CSRF_ENABLED = True
+
+    SASL_UNAME = get_env_variable("SASL_UNAME")
+    SASL_PASSWORD = get_env_variable("SASL_PASSWORD")
+    KAFKA_BROKERS = get_env_variable("KAFKA_BROKERS")
+    SECURITY_PROTOCOL = get_env_variable("SECURITY_PROTOCOL")
+    SASL_MECHANISM = get_env_variable("SASL_MECHANISM")
+    SCHEMA_REGISTRY_URL = get_env_variable("SCHEMA_REGISTRY_URL")
+
+    ELASTIC_APM = {
+        'SERVICE_NAME': get_env_variable("ELASTIC_SERVICE_NAME"),
+        'SECRET_TOKEN': get_env_variable("ELASTIC_SECRET_TOKEN"),
+        'SERVER_URL': get_env_variable("ELASTIC_SERVER_URL"),
+        'DEBUG':True
+    }
+
+
+
+class ProductionConfig(Config):
+    DEBUG = False
+    SECRET_KEY = os.environ.get("SECRET_KEY") or "prod-secret-key"
+
+class DevelopmentConfig(Config):
+    DEVELOPMENT = True
+    TESTING = False
+    DEBUG = True
+    SECRET_KEY = os.environ.get("SECRET_KEY") or "dev-secret-key"
+    #ELASTIC_APM['DEBUG']=True
+
+
+class TestingConfig(Config):
+    TESTING = True
+    SECRET_KEY = os.environ.get("SECRET_KEY") or "test-secret-key"
+
+

platform_in/avro/vehiclecharging.avsc

@@ -0,0 +1,6 @@
+{
+    "name": "vehiclechargindata",
+    "type": "record",
+    "namespace": "fvh.vehiclecharging.avro",
+    "fields": []
+}

platform_in/manage.py

@@ -0,0 +1,11 @@
+import sys
+
+from flask.cli import FlaskGroup
+
+from app import create_app   
+
+app = create_app()   
+cli = FlaskGroup(create_app=create_app)   
+
+if __name__ == '__main__':
+    cli()

platform_in/requirements.txt

@@ -0,0 +1,16 @@
+
+wheel
+avro-python3==1.9.1
+confluent-kafka==1.3.0
+requests
+confluent-kafka[avro,jsonschema,protobuf]
+elastic-apm
+Flask
+Flask-RESTful
+Flask-Security
+gunicorn
+Jinja2
+pytest
+requests
+xmltodict
+certifi