Merge pull request #32 from GeographicaGS/danibishop/longitude-rework

Danibishop/longitude rework

.coveragerc

@@ -0,0 +1,30 @@
+# .coveragerc to control coverage.py
+[run]
+branch = True
+omit = src/core/tests/*, **/__init__.py, src/samples/**
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:
+
+ignore_errors = True
+fail_under = 85
+show_missing = True
+
+[html]
+title = Longitude Core Coverage
+directory = coverage_html_report

.env.sample

@@ -0,0 +1,14 @@
+# Remote db
+CARTO_API_KEY=
+CARTO_USER=
+CARTO_TABLE=
+
+# Local db
+POSTGRES_HOST=localhost
+POSTGRES_PORT=5432
+POSTGRES_DB=longitude
+POSTGRES_USER=longitude
+POSTGRES_PASS=longitude
+
+# Cache
+REDIS_PASSWORD=longitude

.gitignore

@@ -114,3 +114,11 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
+# VScode IDE
+.vscode
+
+# PyCharm IDE
+.idea
+
+# Coverage report
+coverage_html_report

Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.6.6-slim
+
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /usr/src/app
+ENV PATH="$PATH:/usr/src/app"
+
+
+# Install anything missing in the slim image, install dependencies
+# Remove anything only needed for building
+# This is run as one line so docker caches it as a single layer.
+
+COPY pyproject.toml .
+
+RUN set -x \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends git gcc curl \
+    && curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python \
+    && $HOME/.poetry/bin/poetry install \
+    && apt-get remove -y --purge git gcc curl \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY . .

Jenkinsfile

@@ -0,0 +1,127 @@
+#!/usr/bin/env groovy
+
+// Global Environment variables
+FAILURE_EMAIL = "[email protected]"
+DESIRED_REPOSITORY = "https://github.com/GeographicaGS/Longitude.git"
+PUBLISH_BRANCH = "publish"
+REPO_NAME = "longitude"
+
+pipeline{
+  agent { node {
+    label 'master'
+  } }
+
+  options {
+    ansiColor('xterm')
+  }
+
+  stages {
+    stage('Preparing for build') {
+      agent { node {
+        label 'master'
+      } }
+      steps {
+        prepareBuild()
+      }
+    }
+    stage ('Building') {
+      agent { node {
+        label 'docker'
+      } }
+      steps {
+        sh "docker build --pull=true -t geographica/${REPO_NAME}:${git_commit} ."
+      }
+    }
+    stage('Linter')
+    {
+      agent { node {
+        label 'docker'
+      } }
+      steps {
+        sh "docker run --rm geographica/${REPO_NAME}:${git_commit} /root/.poetry/bin/poetry run pylint --ignore=samples -E longitude"
+      }
+    }
+    stage('Testing')
+    {
+      agent { node {
+        label 'docker'
+      } }
+      steps {
+        sh "docker run --rm geographica/${REPO_NAME}:${git_commit} /root/.poetry/bin/poetry run pytest --cov=longitude.core longitude/core/tests/"
+      }
+    }
+    stage ('Publish') {
+      agent { node {
+        label 'docker'
+      } }
+      when { anyOf {
+        branch "${PUBLISH_BRANCH}"
+      } }
+      steps{
+        // TODO: this must be "publish" but we keep "build" while testing the Jenkins pipeline
+        sh "docker run --rm geographica/${REPO_NAME}:${git_commit} /root/.poetry/bin/poetry build"
+      }
+    }
+    // TODO: Stage to check that module can be imported
+  }
+  post {
+    always {
+      deleteDir() /* clean up our workspace */
+    }
+    unstable {
+      notifyStatus(currentBuild.currentResult)
+    }
+    failure {
+      notifyStatus(currentBuild.currentResult)
+    }
+  }
+}
+
+def prepareBuild() {
+  script {
+    checkout scm
+
+    sh "git rev-parse --short HEAD > .git/git_commit"
+    sh "git --no-pager show -s --format='%ae' HEAD > .git/git_committer_email"
+
+    workspace = pwd()
+    branch_name = "${ env.BRANCH_NAME }".replaceAll("/", "_")
+    git_commit = readFile(".git/git_commit").replaceAll("\n", "").replaceAll("\r", "")
+    //git_commit = sh(returnStdout: true, script: "git describe").trim()
+    build_name = "${git_commit}"
+    job_name = "${ env.JOB_NAME }".replaceAll("%2F", "/")
+    committer_email = readFile(".git/git_committer_email").replaceAll("\n", "").replaceAll("\r", "")
+    GIT_URL = sh(returnStdout: true, script: "git config --get remote.origin.url").trim()
+    if ( GIT_URL != DESIRED_REPOSITORY ) {
+      error("This jenkinsfile is configured for '${ DESIRED_REPOSITORY }' but it was executed from '${ GIT_URL }'.")
+    }
+  }
+}
+
+def notifyStatus(buildStatus) {
+  def status
+  def send_to
+
+  try {
+    switch (branch_name) {
+      case 'master':
+        send_to = "${ committer_email }, ${ FAILURE_EMAIL }"
+        break
+      default:
+        send_to = "${ committer_email }"
+        break
+    }
+  } catch(Exception ex) {
+    send_to = "${ FAILURE_EMAIL }"
+  }
+
+  echo "Sending error email to: ${ send_to }"
+  try {
+    mail  to: "${ send_to }",
+          from: "Jenkins Geographica <[email protected]>",
+          subject: "[${ buildStatus }]   ${currentBuild.fullDisplayName}",
+          body: "Something is wrong in '${currentBuild.fullDisplayName}'. \n\nSee ${env.BUILD_URL} for more details."
+  } catch(Exception ex) {
+    echo "Something was wrong sending error email :("
+  }
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Geografía Aplicada S.L
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,121 @@
+# Longitude
+
+A **new** bunch of middleware functions to build applications on top of CARTO.
+
+## Roadmap
+
+- [ ] Database model
+  - [x] CARTO data source
+    - [x] Basic parametrized queries (i.e. templated queries)
+    - [x] Protected parametrized queries (i.e. avoiding injection)
+    - [ ] Bind/dynamic parameters in queries (server-side render)
+  - [x] Postgres data source
+    - [x] psycopg2
+    - [x] SQLAlchemy
+  - [x] Cache
+    - [x] Base cache
+      - [x] Put
+      - [x] Get
+      - [x] Key generation
+      - [x] Flush
+      - [x] Tests
+    - [x] Ram Cache
+      - [x] Tests
+    - [x] Redis Cache
+      - [x] Tests 
+  - [x] Documentation
+    - [x] Sample scripts
+  - [x] Unit tests
+  - [x] Sample scripts
+
+- [x] Config
+
+- [x] CI PyPi versioning
+
+- [ ] Data manipulation
+  - [ ] Carto
+    - [ ] DataFrame read/write
+    - [ ] COPY
+  - [ ] Postgres
+    - [ ] DataFrame read/write
+    - [ ] COPY
+
+- [ ] Validations
+  - [ ] Marshmallow
+    - [ ] Wrapper (?)
+    - [ ] Documentation
+
+- [ ] Swagger
+  - [ ] Decorators
+  - [ ] Flassger (?)
+  - [ ] OAuth integration
+  - [ ] Postman integration
+  - [ ] Documentation
+
+- [ ] SQL Alchemy
+  - [ ] Model definition
+  - [ ] Jenkins integration
+  - [ ] Documentation
+
+- [ ] OAuth
+  - [ ] Role mapping
+  - [ ] Token storage
+  - [ ] Documentation
+
+## As final user...
+
+How to use:
+```bash
+pip install longitude
+```
+
+Or:
+```bash
+pipenv install longitude
+```
+
+Or:
+```bash
+poetry add longitude
+```
+
+Or install from GitHub:
+```bash
+pip install -e git+https://github.com/GeographicaGS/Longitude#egg=longitude
+```
+
+## As developer...
+
+### First time
+
+1. Install ```poetry``` using the [recommended process](https://github.com/sdispater/poetry#installation)
+    1. poetry is installed globally as a tool
+    1. It works along with virtualenvironments
+1. Create a virtual environment for Python 3.x (check the current development version in ```pyproject.toml```)
+    1. You can create it wherever you want but do not put it inside the project
+    1. A nice place is ```$HOME/virtualenvs/longitude```
+1. Clone the ```longitude``` repo
+1. `cd` to the repo and:
+    1. Activate the virtual environment: `. ~/virtualenvs/longitude/bin/activate`
+    1. Run `poetry install`
+1. Configure your IDE to use the virtual environment
+
+### Daily
+
+1. Remember to activate the virtual environment 
+
+### Why Poetry?
+
+Because it handles development dependencies and packaging with a single file (```pyproject.toml```), which is [already standard](https://flit.readthedocs.io/en/latest/pyproject_toml.html).
+
+## Sample scripts
+
+These are intended to be used with real databases (i.e. those in your profile) to check features of the library. They must be run from the virtual environment.
+
+## Testing and coverage 
+
+The [```pytest-cov```](https://pytest-cov.readthedocs.io/en/latest/) plugin is being used. Coverage configuration is at ```.coveragerc``` (including output folder).
+
+You can run something like: ```pytest --cov-report=html --cov=core core``` and the results will go in the defined html folder.
+
+There is a bash script called ```generate_core_coverage.sh``` that runs the coverage analysis and shows the report in your browser.

docker-compose.yml

@@ -0,0 +1,28 @@
+version: "3"
+services:
+  python:
+    image: python:3.6.3-onbuild
+    command: bash
+    volumes:
+      - .:/usr/longitude/app
+
+  cache:
+    image: redis:alpine
+    ports:
+      - "6379:6379"
+    command:
+      - redis-server
+      - --requirepass longitude
+      - --maxmemory 256mb
+      - --maxmemory-policy allkeys-lru
+    restart: unless-stopped
+
+  postgres:
+    image: kartoza/postgis
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_USER: longitude
+      POSTGRES_PASS: longitude
+    volumes:
+      - ./data/:/var/lib/postgresql/data/pgdata

generate_core_coverage.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+pytest --cov-report=html --cov=longitude.core longitude/core/tests/
+sensible-browser coverage_html_report/index.html

longitude/__init__.py

@@ -0,0 +1 @@
+from .core import *