diff --git a/Makefile b/Makefile index b2c264e..0dad71a 100644 --- a/Makefile +++ b/Makefile @@ -76,10 +76,10 @@ docker-push: ## Push docker image docker-run: ## Run docker image with sqlite mkdir -p data sudo chown -R 1000:1000 data - docker run --name ot-recoder --rm -it -p 8000:8000 \ + docker run --name ot-recorder --rm -it -p 8000:8000 \ -v $$(pwd)/config.yml:/app/config.yml \ -v $$(pwd)/data:/persist \ $(DOCKER_IMAGE_NAME):latest docker-migrate: ## Run migrations inside docker - docker exec ot-recoder /app/ot-recorder migrate up + docker exec ot-recorder /app/ot-recorder migrate up diff --git a/README.md b/README.md index 3b72ab0..4834a21 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,234 @@ # OwnTracks Recorder [![](https://img.shields.io/badge/Go-1.19-00ADD8?style=flat&logo=go)](https://golang.org/doc/go1.19) [![build](https://github.com/hrshadhin/ot-recorder/actions/workflows/build.yml/badge.svg)](https://github.com/hrshadhin/ot-recorder/actions?query=workflow%3ABuild) -[![gosec](https://img.shields.io/github/workflow/status/hrshadhin/ot-recorder/Security?label=%F0%9F%94%91%20gosec&style=flat&color=75C46B)](https://github.com/hrshadhin/ot-recorder/actions?query=workflow%3ASecurity) +[![gosec](https://img.shields.io/github/actions/workflow/status/hrshadhin/ot-recorder/security.yml?branch=master&label=%F0%9F%94%91%20gosec&style=flat&color=75C46B)](https://github.com/hrshadhin/ot-recorder/actions?query=workflow%3ASecurity) [![codecov](https://codecov.io/gh/hrshadhin/ot-recorder/branch/master/graph/badge.svg?token=LCSFDYZZGQ)](https://codecov.io/gh/hrshadhin/ot-recorder) [![Go Report Card](https://goreportcard.com/badge/github.com/hrshadhin/ot-recorder)](https://goreportcard.com/report/github.com/hrshadhin/ot-recorder) -Store and access data published by OwnTracks apps in (postgres, mysql or sqlite) via REST API +Store and access data published by OwnTracks apps in (PostgreSQL, MySQL or Sqlite) via REST API. +Self-hosted and Google-free location tracking system. ## Architecture ![architecture of ot-recorder](_doc/arch.png) +## Index +- [Architecture](#architecture) +- [System requirements](#system-requirements) +- [Getting started](#getting-started) + - [Create Database and Users](#create-database-and-users) + - [Configuration](#configuration) + - [Deploy OwnTracks Recorder](#deploy-owntracks-recorder) + - [Using Systemd.d service](#using-systemdd-service) + - [Using Docker](#using-docker) + - [Using Ansible](#ansible) + - [Setup Mobile APP](#setup-mobile-app) +- [Grafana Integration](#grafana-integration) +- [Development](#development) +- [API's](#api) +- [Documentation](#docs) +- [TO-DO](#to-do) + + ## System requirements - OwnTracks app (Android / iOS) -- Postgres/Mysql/Sqlite +- PostgreSQL / MySQL / MariaDB / Sqlite - Optional - Domain for public access - Reverse Proxy(NGINX/HA/Caddy) for TLS, HTTPS - Grafana visualization ## Getting started -**WIP** +### Create Database and Users +> For sqlite no need to create database or user. +- Create database `owntracks` +- Create a user for application(read/write) +- Create a user for grafana(read only) + +### Configuration +- create a `config.yml` file or copy from `_doc/config.yml` +- customize the config + ```yaml + app: + env: production + host: localhost + port: 8000 + read_timeout: 2s + write_timeout: 5s + idle_timeout: 3s + context_timeout: 2s + data_path: ./data # for sqlite | value must be /persist for docker + time_zone: 'Asia/Dhaka' + debug: false + + # For PostgreSQL + database: + type: postgres + host: localhost + port: 5432 + name: owntracks + username: dev + password: dev + ssl_mode: disable + max_open_conn: 1 + max_idle_conn: 1 + max_life_time: 10s + debug: false + + # For MySQL or MariaDB + #database: + # type: mysql + # host: localhost + # port: 3306 + # name: owntracks + # username: dev + # password: dev + # max_open_conn: 1 + # max_idle_conn: 1 + # max_life_time: 10s + # debug: false + + # For Sqlite + #database: + # type: sqlite + # name: owntracks + # max_open_conn: 1 + # max_idle_conn: 1 + # max_life_time: 10s + # debug: false + ``` +- NGINX config (Optional) [here](_deploy/nginx.conf) + +### Deploy OwnTracks Recorder +#### Using Systemd.d service +- Download the binary from release page or build from source [Check here](#development) +- Move binary `mv ot-recorder /usr/local/bin/ot-recorder` +- It's assumed that your config file is located under `/opt/owntracks-recorder` +- Place this [owntracks-recorder.service](_deploy/owntracks-recorder.service) file into `/etc/systemd/system` + ``` + [Unit] + Description=Owntracks Recorder server + Requires=network.target + After=network.target + + [Service] + Type=simple + WorkingDirectory=/opt/owntracks-recorder + ExecReload=/bin/kill -HUP $MAINPID + ExecStart=/usr/local/bin/ot-recorder --config /opt/owntracks-recorder/config.yml serve + + Restart=always + RestartSec=3 + + [Install] + WantedBy=multi-user.target + ``` + Then run `sudo systemctl enable owntracks-recorder.service` + Finally run `sudo systemctl start owntracks-recorder.service` to start and check status by + running `sudo systemctl status owntracks-recorder.service` + +#### Using Docker +- Docker compose + ```bash + # persistance data directory for sqlite + mkdir data + sudo chown -R 1000:1000 data + ``` + ```yaml + # _deploy/docker-compose.yml + version: "3.5" + + services: + owntracks-recorder: + image: hrshadhin/ot-recorder + container_name: owntracks-recorder + restart: unless-stopped + volumes: + - ${PWD}/config.yml:/app/config.yml + - ${PWD}/data:/persist # only for sqlite + ports: + - "8000:8000" + environment: + - TZ=Asia/Dhaka + ``` + ```bash + # run container in background + docker-compose up -d + + # run migrations + docker-compose exec owntracks-recorder /app/ot-recorder migrate up + ``` +- Docker CLI + ```bash + # run container in background + docker run --name owntracks-recorder -p 8000:8000 \ + -v $(pwd)/config.yml:/app/config.yml \ + -v $(pwd)/data:/persist \ + hrshadhin/ot-recorder:latest + + # run migrations + docker exec owntracks-recorder /app/ot-recorder migrate up + ``` +#### Ansible +- Ansible Role [here](https://github.com/hrshadhin/vps/tree/master/provisioner/ansible/roles/owntracks) + +### Setup Mobile APP +- Install app [here](https://owntracks.org/) +- Configure APP + ```yaml + mode: HTTP + + endpoint: + http://host[:port]/api/v1/ping #HTTP + OR + https://host[:port]/api/v1/ping # HTTPS/TLS + OR + http[s]://[user[:password]@]host[:port]/api/v1/ping # If basic auth enabled + + username: dev # <= 20 letter + password: dev # set it, if basic auth enabled and endpoint dont have auth info + device: phone # <= 20 letter + trackingid: t1 # <= 2 letter + ``` ## Grafana Integration -**WIP** +- GeoMap Panel +![grafan-dashboard](_doc/grafana.png) +- Queries + ```sql + -- Location pings + SELECT + to_timestamp(created_at) as "time", lat, lon, + acc as accuracy, vel as velocity + FROM locations + WHERE + $__unixEpochFilter(created_at) + ORDER BY created_at + + -- Battery Level + SELECT + created_at as time, batt + FROM locations + WHERE + $__unixEpochFilter(created_at) + ORDER BY created_at + + -- Velocity + SELECT + created_at as time, vel + FROM locations + WHERE + $__unixEpochFilter(created_at) + ORDER BY created_at + + -- Last Location + SELECT + to_timestamp(created_at) as "time", + lat, lon, acc as accuracy, vel as velocity + FROM locations + ORDER BY created_at DESC + LIMIT 1 + ``` +- [Dashboard Json](_doc/grafana-dashboard.json) ## Development - Copy config file `mv _doc/config ./` to root directory and change it @@ -47,11 +253,16 @@ Store and access data published by OwnTracks apps in (postgres, mysql or sqlite) ``` - Visit **`http://localhost:8000`** - Stop `CTRL + C` +- About [OwnTracks](https://owntracks.org/) -## API's +## API - Location Ping - User Last Location ## Docs - [ERD](_doc/erd.png) - [API documentation](https://hrshadhin.github.io/projects/ot-recorder/swagger.html) + +## TO-Do +- [ ] Telegram Bot for last location check +- [ ] Reverse geocoding diff --git a/_deploy/nginx.conf b/_deploy/nginx.conf new file mode 100644 index 0000000..d16f07b --- /dev/null +++ b/_deploy/nginx.conf @@ -0,0 +1,77 @@ +upstream owntrackbackend { + server 127.0.0.1:{{owntracks_port}}; +} + +server { + if ($host = {{owntracks_domain_name}}) { + return 301 https://$host$request_uri; + } + + listen 80; + server_name {{owntracks_domain_name}}; + return 404; +} + +server { + ## + # Basic Settings + # + listen 443 ssl; + server_name {{owntracks_domain_name}}; + + root /var/www/html/{{owntracks_domain_name}}; + index index.html; + + charset utf-8; + expires $expires; + add_header Strict-Transport-Security max-age=15768000; + add_header X-Content-Type-Options nosniff; + add_header X-XSS-Protection "1; mode=block"; + + ## + # SSL Settings + ## + ssl_certificate /etc/letsencrypt/live/{{owntracks_domain_name}}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/{{owntracks_domain_name}}/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + ## + # Path & Error Settings + ## + location / { + auth_basic "Who are you?"; + auth_basic_user_file /etc/nginx/.htpasswd; + proxy_pass http://owntrackbackend; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + error_page 404 /404.html; + location /404.html { + internal; + } + + ## + # Logging Settings + ## + access_log /var/log/nginx/{{owntracks_domain_name}}_access.log; + error_log /var/log/nginx/{{owntracks_domain_name}}_error.log; + + location /robots.txt { + access_log off; + } + + location ~ /favicon.png { + access_log off; + } + + + ## + # Gzip Settings + ## + gzip on; + gzip_types text/plain application/json; +} diff --git a/_deploy/owntracks-recorder.service b/_deploy/owntracks-recorder.service new file mode 100644 index 0000000..6e7821b --- /dev/null +++ b/_deploy/owntracks-recorder.service @@ -0,0 +1,16 @@ +[Unit] +Description=Owntracks Recorder server +Requires=network.target +After=network.target + +[Service] +Type=simple +WorkingDirectory=/opt/owntracks-recorder +ExecReload=/bin/kill -HUP $MAINPID +ExecStart=/usr/local/bin/ot-recorder --config /opt/owntracks-recorder/config.yml serve + +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target diff --git a/_doc/grafana-dashboard.json b/_doc/grafana-dashboard.json new file mode 100644 index 0000000..3018faa --- /dev/null +++ b/_doc/grafana-dashboard.json @@ -0,0 +1,722 @@ +{ + "__inputs": [ + { + "name": "DS_POSTGRESQL", + "label": "PostgreSQL", + "description": "", + "type": "datasource", + "pluginId": "postgres", + "pluginName": "PostgreSQL" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "geomap", + "name": "Geomap", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.3.1" + }, + { + "type": "datasource", + "id": "postgres", + "name": "PostgreSQL", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "basemap": { + "config": {}, + "name": "Layer 0", + "type": "default" + }, + "controls": { + "mouseWheelZoom": true, + "showAttribution": false, + "showDebug": false, + "showMeasure": false, + "showScale": true, + "showZoom": true + }, + "layers": [ + { + "config": { + "showLegend": false, + "style": { + "color": { + "fixed": "dark-green" + }, + "opacity": 0.4, + "rotation": { + "fixed": 0, + "max": 360, + "min": -360, + "mode": "mod" + }, + "size": { + "fixed": 10, + "max": 15, + "min": 2 + }, + "symbol": { + "fixed": "img/icons/marker/circle.svg", + "mode": "fixed" + }, + "textConfig": { + "fontSize": 12, + "offsetX": 0, + "offsetY": 0, + "textAlign": "center", + "textBaseline": "middle" + } + } + }, + "location": { + "mode": "auto" + }, + "name": "Layer 1", + "tooltip": true, + "type": "markers" + } + ], + "tooltip": { + "mode": "details" + }, + "view": { + "allLayers": true, + "id": "coords", + "lat": 23.7445, + "lon": 90.4099, + "zoom": 12 + } + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT \nto_timestamp(created_at) as \"time\", lat, lon, acc as accuracy, vel as velocity\nFROM locations\nORDER BY created_at DESC LIMIT 1 ", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "created_at", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "parameters": [ + { + "name": "lat", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "parameters": [ + { + "name": "lon", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "parameters": [ + { + "name": "vel", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "parameters": [ + { + "name": "acc", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 1, + "orderBy": { + "property": { + "name": [ + "created_at" + ], + "type": "string" + }, + "type": "property" + }, + "orderByDirection": "DESC" + }, + "table": "locations" + } + ], + "title": "Last location", + "type": "geomap" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "green", + "value": 20 + }, + { + "color": "semi-dark-green", + "value": 60 + }, + { + "color": "dark-green", + "value": 100 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 6, + "y": 0 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT \ncreated_at as time, batt\nFROM locations\nWHERE $__unixEpochFilter(created_at)\nORDER BY created_at", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "created_at", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "locations" + } + ], + "title": "Battery %", + "type": "timeseries" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + } + ] + }, + "unit": "velocityms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 0 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT \ncreated_at as time, vel\nFROM locations\nWHERE $__unixEpochFilter(created_at)\nORDER BY created_at", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "created_at", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "locations" + } + ], + "title": "Velocity", + "type": "timeseries" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "basemap": { + "config": {}, + "name": "Layer 0", + "type": "default" + }, + "controls": { + "mouseWheelZoom": true, + "showAttribution": false, + "showDebug": false, + "showMeasure": false, + "showScale": true, + "showZoom": true + }, + "layers": [ + { + "config": { + "showLegend": false, + "style": { + "color": { + "fixed": "dark-green" + }, + "opacity": 0.4, + "rotation": { + "fixed": 0, + "max": 360, + "min": -360, + "mode": "clamped" + }, + "size": { + "fixed": 10, + "max": 30, + "min": 10 + }, + "symbol": { + "fixed": "img/icons/marker/circle.svg", + "mode": "fixed" + }, + "text": { + "fixed": "", + "mode": "fixed" + }, + "textConfig": { + "fontSize": 14, + "offsetX": 0, + "offsetY": 0, + "textAlign": "center", + "textBaseline": "middle" + } + } + }, + "location": { + "mode": "auto" + }, + "name": "Layer 1", + "tooltip": true, + "type": "markers" + } + ], + "tooltip": { + "mode": "details" + }, + "view": { + "allLayers": true, + "id": "coords", + "lat": 23.7731, + "lon": 90.3561, + "zoom": 12 + } + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \nto_timestamp(created_at) as \"time\", lat, lon, acc as accuracy, vel as velocity \nFROM locations \nWHERE\n$__unixEpochFilter(created_at)\nORDER BY created_at desc", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "lat", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "parameters": [ + { + "name": "lon", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "parameters": [ + { + "name": "vel", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "parameters": [ + { + "name": "created_at", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [], + "limit": 50, + "orderBy": { + "property": { + "name": "created_at", + "type": "string" + }, + "type": "property" + }, + "orderByDirection": "DESC", + "whereJsonTree": { + "children1": [ + { + "id": "bb9aabaa-4567-489a-bcde-f1851626d76a", + "properties": { + "field": "created_at", + "operator": "equal", + "value": [ + "2022-12-14 14:18:47" + ], + "valueSrc": [ + "value" + ], + "valueType": [ + "datetime" + ] + }, + "type": "rule" + } + ], + "id": "9ba8aaab-89ab-4cde-b012-318515f965fa", + "type": "group" + }, + "whereString": "created_at = '2022-12-15 00:00:00.000'" + }, + "table": "locations" + } + ], + "title": "Location Ping", + "type": "geomap" + } + ], + "refresh": "", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Locations", + "uid": "zeNlsp5Vk", + "version": 32, + "weekStart": "" +} diff --git a/_doc/grafana.png b/_doc/grafana.png new file mode 100644 index 0000000..723d6c4 Binary files /dev/null and b/_doc/grafana.png differ