Merge pull request #261 from taosdata/adam/TD-28951

feat: add data-gen script

scripts/data-generation/.gitignore

@@ -0,0 +1 @@
+__pycache__/*

scripts/data-generation/Dockerfile

@@ -0,0 +1,9 @@
+FROM tdengine/tdengine:3.2.3.0
+
+RUN apt update && apt install -y python3 python3-pip
+RUN pip3 install taospy
+
+COPY main.py /main.py
+COPY metrics.py /metrics.py
+
+CMD ["python3", "/main.py"]

scripts/data-generation/docker-compose.yaml

@@ -0,0 +1,40 @@
+version: "3"
+services:
+    grafana: # min version image: grafana/grafana:7.5.0
+    # max version #
+      image: grafana/grafana:latest
+      pull_policy: always
+      ports:
+      - 3002:3000
+      environment:
+        GF_AUTH_ANONYMOUS_ENABLED: 1
+        GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
+        GF_INSTALL_PLUGINS: https://github.com/taosdata/grafanaplugin/releases/download/v3.5.0/tdengine-datasource-3.5.0.zip;tdengine-datasource,marcusolsson-static-datasource,marcusolsson-dynamictext-panel
+      # volumes:
+      # - ./provisioning:/etc/grafana/provisioning
+      depends_on:
+      - tdengine
+    tdengine:
+        image: tdengine/tdengine:3.2.3.0
+        volumes:
+        - ./tdengine/taos.cfg:/etc/taos/taos.cfg
+        ports:
+        - "8041:6041"
+    taoskeeper:
+        image: tdengine/tdengine:3.2.3.0
+        depends_on:
+        - tdengine
+        environment:
+        - TAOS_KEEPER_TDENGINE_HOST=tdengine
+        volumes:
+        - ./tdengine/taos.cfg:/etc/taos/taos.cfg
+        ports:
+        - "8043:6043"
+        entrypoint:
+        - /bin/sh
+        - -c
+        - "while true; do taoskeeper --tdengine.host tdengine --taosAdapter.address tdengine:6041; done"
+    simulator-taosx:
+        image: image.cloud.taosdata.com/tdinsight/simulator-taosx:1.0.0
+        depends_on:
+        - tdengine

scripts/data-generation/main.py

@@ -0,0 +1,78 @@
+# coding:utf-8
+
+import time
+import random
+import taos
+import re
+from metrics import *
+
+host = "tdengine"
+conn = taos.connect(host=host)
+dbname = "log"
+write_interval = 3
+
+
+def select_db():
+    conn.execute("create database IF NOT EXISTS " + dbname)
+    conn.select_db(dbname)
+
+
+def clear_stable():
+    for stable in all_metrics:
+        stname = stable["stable_name"]
+        print(f"drop stable {stname}")
+        conn.execute("drop stable IF EXISTS " + stname)
+
+def rand_data(v):
+    try:
+        num, suffix = re.match(r"(\d+|\.\d+|\d+\.\d+)([a-z]\d+)", v).groups()
+        if suffix in ["u8", "u16", "u32", "u64", "i8", "i16", "i32", "i64"]:
+            num = int(num)
+            num = random.randint(0, num * 2)
+            v = f"{num}{suffix}"
+        elif suffix in ["f32", "f64"]:
+            num = float(num)
+            num = random.uniform(0, num * 2)
+            v = f"{num:.0f}{suffix}"
+    except:
+        print(f"indivisible v={v}")
+    return v
+
+
+def task():
+    lines = []
+    for stable in all_metrics:
+        stname = stable["stable_name"]
+        tags = stable["tags"]
+        metrics = stable["metrics"]
+
+        tags_list = []
+        for tag in tags:
+            tags_list.append(f'{tag["name"]}={tag["value"]}')
+
+        metrics_list = []
+        for metric in metrics:
+            k = metric["name"]
+            v = metric["value"]
+            v = rand_data(v)
+            metrics_list.append(f"{k}={v}")
+
+        line = f"{stname},{','.join(tags_list)} {','.join(metrics_list)} {int(time.time() * 1000)}"
+        lines.append(line)
+        print(line)
+
+    conn.schemaless_insert(
+        lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.MILLI_SECONDS
+    )
+
+
+def main():
+    select_db()
+    clear_stable()
+    while True:
+        task()
+        time.sleep(write_interval)
+
+
+if __name__ == "__main__":
+    main()