diff --git a/scripts/data-generation/.gitignore b/scripts/data-generation/.gitignore new file mode 100644 index 0000000..96403d3 --- /dev/null +++ b/scripts/data-generation/.gitignore @@ -0,0 +1 @@ +__pycache__/* diff --git a/scripts/data-generation/Dockerfile b/scripts/data-generation/Dockerfile new file mode 100644 index 0000000..c7ab9f4 --- /dev/null +++ b/scripts/data-generation/Dockerfile @@ -0,0 +1,9 @@ +FROM tdengine/tdengine:3.2.3.0 + +RUN apt update && apt install -y python3 python3-pip +RUN pip3 install taospy + +COPY main.py /main.py +COPY metrics.py /metrics.py + +CMD ["python3", "/main.py"] diff --git a/scripts/data-generation/docker-compose.yaml b/scripts/data-generation/docker-compose.yaml new file mode 100644 index 0000000..32132c7 --- /dev/null +++ b/scripts/data-generation/docker-compose.yaml @@ -0,0 +1,40 @@ +version: "3" +services: + grafana: # min version image: grafana/grafana:7.5.0 + # max version # + image: grafana/grafana:latest + pull_policy: always + ports: + - 3002:3000 + environment: + GF_AUTH_ANONYMOUS_ENABLED: 1 + GF_AUTH_ANONYMOUS_ORG_ROLE: Admin + GF_INSTALL_PLUGINS: https://github.com/taosdata/grafanaplugin/releases/download/v3.5.0/tdengine-datasource-3.5.0.zip;tdengine-datasource,marcusolsson-static-datasource,marcusolsson-dynamictext-panel + # volumes: + # - ./provisioning:/etc/grafana/provisioning + depends_on: + - tdengine + tdengine: + image: tdengine/tdengine:3.2.3.0 + volumes: + - ./tdengine/taos.cfg:/etc/taos/taos.cfg + ports: + - "8041:6041" + taoskeeper: + image: tdengine/tdengine:3.2.3.0 + depends_on: + - tdengine + environment: + - TAOS_KEEPER_TDENGINE_HOST=tdengine + volumes: + - ./tdengine/taos.cfg:/etc/taos/taos.cfg + ports: + - "8043:6043" + entrypoint: + - /bin/sh + - -c + - "while true; do taoskeeper --tdengine.host tdengine --taosAdapter.address tdengine:6041; done" + simulator-taosx: + image: image.cloud.taosdata.com/tdinsight/simulator-taosx:1.0.0 + depends_on: + - tdengine diff --git a/scripts/data-generation/main.py b/scripts/data-generation/main.py new file mode 100644 index 0000000..a5f2769 --- /dev/null +++ b/scripts/data-generation/main.py @@ -0,0 +1,78 @@ +# coding:utf-8 + +import time +import random +import taos +import re +from metrics import * + +host = "tdengine" +conn = taos.connect(host=host) +dbname = "log" +write_interval = 3 + + +def select_db(): + conn.execute("create database IF NOT EXISTS " + dbname) + conn.select_db(dbname) + + +def clear_stable(): + for stable in all_metrics: + stname = stable["stable_name"] + print(f"drop stable {stname}") + conn.execute("drop stable IF EXISTS " + stname) + +def rand_data(v): + try: + num, suffix = re.match(r"(\d+|\.\d+|\d+\.\d+)([a-z]\d+)", v).groups() + if suffix in ["u8", "u16", "u32", "u64", "i8", "i16", "i32", "i64"]: + num = int(num) + num = random.randint(0, num * 2) + v = f"{num}{suffix}" + elif suffix in ["f32", "f64"]: + num = float(num) + num = random.uniform(0, num * 2) + v = f"{num:.0f}{suffix}" + except: + print(f"indivisible v={v}") + return v + + +def task(): + lines = [] + for stable in all_metrics: + stname = stable["stable_name"] + tags = stable["tags"] + metrics = stable["metrics"] + + tags_list = [] + for tag in tags: + tags_list.append(f'{tag["name"]}={tag["value"]}') + + metrics_list = [] + for metric in metrics: + k = metric["name"] + v = metric["value"] + v = rand_data(v) + metrics_list.append(f"{k}={v}") + + line = f"{stname},{','.join(tags_list)} {','.join(metrics_list)} {int(time.time() * 1000)}" + lines.append(line) + print(line) + + conn.schemaless_insert( + lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.MILLI_SECONDS + ) + + +def main(): + select_db() + clear_stable() + while True: + task() + time.sleep(write_interval) + + +if __name__ == "__main__": + main() diff --git a/scripts/data-generation/metrics.py b/scripts/data-generation/metrics.py new file mode 100644 index 0000000..6d0d6ae --- /dev/null +++ b/scripts/data-generation/metrics.py @@ -0,0 +1,755 @@ +all_metrics = [ + { + "ts": "1703484955810", + "stable_name": "taosx_sys", + "tags": [{"name": "taosx_id", "value": "taosx_id_1"}], + "metrics": [ + {"name": "sys_cpu_cores", "value": "12f64"}, + {"name": "sys_total_memory", "value": "8000000000"}, + {"name": "sys_used_memory", "value": "4000000000f64"}, + {"name": "sys_available_memory", "value": "2000000000f64"}, + {"name": "process_uptime", "value": "100f64"}, + {"name": "process_cpu_percent", "value": "30f64"}, + {"name": "process_memory_percent", "value": "30f64"}, + {"name": "process_net_read_bytes", "value": "6000000000f64"}, + {"name": "process_net_written_bytes", "value": "6000000000f64"}, + {"name": "process_disk_read_bytes", "value": "6000000000f64"}, + {"name": "process_disk_written_bytes", "value": "6000000000f64"}, + {"name": "process_id", "value": "12343f64"}, + {"name": "running_tasks", "value": "31f64"}, + {"name": "completed_tasks", "value": "32f64"}, + {"name": "failed_tasks", "value": "33f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_agent", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "agent_id", "value": "agent_id_1"}, + {"name": "agent_name", "value": "agent_name_1"}, + ], + "metrics": [ + {"name": "sys_cpu_cores", "value": "12f64"}, + {"name": "sys_total_memory", "value": "8000000000"}, + {"name": "sys_used_memory", "value": "4000000000f64"}, + {"name": "sys_available_memory", "value": "2000000000f64"}, + {"name": "process_uptime", "value": "100f64"}, + {"name": "process_cpu_percent", "value": "30f64"}, + {"name": "process_memory_percent", "value": "30f64"}, + {"name": "process_start_time", "value": "350f64"}, + {"name": "process_id", "value": "12343f64"}, + {"name": "process_disk_read_bytes", "value": "6000000000f64"}, + {"name": "process_disk_written_bytes", "value": "6000000000f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_agent", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "agent_id", "value": "agent_id_2"}, + {"name": "agent_name", "value": "agent_name_2"}, + ], + "metrics": [ + {"name": "sys_cpu_cores", "value": "12f64"}, + {"name": "sys_total_memory", "value": "8000000000"}, + {"name": "sys_used_memory", "value": "4000000000f64"}, + {"name": "sys_available_memory", "value": "2000000000f64"}, + {"name": "process_uptime", "value": "100f64"}, + {"name": "process_cpu_percent", "value": "30f64"}, + {"name": "process_memory_percent", "value": "30f64"}, + {"name": "process_start_time", "value": "350f64"}, + {"name": "process_id", "value": "12343f64"}, + {"name": "process_disk_read_bytes", "value": "6000000000f64"}, + {"name": "process_disk_written_bytes", "value": "6000000000f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_connector", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "ds_name", "value": "influxdb"}, + {"name": "task_id", "value": "task_id_6"}, + ], + "metrics": [ + {"name": "process_uptime", "value": "100f64"}, + {"name": "process_cpu_percent", "value": "30f64"}, + {"name": "process_memory_percent", "value": "30f64"}, + {"name": "process_disk_read_bytes", "value": "6000000000f64"}, + {"name": "process_disk_written_bytes", "value": "6000000000f64"}, + {"name": "process_id", "value": "12343f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_connector", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "ds_name", "value": "opentsdb"}, + {"name": "task_id", "value": "task_id_7"}, + ], + "metrics": [ + {"name": "process_uptime", "value": "100f64"}, + {"name": "process_cpu_percent", "value": "30f64"}, + {"name": "process_memory_percent", "value": "30f64"}, + {"name": "process_disk_read_bytes", "value": "6000000000f64"}, + {"name": "process_disk_written_bytes", "value": "6000000000f64"}, + {"name": "process_id", "value": "12343f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_connector", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "ds_name", "value": "opcua"}, + {"name": "task_id", "value": "task_id_2"}, + ], + "metrics": [ + {"name": "process_uptime", "value": "100f64"}, + {"name": "process_cpu_percent", "value": "30f64"}, + {"name": "process_memory_percent", "value": "30f64"}, + {"name": "process_disk_read_bytes", "value": "6000000000f64"}, + {"name": "process_disk_written_bytes", "value": "6000000000f64"}, + {"name": "process_id", "value": "12343f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_connector", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "ds_name", "value": "opcda"}, + {"name": "task_id", "value": "task_id_opcda_1"}, + ], + "metrics": [ + {"name": "process_uptime", "value": "100f64"}, + {"name": "process_cpu_percent", "value": "30f64"}, + {"name": "process_memory_percent", "value": "30f64"}, + {"name": "process_disk_read_bytes", "value": "6000000000f64"}, + {"name": "process_disk_written_bytes", "value": "6000000000f64"}, + {"name": "process_id", "value": "12343f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_connector", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "ds_name", "value": "pi"}, + {"name": "task_id", "value": "task_id_8"}, + ], + "metrics": [ + {"name": "process_uptime", "value": "100f64"}, + {"name": "process_cpu_percent", "value": "30f64"}, + {"name": "process_memory_percent", "value": "30f64"}, + {"name": "process_disk_read_bytes", "value": "6000000000f64"}, + {"name": "process_disk_written_bytes", "value": "6000000000f64"}, + {"name": "process_id", "value": "12343f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_connector", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "ds_name", "value": "pibackfill"}, + {"name": "task_id", "value": "task_id_9"}, + ], + "metrics": [ + {"name": "process_uptime", "value": "100f64"}, + {"name": "process_cpu_percent", "value": "30f64"}, + {"name": "process_memory_percent", "value": "30f64"}, + {"name": "process_disk_read_bytes", "value": "6000000000f64"}, + {"name": "process_disk_written_bytes", "value": "6000000000f64"}, + {"name": "process_id", "value": "12343f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_connector", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "ds_name", "value": "mqtt"}, + {"name": "task_id", "value": "task_id_4"}, + ], + "metrics": [ + {"name": "process_uptime", "value": "100f64"}, + {"name": "process_cpu_percent", "value": "30f64"}, + {"name": "process_memory_percent", "value": "30f64"}, + {"name": "process_disk_read_bytes", "value": "6000000000f64"}, + {"name": "process_disk_written_bytes", "value": "6000000000f64"}, + {"name": "process_id", "value": "12343f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_connector", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "ds_name", "value": "mqtt"}, + {"name": "task_id", "value": "task_id_5"}, + ], + "metrics": [ + {"name": "process_uptime", "value": "100f64"}, + {"name": "process_cpu_percent", "value": "30f64"}, + {"name": "process_memory_percent", "value": "30f64"}, + {"name": "process_disk_read_bytes", "value": "6000000000f64"}, + {"name": "process_disk_written_bytes", "value": "6000000000f64"}, + {"name": "process_id", "value": "12343f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_task_tdengine2", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "task_id", "value": "task_id_13"}, + {"name": "task_name", "value": "task_name_13"}, + ], + "metrics": [ + {"name": "total_execute_time", "value": "123456f64"}, + {"name": "total_written_rows", "value": "1200f64"}, + {"name": "total_written_points", "value": "60000f64"}, + {"name": "start_time", "value": "12345678f64"}, + {"name": "written_rows", "value": "1234f64"}, + {"name": "written_points", "value": "123400f64"}, + {"name": "execute_time", "value": "12345678f64"}, + {"name": "read_concurrency", "value": "123f64"}, + {"name": "total_stables", "value": "12f64"}, + {"name": "total_updated_tags", "value": "11f64"}, + {"name": "total_created_tables", "value": "12340f64"}, + {"name": "total_tables", "value": "12345f64"}, + {"name": "total_finished_tables", "value": "1234f64"}, + {"name": "total_success_blocks", "value": "123450f64"}, + {"name": "finished_tables", "value": "12345f64"}, + {"name": "success_blocks", "value": "12345f64"}, + {"name": "created_tables", "value": "12345f64"}, + {"name": "updated_tags", "value": "15f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_task_tdengine3", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "task_id", "value": "task_id_1"}, + {"name": "task_name", "value": "task_name_1"}, + ], + "metrics": [ + {"name": "total_execute_time", "value": "123456f64"}, + {"name": "total_written_rows", "value": "1200f64"}, + {"name": "total_written_points", "value": "60000f64"}, + {"name": "start_time", "value": "12345678f64"}, + {"name": "written_rows", "value": "1234f64"}, + {"name": "written_points", "value": "123400f64"}, + {"name": "execute_time", "value": "12345678f64"}, + {"name": "total_messages", "value": "1234567f64"}, + {"name": "total_messages_of_meta", "value": "1234f64"}, + {"name": "total_messages_of_data", "value": "12345f64"}, + {"name": "total_write_raw_fails", "value": "12f64"}, + {"name": "total_success_blocks", "value": "123455f64"}, + {"name": "topics", "value": "12f64"}, + {"name": "consumers", "value": "123f64"}, + {"name": "messages", "value": "12345678f64"}, + {"name": "messages_of_meta", "value": "12345f64"}, + {"name": "messages_of_data", "value": "123456f64"}, + {"name": "write_raw_fails", "value": "15f64"}, + {"name": "success_blocks", "value": "15000f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_task_opcua", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "task_id", "value": "task_id_2"}, + {"name": "task_name", "value": "task_name_2"}, + ], + "metrics": [ + {"name": "total_execute_time", "value": "123456f64"}, + {"name": "total_written_rows", "value": "1200f64"}, + {"name": "total_written_points", "value": "60000f64"}, + {"name": "start_time", "value": "12345678f64"}, + {"name": "written_rows", "value": "1234f64"}, + {"name": "written_points", "value": "123400f64"}, + {"name": "execute_time", "value": "12345678f64"}, + {"name": "total_received_batches", "value": "1234567f64"}, + {"name": "total_processed_batches", "value": "1234f64"}, + {"name": "total_processed_rows", "value": "12345f64"}, + {"name": "total_inserted_sqls", "value": "12f64"}, + {"name": "total_failed_sqls", "value": "123455f64"}, + {"name": "total_created_stables", "value": "12f64"}, + {"name": "total_created_tables", "value": "123f64"}, + {"name": "total_failed_rows", "value": "12345678f64"}, + {"name": "total_failed_point", "value": "12345f64"}, + {"name": "total_written_blocks", "value": "123456f64"}, + {"name": "total_failed_blocks", "value": "15f64"}, + {"name": "received_batches", "value": "15000f64"}, + {"name": "processed_batches", "value": "15000f64"}, + {"name": "processed_rows", "value": "15000f64"}, + {"name": "received_records", "value": "15000f64"}, + {"name": "insert_sqls", "value": "15000f64"}, + {"name": "failed_sqls", "value": "15000f64"}, + {"name": "created_stables", "value": "15000f64"}, + {"name": "created_tables", "value": "15000f64"}, + {"name": "failed_rows", "value": "15000f64"}, + {"name": "failed_points", "value": "15000f64"}, + {"name": "written_blocks", "value": "15000f64"}, + {"name": "failed_blocks", "value": "15000f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_task_opcua", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "task_id", "value": "task_id_3"}, + {"name": "task_name", "value": "task_name_3"}, + ], + "metrics": [ + {"name": "total_execute_time", "value": "123456f64"}, + {"name": "total_written_rows", "value": "1200f64"}, + {"name": "total_written_points", "value": "60000f64"}, + {"name": "start_time", "value": "12345678f64"}, + {"name": "written_rows", "value": "1234f64"}, + {"name": "written_points", "value": "123400f64"}, + {"name": "execute_time", "value": "12345678f64"}, + {"name": "total_received_batches", "value": "1234567f64"}, + {"name": "total_processed_batches", "value": "1234f64"}, + {"name": "total_processed_rows", "value": "12345f64"}, + {"name": "total_inserted_sqls", "value": "12f64"}, + {"name": "total_failed_sqls", "value": "123455f64"}, + {"name": "total_created_stables", "value": "12f64"}, + {"name": "total_created_tables", "value": "123f64"}, + {"name": "total_failed_rows", "value": "12345678f64"}, + {"name": "total_failed_point", "value": "12345f64"}, + {"name": "total_written_blocks", "value": "123456f64"}, + {"name": "total_failed_blocks", "value": "15f64"}, + {"name": "received_batches", "value": "15000f64"}, + {"name": "processed_batches", "value": "15000f64"}, + {"name": "processed_rows", "value": "15000f64"}, + {"name": "received_records", "value": "15000f64"}, + {"name": "insert_sqls", "value": "15000f64"}, + {"name": "failed_sqls", "value": "15000f64"}, + {"name": "created_stables", "value": "15000f64"}, + {"name": "created_tables", "value": "15000f64"}, + {"name": "failed_rows", "value": "15000f64"}, + {"name": "failed_points", "value": "15000f64"}, + {"name": "written_blocks", "value": "15000f64"}, + {"name": "failed_blocks", "value": "15000f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_task_opcda", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "task_id", "value": "task_id_opcda_1"}, + {"name": "task_name", "value": "task_name_opcda_1"}, + ], + "metrics": [ + {"name": "total_execute_time", "value": "123456f64"}, + {"name": "total_written_rows", "value": "1200f64"}, + {"name": "total_written_points", "value": "60000f64"}, + {"name": "start_time", "value": "12345678f64"}, + {"name": "written_rows", "value": "1234f64"}, + {"name": "written_points", "value": "123400f64"}, + {"name": "execute_time", "value": "12345678f64"}, + {"name": "total_received_batches", "value": "1234567f64"}, + {"name": "total_processed_batches", "value": "1234f64"}, + {"name": "total_processed_rows", "value": "12345f64"}, + {"name": "total_inserted_sqls", "value": "12f64"}, + {"name": "total_failed_sqls", "value": "123455f64"}, + {"name": "total_created_stables", "value": "12f64"}, + {"name": "total_created_tables", "value": "123f64"}, + {"name": "total_failed_rows", "value": "12345678f64"}, + {"name": "total_failed_point", "value": "12345f64"}, + {"name": "total_written_blocks", "value": "123456f64"}, + {"name": "total_failed_blocks", "value": "15f64"}, + {"name": "received_batches", "value": "15000f64"}, + {"name": "processed_batches", "value": "15000f64"}, + {"name": "processed_rows", "value": "15000f64"}, + {"name": "received_records", "value": "15000f64"}, + {"name": "insert_sqls", "value": "15000f64"}, + {"name": "failed_sqls", "value": "15000f64"}, + {"name": "created_stables", "value": "15000f64"}, + {"name": "created_tables", "value": "15000f64"}, + {"name": "failed_rows", "value": "15000f64"}, + {"name": "failed_points", "value": "15000f64"}, + {"name": "written_blocks", "value": "15000f64"}, + {"name": "failed_blocks", "value": "15000f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_task_mqtt", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "task_id", "value": "task_id_4"}, + {"name": "task_name", "value": "task_name_4"}, + ], + "metrics": [ + {"name": "total_execute_time", "value": "123456f64"}, + {"name": "total_written_rows", "value": "1200f64"}, + {"name": "total_written_points", "value": "60000f64"}, + {"name": "start_time", "value": "12345678f64"}, + {"name": "written_rows", "value": "1234f64"}, + {"name": "written_points", "value": "123400f64"}, + {"name": "execute_time", "value": "12345678f64"}, + {"name": "total_received_batches", "value": "1234567f64"}, + {"name": "total_processed_batches", "value": "1234f64"}, + {"name": "total_processed_rows", "value": "12345f64"}, + {"name": "total_inserted_sqls", "value": "12f64"}, + {"name": "total_failed_sqls", "value": "123455f64"}, + {"name": "total_created_stables", "value": "12f64"}, + {"name": "total_created_tables", "value": "123f64"}, + {"name": "total_failed_rows", "value": "12345678f64"}, + {"name": "total_failed_point", "value": "12345f64"}, + {"name": "total_written_blocks", "value": "123456f64"}, + {"name": "total_failed_blocks", "value": "15f64"}, + {"name": "received_batches", "value": "15000f64"}, + {"name": "processed_batches", "value": "15000f64"}, + {"name": "processed_rows", "value": "15000f64"}, + {"name": "received_records", "value": "15000f64"}, + {"name": "insert_sqls", "value": "15000f64"}, + {"name": "failed_sqls", "value": "15000f64"}, + {"name": "created_stables", "value": "15000f64"}, + {"name": "created_tables", "value": "15000f64"}, + {"name": "failed_rows", "value": "15000f64"}, + {"name": "failed_points", "value": "15000f64"}, + {"name": "written_blocks", "value": "15000f64"}, + {"name": "failed_blocks", "value": "15000f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_task_mqtt", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "task_id", "value": "task_id_5"}, + {"name": "task_name", "value": "task_name_5"}, + ], + "metrics": [ + {"name": "total_execute_time", "value": "123456f64"}, + {"name": "total_written_rows", "value": "1200f64"}, + {"name": "total_written_points", "value": "60000f64"}, + {"name": "start_time", "value": "12345678f64"}, + {"name": "written_rows", "value": "1234f64"}, + {"name": "written_points", "value": "123400f64"}, + {"name": "execute_time", "value": "12345678f64"}, + {"name": "total_received_batches", "value": "1234567f64"}, + {"name": "total_processed_batches", "value": "1234f64"}, + {"name": "total_processed_rows", "value": "12345f64"}, + {"name": "total_inserted_sqls", "value": "12f64"}, + {"name": "total_failed_sqls", "value": "123455f64"}, + {"name": "total_created_stables", "value": "12f64"}, + {"name": "total_created_tables", "value": "123f64"}, + {"name": "total_failed_rows", "value": "12345678f64"}, + {"name": "total_failed_point", "value": "12345f64"}, + {"name": "total_written_blocks", "value": "123456f64"}, + {"name": "total_failed_blocks", "value": "15f64"}, + {"name": "received_batches", "value": "15000f64"}, + {"name": "processed_batches", "value": "15000f64"}, + {"name": "processed_rows", "value": "15000f64"}, + {"name": "received_records", "value": "15000f64"}, + {"name": "insert_sqls", "value": "15000f64"}, + {"name": "failed_sqls", "value": "15000f64"}, + {"name": "created_stables", "value": "15000f64"}, + {"name": "created_tables", "value": "15000f64"}, + {"name": "failed_rows", "value": "15000f64"}, + {"name": "failed_points", "value": "15000f64"}, + {"name": "written_blocks", "value": "15000f64"}, + {"name": "failed_blocks", "value": "15000f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_task_influxdb", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "task_id", "value": "task_id_6"}, + {"name": "task_name", "value": "task_name_6"}, + ], + "metrics": [ + {"name": "total_execute_time", "value": "123456f64"}, + {"name": "total_written_rows", "value": "1200f64"}, + {"name": "total_written_points", "value": "60000f64"}, + {"name": "start_time", "value": "12345678f64"}, + {"name": "written_rows", "value": "1234f64"}, + {"name": "written_points", "value": "123400f64"}, + {"name": "execute_time", "value": "12345678f64"}, + {"name": "total_received_batches", "value": "1234567f64"}, + {"name": "total_processed_batches", "value": "1234f64"}, + {"name": "total_processed_rows", "value": "12345f64"}, + {"name": "total_inserted_sqls", "value": "12f64"}, + {"name": "total_failed_sqls", "value": "123455f64"}, + {"name": "total_created_stables", "value": "12f64"}, + {"name": "total_created_tables", "value": "123f64"}, + {"name": "total_failed_rows", "value": "12345678f64"}, + {"name": "total_failed_point", "value": "12345f64"}, + {"name": "total_written_blocks", "value": "123456f64"}, + {"name": "total_failed_blocks", "value": "15f64"}, + {"name": "received_batches", "value": "15000f64"}, + {"name": "processed_batches", "value": "15000f64"}, + {"name": "processed_rows", "value": "15000f64"}, + {"name": "received_records", "value": "15000f64"}, + {"name": "insert_sqls", "value": "15000f64"}, + {"name": "failed_sqls", "value": "15000f64"}, + {"name": "created_stables", "value": "15000f64"}, + {"name": "created_tables", "value": "15000f64"}, + {"name": "failed_rows", "value": "15000f64"}, + {"name": "failed_points", "value": "15000f64"}, + {"name": "written_blocks", "value": "15000f64"}, + {"name": "failed_blocks", "value": "15000f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_task_opentsdb", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "task_id", "value": "task_id_7"}, + {"name": "task_name", "value": "task_name_7"}, + ], + "metrics": [ + {"name": "total_execute_time", "value": "123456f64"}, + {"name": "total_written_rows", "value": "1200f64"}, + {"name": "total_written_points", "value": "60000f64"}, + {"name": "start_time", "value": "12345678f64"}, + {"name": "written_rows", "value": "1234f64"}, + {"name": "written_points", "value": "123400f64"}, + {"name": "execute_time", "value": "12345678f64"}, + {"name": "total_received_batches", "value": "1234567f64"}, + {"name": "total_processed_batches", "value": "1234f64"}, + {"name": "total_processed_rows", "value": "12345f64"}, + {"name": "total_inserted_sqls", "value": "12f64"}, + {"name": "total_failed_sqls", "value": "123455f64"}, + {"name": "total_created_stables", "value": "12f64"}, + {"name": "total_created_tables", "value": "123f64"}, + {"name": "total_failed_rows", "value": "12345678f64"}, + {"name": "total_failed_point", "value": "12345f64"}, + {"name": "total_written_blocks", "value": "123456f64"}, + {"name": "total_failed_blocks", "value": "15f64"}, + {"name": "received_batches", "value": "15000f64"}, + {"name": "processed_batches", "value": "15000f64"}, + {"name": "processed_rows", "value": "15000f64"}, + {"name": "received_records", "value": "15000f64"}, + {"name": "insert_sqls", "value": "15000f64"}, + {"name": "failed_sqls", "value": "15000f64"}, + {"name": "created_stables", "value": "15000f64"}, + {"name": "created_tables", "value": "15000f64"}, + {"name": "failed_rows", "value": "15000f64"}, + {"name": "failed_points", "value": "15000f64"}, + {"name": "written_blocks", "value": "15000f64"}, + {"name": "failed_blocks", "value": "15000f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_task_pi", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "task_id", "value": "task_id_8"}, + {"name": "task_name", "value": "task_name_8"}, + ], + "metrics": [ + {"name": "total_execute_time", "value": "123456f64"}, + {"name": "total_written_rows", "value": "1200f64"}, + {"name": "total_written_points", "value": "60000f64"}, + {"name": "start_time", "value": "12345678f64"}, + {"name": "written_rows", "value": "1234f64"}, + {"name": "written_points", "value": "123400f64"}, + {"name": "execute_time", "value": "12345678f64"}, + {"name": "total_received_batches", "value": "1234567f64"}, + {"name": "total_processed_batches", "value": "1234f64"}, + {"name": "total_processed_rows", "value": "12345f64"}, + {"name": "total_inserted_sqls", "value": "12f64"}, + {"name": "total_failed_sqls", "value": "123455f64"}, + {"name": "total_created_stables", "value": "12f64"}, + {"name": "total_created_tables", "value": "123f64"}, + {"name": "total_failed_rows", "value": "12345678f64"}, + {"name": "total_failed_point", "value": "12345f64"}, + {"name": "total_written_blocks", "value": "123456f64"}, + {"name": "total_failed_blocks", "value": "15f64"}, + {"name": "received_batches", "value": "15000f64"}, + {"name": "processed_batches", "value": "15000f64"}, + {"name": "processed_rows", "value": "15000f64"}, + {"name": "received_records", "value": "15000f64"}, + {"name": "insert_sqls", "value": "15000f64"}, + {"name": "failed_sqls", "value": "15000f64"}, + {"name": "created_stables", "value": "15000f64"}, + {"name": "created_tables", "value": "15000f64"}, + {"name": "failed_rows", "value": "15000f64"}, + {"name": "failed_points", "value": "15000f64"}, + {"name": "written_blocks", "value": "15000f64"}, + {"name": "failed_blocks", "value": "15000f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_task_pibackfill", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "task_id", "value": "task_id_9"}, + {"name": "task_name", "value": "task_name_9"}, + ], + "metrics": [ + {"name": "total_execute_time", "value": "123456f64"}, + {"name": "total_written_rows", "value": "1200f64"}, + {"name": "total_written_points", "value": "60000f64"}, + {"name": "start_time", "value": "12345678f64"}, + {"name": "written_rows", "value": "1234f64"}, + {"name": "written_points", "value": "123400f64"}, + {"name": "execute_time", "value": "12345678f64"}, + {"name": "total_received_batches", "value": "1234567f64"}, + {"name": "total_processed_batches", "value": "1234f64"}, + {"name": "total_processed_rows", "value": "12345f64"}, + {"name": "total_inserted_sqls", "value": "12f64"}, + {"name": "total_failed_sqls", "value": "123455f64"}, + {"name": "total_created_stables", "value": "12f64"}, + {"name": "total_created_tables", "value": "123f64"}, + {"name": "total_failed_rows", "value": "12345678f64"}, + {"name": "total_failed_point", "value": "12345f64"}, + {"name": "total_written_blocks", "value": "123456f64"}, + {"name": "total_failed_blocks", "value": "15f64"}, + {"name": "received_batches", "value": "15000f64"}, + {"name": "processed_batches", "value": "15000f64"}, + {"name": "processed_rows", "value": "15000f64"}, + {"name": "received_records", "value": "15000f64"}, + {"name": "insert_sqls", "value": "15000f64"}, + {"name": "failed_sqls", "value": "15000f64"}, + {"name": "created_stables", "value": "15000f64"}, + {"name": "created_tables", "value": "15000f64"}, + {"name": "failed_rows", "value": "15000f64"}, + {"name": "failed_points", "value": "15000f64"}, + {"name": "written_blocks", "value": "15000f64"}, + {"name": "failed_blocks", "value": "15000f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_task_csv", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "task_id", "value": "task_id_10"}, + {"name": "task_name", "value": "task_name_10"}, + ], + "metrics": [ + {"name": "total_execute_time", "value": "123456f64"}, + {"name": "total_written_rows", "value": "1200f64"}, + {"name": "total_written_points", "value": "60000f64"}, + {"name": "start_time", "value": "12345678f64"}, + {"name": "written_rows", "value": "1234f64"}, + {"name": "written_points", "value": "123400f64"}, + {"name": "execute_time", "value": "12345678f64"}, + {"name": "total_received_batches", "value": "1234567f64"}, + {"name": "total_processed_batches", "value": "1234f64"}, + {"name": "total_processed_rows", "value": "12345f64"}, + {"name": "total_inserted_sqls", "value": "12f64"}, + {"name": "total_failed_sqls", "value": "123455f64"}, + {"name": "total_created_stables", "value": "12f64"}, + {"name": "total_created_tables", "value": "123f64"}, + {"name": "total_failed_rows", "value": "12345678f64"}, + {"name": "total_failed_point", "value": "12345f64"}, + {"name": "total_written_blocks", "value": "123456f64"}, + {"name": "total_failed_blocks", "value": "15f64"}, + {"name": "received_batches", "value": "15000f64"}, + {"name": "processed_batches", "value": "15000f64"}, + {"name": "processed_rows", "value": "15000f64"}, + {"name": "received_records", "value": "15000f64"}, + {"name": "insert_sqls", "value": "15000f64"}, + {"name": "failed_sqls", "value": "15000f64"}, + {"name": "created_stables", "value": "15000f64"}, + {"name": "created_tables", "value": "15000f64"}, + {"name": "failed_rows", "value": "15000f64"}, + {"name": "failed_points", "value": "15000f64"}, + {"name": "written_blocks", "value": "15000f64"}, + {"name": "failed_blocks", "value": "15000f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_task_avevahistorian", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "task_id", "value": "task_id_11"}, + {"name": "task_name", "value": "task_name_11"}, + ], + "metrics": [ + {"name": "total_execute_time", "value": "123456f64"}, + {"name": "total_written_rows", "value": "1200f64"}, + {"name": "total_written_points", "value": "60000f64"}, + {"name": "start_time", "value": "12345678f64"}, + {"name": "written_rows", "value": "1234f64"}, + {"name": "written_points", "value": "123400f64"}, + {"name": "execute_time", "value": "12345678f64"}, + {"name": "total_received_batches", "value": "1234567f64"}, + {"name": "total_processed_batches", "value": "1234f64"}, + {"name": "total_processed_rows", "value": "12345f64"}, + {"name": "total_inserted_sqls", "value": "12f64"}, + {"name": "total_failed_sqls", "value": "123455f64"}, + {"name": "total_created_stables", "value": "12f64"}, + {"name": "total_created_tables", "value": "123f64"}, + {"name": "total_failed_rows", "value": "12345678f64"}, + {"name": "total_failed_point", "value": "12345f64"}, + {"name": "total_written_blocks", "value": "123456f64"}, + {"name": "total_failed_blocks", "value": "15f64"}, + {"name": "received_batches", "value": "15000f64"}, + {"name": "processed_batches", "value": "15000f64"}, + {"name": "processed_rows", "value": "15000f64"}, + {"name": "received_records", "value": "15000f64"}, + {"name": "insert_sqls", "value": "15000f64"}, + {"name": "failed_sqls", "value": "15000f64"}, + {"name": "created_stables", "value": "15000f64"}, + {"name": "created_tables", "value": "15000f64"}, + {"name": "failed_rows", "value": "15000f64"}, + {"name": "failed_points", "value": "15000f64"}, + {"name": "written_blocks", "value": "15000f64"}, + {"name": "failed_blocks", "value": "15000f64"}, + ], + }, + { + "ts": "1703484955810", + "stable_name": "taosx_task_kafka", + "tags": [ + {"name": "taosx_id", "value": "taosx_id_1"}, + {"name": "task_id", "value": "task_id_12"}, + {"name": "task_name", "value": "task_name_12"}, + ], + "metrics": [ + {"name": "total_execute_time", "value": "123456f64"}, + {"name": "total_written_rows", "value": "1200f64"}, + {"name": "total_written_points", "value": "60000f64"}, + {"name": "start_time", "value": "12345678f64"}, + {"name": "written_rows", "value": "1234f64"}, + {"name": "written_points", "value": "123400f64"}, + {"name": "execute_time", "value": "12345678f64"}, + {"name": "total_received_batches", "value": "1234567f64"}, + {"name": "total_processed_batches", "value": "1234f64"}, + {"name": "total_processed_rows", "value": "12345f64"}, + {"name": "total_inserted_sqls", "value": "12f64"}, + {"name": "total_failed_sqls", "value": "123455f64"}, + {"name": "total_created_stables", "value": "12f64"}, + {"name": "total_created_tables", "value": "123f64"}, + {"name": "total_failed_rows", "value": "12345678f64"}, + {"name": "total_failed_point", "value": "12345f64"}, + {"name": "total_written_blocks", "value": "123456f64"}, + {"name": "total_failed_blocks", "value": "15f64"}, + {"name": "received_batches", "value": "15000f64"}, + {"name": "processed_batches", "value": "15000f64"}, + {"name": "processed_rows", "value": "15000f64"}, + {"name": "received_records", "value": "15000f64"}, + {"name": "insert_sqls", "value": "15000f64"}, + {"name": "failed_sqls", "value": "15000f64"}, + {"name": "created_stables", "value": "15000f64"}, + {"name": "created_tables", "value": "15000f64"}, + {"name": "failed_rows", "value": "15000f64"}, + {"name": "failed_points", "value": "15000f64"}, + {"name": "written_blocks", "value": "15000f64"}, + {"name": "failed_blocks", "value": "15000f64"}, + ], + }, +] diff --git a/scripts/data-generation/tdengine/taos.cfg b/scripts/data-generation/tdengine/taos.cfg new file mode 100644 index 0000000..da2ed39 --- /dev/null +++ b/scripts/data-generation/tdengine/taos.cfg @@ -0,0 +1,193 @@ +######################################################## +# # +# Configuration # +# # +######################################################## + +######### 0. Client only configurations ############# + +# The interval for CLI to send heartbeat to mnode +# shellActivityTimer 3 + + +############### 1. Cluster End point ############################ + +# The end point of the first dnode in the cluster to be connected to when this dnode or the CLI utility is started +# firstEp hostname:6030 + +# The end point of the second dnode to be connected to if the firstEp is not available +# secondEp + + +############### 2. Configuration Parameters of current dnode ##### + +# The FQDN of the host on which this dnode will be started. It can be IP address +fqdn tdengine + +# The port for external access after this dnode is started +# serverPort 6030 + +# The maximum number of connections a dnode can accept +# maxShellConns 5000 + +# The directory for writing log files, if you are using Windows platform please change to Windows path +# logDir /var/log/taos + +# All data files are stored in this directory, if you are using Windows platform please change to Windows path +# dataDir /var/lib/taos + +# temporary file's directory, if you are using Windows platform please change to Windows path +# tempDir /tmp/ + +# Switch for allowing to collect and report service usage information +# telemetryReporting 1 + +# Switch for allowing to collect and report crash information +# crashReporting 1 + +# The maximum number of vnodes supported by this dnode +supportVnodes 256 + +# The interval of this dnode reporting status to mnode, [1..10] seconds +# statusInterval 1 + +# The minimum sliding window time, milli-second +# minSlidingTime 10 + +# The minimum time window, milli-second +# minIntervalTime 10 + +# The maximum allowed query buffer size in MB during query processing for each data node +# -1 no limit (default) +# 0 no query allowed, queries are disabled +# queryBufferSize -1 + +# The compressed rpc message, option: +# -1 (no compression) +# 0 (all message compressed), +# > 0 (rpc message body which larger than this value will be compressed) +# compressMsgSize -1 + +# query retrieved column data compression option: +# -1 (no compression) +# 0 (all retrieved column data compressed), +# > 0 (any retrieved column size greater than this value all data will be compressed.) +# compressColData -1 + +# system time zone +# timezone UTC-8 + +# system time zone (for windows 10) +# timezone Asia/Shanghai (CST, +0800) + +# system locale +# locale en_US.UTF-8 + +# system charset +# charset UTF-8 + +# stop writing logs when the disk size of the log folder is less than this value +# minimalLogDirGB 1.0 + +# stop writing temporary files when the disk size of the tmp folder is less than this value +# minimalTmpDirGB 1.0 + +# if free disk space is less than this value, this dnode will fail to start +# minimalDataDirGB 2.0 + +# enable/disable system monitor +# monitor 1 + +# enable/disable audit log +# audit 1 + +# enable/disable audit create table +# auditCreateTable 1 + +# The following parameter is used to limit the maximum number of lines in log files. +# max number of lines per log filters +# numOfLogLines 10000000 + +# write log in async way: 1 - async, 0 - sync +# asyncLog 1 + +# time period of keeping log files, in days +# logKeepDays 0 + + +############ 3. Debug Flag and levels ############################################# + +# The following parameters are used for debug purpose only by this dnode. +# debugFlag is a 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR +# Available debug levels are: +# 131: output warning and error +# 135: output debug, warning and error +# 143: output trace, debug, warning and error to log +# 199: output debug, warning and error to both screen and file +# 207: output trace, debug, warning and error to both screen and file + +# debug flag for all log type, take effect when non-zero value +# debugFlag 0 + +# debug flag for timer +# tmrDebugFlag 131 + +# debug flag for util +# uDebugFlag 131 + +# debug flag for rpc +# rpcDebugFlag 131 + +# debug flag for jni +# jniDebugFlag 131 + +# debug flag for query +# qDebugFlag 131 + +# debug flag for client driver +# cDebugFlag 131 + +# debug flag for dnode messages +# dDebugFlag 135 + +# debug flag for vnode +# vDebugFlag 131 + +# debug flag for meta management messages +# mDebugFlag 135 + +# debug flag for wal +# wDebugFlag 135 + +# debug flag for sync module +# sDebugFlag 135 + +# debug flag for tsdb +# tsdbDebugFlag 131 + +# debug flag for tq +# tqDebugFlag 131 + +# debug flag for fs +# fsDebugFlag 131 + +# debug flag for udf +# udfDebugFlag 131 + +# debug flag for sma +# smaDebugFlag 131 + +# debug flag for index +# idxDebugFlag 131 + +# debug flag for tdb +# tdbDebugFlag 131 + +# debug flag for meta +# metaDebugFlag 131 + +# generate core file when service crash +# enableCoreFile 1 +monitor 1 +monitorFQDN taoskeeper +audit 1