diff --git a/docs/monitoring.md b/docs/monitoring.md new file mode 100644 index 0000000..d52b802 --- /dev/null +++ b/docs/monitoring.md @@ -0,0 +1,202 @@ +# Spacebox monitoring + +Spacebox supports text-based metrics, compatible with [Prometheus](https://github.com/prometheus/prometheus). Port is mapped to the localhost in default docker-compose configuration, and could be configured in `.env`: + +```bash +# Metrics settings +METRICS_ENABLED=true # Metrics enabled +SERVER_PORT=2112 # Metrics server port +``` + +To make it appear in Prometheus, launch crawler add a new target to your prometheus.yml, and restart Prometheus: + +```bash + - job_name: 'spacebox-crawler' + scrape_interval: 5s + static_configs: + - targets: ['localhost:2112'] +``` + +An example of the Grafana dashboard can be found [here](./spacebox_grafana_sample.json). To make this board fully functional it's necessary to install [ClickHouse plugin](https://grafana.com/grafana/plugins/grafana-clickhouse-datasource/) to Grafana. + +List of the supported metrics with some sample values: + +```bash +# HELP client_api_requests_total A counter for requests from the wrapped client. +# TYPE client_api_requests_total counter +client_api_requests_total{code="200",method="post"} 235915 +# HELP client_in_flight_requests A gauge of in-flight requests for the wrapped client. +# TYPE client_in_flight_requests gauge +client_in_flight_requests 0 +# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="*"} 0.00013202 +go_gc_duration_seconds_sum 569.665748093 +go_gc_duration_seconds_count 320180 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 144 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{version="go1.22.4"} 1 +# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes 3.5313344e+08 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total 1.49838467006912e+14 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes 7.119855e+06 +# HELP go_memstats_frees_total Total number of frees. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total 2.228701493759e+12 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes 4.607688e+07 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes 3.5313344e+08 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes 3.821551616e+09 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes 3.74980608e+08 +# HELP go_memstats_heap_objects Number of allocated objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects 4.765757e+06 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes 3.750273024e+09 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes 4.196532224e+09 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds 1.726229858651255e+09 +# HELP go_memstats_lookups_total Total number of pointer lookups. +# TYPE go_memstats_lookups_total counter +go_memstats_lookups_total 0 +# HELP go_memstats_mallocs_total Total number of mallocs. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total 2.228706259516e+12 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes 105600 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes 109200 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes 5.52096e+06 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes 6.190176e+07 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes 4.86296432e+08 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes 1.5349049e+07 +# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes 1.0354688e+07 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes 1.0354688e+07 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes 4.337443656e+09 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads 100 +# HELP grpc_client_handled_total Total number of RPCs completed by the client, regardless of success or failure. +# TYPE grpc_client_handled_total counter +grpc_client_handled_total{grpc_code="OK",grpc_method="*",grpc_service="*",grpc_type="unary"} 117957 +# HELP grpc_client_handling_seconds Histogram of response latency (seconds) of the gRPC until it is finished by the application. +# TYPE grpc_client_handling_seconds histogram +grpc_client_handling_seconds_bucket{grpc_method="*",grpc_service="*",grpc_type="unary",le="0.005"} 0 +grpc_client_handling_seconds_sum{grpc_method="*",grpc_service="*",grpc_type="unary"} 1704.9106808000047 +grpc_client_handling_seconds_count{grpc_method="*",grpc_service="*",grpc_type="unary"} 117957 +# HELP grpc_client_msg_received_total Total number of RPC stream messages received by the client. +# TYPE grpc_client_msg_received_total counter +grpc_client_msg_received_total{grpc_method="*",grpc_service="*",grpc_type="unary"} 117957 +# HELP grpc_client_msg_sent_total Total number of gRPC stream messages sent by the client. +# TYPE grpc_client_msg_sent_total counter +grpc_client_msg_sent_total{grpc_method="*",grpc_service="*",grpc_type="unary"} 117957 +# HELP grpc_client_started_total Total number of RPCs started on the client. +# TYPE grpc_client_started_total counter +grpc_client_started_total{grpc_method="*",grpc_service="*",grpc_type="unary"} 117957 +# HELP mongo_commands Histogram of MongoDB commands +# TYPE mongo_commands histogram +mongo_commands_bucket{command="*",instance="*",le="*"} 6706 +mongo_commands_sum{command="*",instance="*"} 46649.20434609119 +mongo_commands_count{command="*",instance="*"} 13632 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 3.77212593e+06 +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 1.048576e+06 +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 36 +# HELP process_resident_memory_bytes Resident memory size in bytes. +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 5.58522368e+08 +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.72552562346e+09 +# HELP process_virtual_memory_bytes Virtual memory size in bytes. +# TYPE process_virtual_memory_bytes gauge +process_virtual_memory_bytes 5.791121408e+09 +# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. +# TYPE process_virtual_memory_max_bytes gauge +process_virtual_memory_max_bytes 1.8446744073709552e+19 +# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served. +# TYPE promhttp_metric_handler_requests_in_flight gauge +promhttp_metric_handler_requests_in_flight 1 +# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code. +# TYPE promhttp_metric_handler_requests_total counter +promhttp_metric_handler_requests_total{code="*"} 140841 +# HELP request_duration_seconds A histogram of request latencies. +# TYPE request_duration_seconds histogram +request_duration_seconds_bucket{method="post",le="*"} 1 +request_duration_seconds_sum{method="post"} 5616.343836235997 +request_duration_seconds_count{method="post"} 235915 +# HELP spacebox_crawler_blocks_by_status Total blocks for each status +# TYPE spacebox_crawler_blocks_by_status gauge +spacebox_crawler_blocks_by_status{status="error"} 1.046201e+06 +spacebox_crawler_blocks_by_status{status="processed"} 1.6964531e+07 +spacebox_crawler_blocks_by_status{status="processing"} 0 +# HELP spacebox_crawler_last_processed_block_height Last processed block height +# TYPE spacebox_crawler_last_processed_block_height gauge +spacebox_crawler_last_processed_block_height 2.216532e+07 +# HELP spacebox_crawler_process_duration Duration of parsed blockchain objects +# TYPE spacebox_crawler_process_duration histogram +spacebox_crawler_process_duration_bucket{type="*",le="*"} 117957 +spacebox_crawler_process_duration_sum{type="*"} 0.48113770700000247 +spacebox_crawler_process_duration_count{type="*"} 117957 +# HELP spacebox_crawler_start_height Start height for processing +# TYPE spacebox_crawler_start_height gauge +spacebox_crawler_start_height 1.8e+07 +# HELP spacebox_crawler_stop_height Stop height for processing +# TYPE spacebox_crawler_stop_height gauge +spacebox_crawler_stop_height 2.2047366e+07 +# HELP spacebox_crawler_total_error_messages Total error messages +# TYPE spacebox_crawler_total_error_messages gauge +spacebox_crawler_total_error_messages 0 +# HELP spacebox_crawler_total_error_txs Total error txs +# TYPE spacebox_crawler_total_error_txs gauge +spacebox_crawler_total_error_txs 6.719612e+06 +# HELP spacebox_crawler_total_workers Count of workers +# TYPE spacebox_crawler_total_workers gauge +spacebox_crawler_total_workers 5 +# HELP spacebox_crawler_version Crawler version +# TYPE spacebox_crawler_version gauge +spacebox_crawler_version{version="v2.0.0"} 1 +# HELP spacebox_crawler_worker_recovery_mode Is worker recovery mode enabled +# TYPE spacebox_crawler_worker_recovery_mode gauge +spacebox_crawler_worker_recovery_mode 0 +``` diff --git a/docs/setup.md b/docs/setup.md new file mode 100644 index 0000000..adbc61b --- /dev/null +++ b/docs/setup.md @@ -0,0 +1,412 @@ +# Setup + +## Requirements + +To operate the Spacebox you'll need an x86-64/arm machine with >8GB of RAM, and >4 CPU cores. For storage it is better to use faster SSD or NVME-like drives, size depends on the chain and TX volume(cosmoshub-4 index utilizes ~1 TB). Spacebox runs in [Docker](https://docs.docker.com/engine/install/) with help of[Docker-compose](https://docs.docker.com/compose/install/). Also, you will need the RPC and GRPC endpoints of the chain, which you will index with the appropriate historical state. + +## Quick start + +### Get + +Clone the [Spacebox](https://github.com/bro-n-bro/spacebox.git) repo, it contains everything to get started. + +`console` + +```console +$ git clone https://github.com/bro-n-bro/spacebox.git +Cloning into 'spacebox'... +remote: Enumerating objects: 1111, done. +remote: Counting objects: 100% (59/59), done. +remote: Compressing objects: 100% (41/41), done. +remote: Total 1111 (delta 17), reused 23 (delta 16), pack-reused 1052 +Receiving objects: 100% (1111/1111), 2.82 MiB | 10.48 MiB/s, done. +Resolving deltas: 100% (601/601), done. +$ ls spacebox/ +config docker-compose-local.yaml docker-compose.yaml docs go.mod LICENSE migrations mkdocs.yml README.md +``` + +### Set + +Fill .env with chain settings like chain prefix, node RPC & GRPC, and start \ stop height: + +`.env` + +```bash +START_HEIGHT=5048767 # Start block height +STOP_HEIGHT=0 # Stop block height, 0 for actual height +WORKERS_COUNT=15 # Go workers to pull data in async mode +SUBSCRIBE_NEW_BLOCKS=true # pull actual blocks + + +# Chain settings +CHAIN_PREFIX=cosmos # Prefix of indexing chain +RPC_URL=http://0.0.0.0:26657 # RPC API +GRPC_URL=0.0.0.0:9090 # GRPC API, no HTTP\S prefix +GRPC_SECURE_CONNECTION=false # GRPC secure connection +``` + +### Launch + +When `.env` is filled to start all containers run: + +`console` + +```bash +$ docker-compose up -d +Creating network "spacebox_default" with the default driver +Creating spacebox_hasura_1 ... done +Creating spacebox_mongo-crawler_1 ... done +Creating spacebox_ndc-clickhouse_1 ... done +Creating spacebox_clickhouse_1 ... done +Creating spacebox-zookeeper ... done +Creating spacebox_postgres_1 ... done +Creating spacebox_kafka_1 ... done +Creating spacebox-kafka-ui ... done +Creating spacebox_migration_1 ... done +``` + +If everything is set correctly you will see the following in `spacebox_crawler_1`: + +`console` + +```console +$ docker logs spacebox_crawler_1 -f +INF starting app cmp=app version=v1.2.0 +INF module registered cmp=app name=raw version=v1.2.0 +INF all modules registered cmp=app count=1 version=v1.2.0 +INF starting cmp=app service=storage version=v1.2.0 +INF started cmp=app service=storage version=v1.2.0 +INF starting cmp=app service=grpc_client version=v1.2.0 +INF started cmp=app service=grpc_client version=v1.2.0 +INF starting cmp=app service=rpc_client version=v1.2.0 +INF started cmp=app service=rpc_client version=v1.2.0 +INF starting cmp=app service=broker version=v1.2.0 +INF started cmp=app service=broker version=v1.2.0 +INF starting cmp=app service=worker version=v1.2.0 +INF started cmp=app service=worker version=v1.2.0 +INF starting cmp=app service=server version=v1.2.0 +INF listening for new block events cmp=worker version=v1.2.0 +INF started cmp=app service=server version=v1.2.0 +INF starting cmp=app service=health_checker version=v1.2.0 +INF exit not needed cmp=worker version=v1.2.0 +INF started cmp=app service=health_checker version=v1.2.0 +INF start metrics scraper cmp=server version=v1.2.0 +INF application started cmp=app version=v1.2.0 +INF parse block cmp=worker height=1 version=v1.2.0 worker_number=1 +INF parse block cmp=worker height=2 version=v1.2.0 worker_number=11 +INF parse block cmp=worker height=4 version=v1.2.0 worker_number=19 +INF parse block cmp=worker height=5 version=v1.2.0 worker_number=17 +INF parse block cmp=worker height=3 version=v1.2.0 worker_number=6 +``` + +To stop and remove all containers use the following: + +`console` + +```bash +docker-compose down +``` + +### Issues? + +If after `docker-compose up` you see the error: + +`console` + +```console +ERROR: for crawler Container "d0d8ccc463b8" is unhealthy. +ERROR: Encountered errors while bringing up the project. +``` + +Then it is necessary to grant the ownership of the `volumes` folder to the docker user (uid 1001): + +`console` + +```console +$ docker-compose down +$ chown -R 1001:1001 volumes/ +$ docker-compose up -d +``` + +It usually requires around ~2 minutes to start everything. Check the log of each container with `docker logs ` to see what's going under the hood. It is possible to adjust some parameters on the fly, edit `.env`, and restart the appropriate container. + +## Upgrading + +Spacebox main part [spacebox-crawler](https://github.com/bro-n-bro/spacebox-crawler/releases) follows [semantic versioning:](https://semver.org/) `MAJOR.MINOR.PATCH` + some releases are chain-specific and contain chain name in the tag, e.g. `v2.0.0-neutron`. + +As long as spacebox is considered as a beta software, every major version upgrade will require re-sync, check the release details for more info on that. + +We recommend using a specified version for the crawler in `docker-compose.yml`, instead of `:latest` that will ease troubleshooting and upgrades: + +`docker-compose.yaml` + +```console +version: "3.9" + +services: + crawler: + image: bronbro/spacebox-crawler:v1.2.0 +``` + +For upgrade with full resync stop everything remove the volmes folder and start over with the newer version of the crawler: + +`console` + +```console +$ docker-compose down +$ rm -rf volumes/ +$ docker-compose up -d +``` + +If an upgrade is happening alongside the chain upgrade, then it is possible to relaunch only the crawler: + +`console` + +```bash +$ docker stop spacebox_crawler-1 +$ docker rm spacebox_crawler-1 +``` + +Set new version in `docker-compose.yaml` + +```diff +services: + crawler: +- image: bronbro/spacebox-crawler:v1.2.0 ++ image: bronbro/spacebox-crawler:v1.3.0 +``` + +Start crawler container + +`console` + +```bash +docker-compose up -d crawler +``` + +In such case, indexation with the new version will start since the block you will upgrade the crawler's container. To avoid data inconsistency it might be a good idea to set the upgrade height in the .env as `STOP_HEIGHT=` and perform the container upgrade safely. + +## Config options + +Spacebox is configured over environment variables, set in the `.env` file. + +> [!TIP] +> To apply .env changes just restart the `spacebox_crawler-1` container. However same does not apply to the rest of the containers(aka kafka or clickhouse), and they would need to be **recreated.** + +### Crawler + +--- + +`START_TIMEOUT` + +Timeout to start application. Do not change without a strict purpose. + +--- + +`STOP_TIMEOUT` + +Timeout to stop application. Do not change without a strict purpose. + +--- + +`LOG_LEVEL` + +Supported values: `info`, `debug`, `error`. + +Debug level will produce more logs, including checks of the previously parsed blocks. Error log will show only errors if there's any. + +--- + +`RECOVERY_MODE` + +Default value: false + +Intended for debugging purposes mostly. In case of *`panic`* will output a full error log, without crashing the crawler. Significantly(!) lowers the perfomance. + +--- + +`START_HEIGHT` + +Default value: 0 + +Height to start crawling from. Ex: for `cosmoshub-4` must be set to 5200792 as it was the first block in this chain. +If set to 0 genesis will be parced automatically. + +--- + +`STOP_HEIGHT` + +Default value: 0 + +Height to stop indexing at. Might be useful when necessary to index only certain parts of the chain. When set to 0 crawler will pull the latest block from the RPC at the moment of launch and use it as stop height. + +--- + +`WORKERS_COUNT` + +A number of the asynchronous Go workers inside the crawler. Recommended value: **<=** *CPU cores* in your system. A huge number of workers may exhaust the RPC node, resulting in missed blocks. Usually, more workers don't mean faster crawling, best performing number is 30-40 workers (even on 88-core machine). + +--- + +`SUBSCRIBE_NEW_BLOCKS` + +Default value: true + +If set to **false**, the crawler will stop indexation on `STOP_HEIGHT` without an app crash. If set to **true**, the crawler will subscribe to the new coming blocks over the websocket and will parce them alongside to historical blocks. +>[Warning!] If SUBSCRIBE_NEW_BLOCKS=true `spacebox_crawler_last_processed_block_height` metric will display the highest indexed block, even though historical blocks might still be processed. + +--- + +`PROCESS_ERROR_BLOCKS` + +If set to **true** crawler will attempt to re-index the blocks that had errors during previous processing(based on info stored in Mongo). Re-indexation would be attempted every `PROCESS_ERROR_BLOCKS_INTERVAL`. + +--- + +`PROCESS_GENESIS` + +If set to **true** crawler will parce genesis to have genesis initial data in db. If `START_HEIGHT` set to 0 genesis would be parced as well. + +### Text metrics + +`METRICS_ENABLED` + +Enable or disable crawler text metrics endpoint. Compatible with Prometheus. + +`SERVER_PORT` + +Specify the port for text metrics. + +--- + +### Chain settings + +`CHAIN_PREFIX` + +Specify the account prefix for the chain you going to index. Typically found in the beginning of every address, e.g. for `cosmos106yp7zw35wftheyyv9f9pe69t8rteumjxjql7m` chain prefix is `cosmos`. + +`RPC_URL` & `GRPC_URL` + +`GRPC_TIMEOUT` + +Timeout for the GRPC requests. Default value: 15s. + +`RPC_TIMEOUT` + +Timeout for the RPC requests. Default value: 15s. + +Node RPC and GRPC endpoints. By default served on port 26657 and 9090. + +>[TIP!] For better performance, we advise running the indexer as "close" to the chain node as possible, ideally on the same machine. + +If you running both crawler and chain node on the same host you need to add the following line to the crawler's `docker-compose.yaml` section: + +`docker-compose.yaml` + +```diff + ports: + - '2112:2112' ++ extra_hosts: ++ - "host.docker.internal:host-gateway" +``` + +And set RPC and GRPC addresses in the `.env` accordingly: + +```bash +RPC_URL=http://host.docker.internal:26657 +GRPC_URL=host.docker.internal:9090 +``` + +Also, depending on your firewall setup, it might be required to allow connections from the docker subnet to the chain endpoint. + +`GRPC_SECURE_CONNECTION` + +Set to **true** if your GRPC endpoint running with SSL encryption enabled. Leave as **false** if connecting directly to node port or within the same localhost. + +`GRPC_MAX_RECEIVE_MESSAGE_SIZE_BYTES` + +Defines the maximum size of a single message crawler will process. If you see something similar to `...failed to get block: rpc error: code = ResourceExhausted desc = grpc: received message larger than max...` in the crawler log - increase this parameter and restart the container with spacebox_cralwer. Chains might have very heavy blocks, for ex. neutron's testnet pion-1 have some block_results of the size ~170mb. + +--- + +### Broker settings + +`BROKER_SERVER` + +Address of Kafka broker. Do not change without a strict purpose. + +`BROKER_ENABLED` + +Enable\disable message publishing messages to the broker. Do not change without a strict purpose. + +`PARTITIONS_COUNT` + +Count of the partitions in the broker. Do not change without a strict purpose. + +`MAX_MESSAGE_MAX_BYTES` + +Define the maximum size of the message in Kafka. Requires Kafka container recreation. Default value: 5242880 (5Mb). + +`BATCH_PRODUCER` + +Enable batch producer to process messages in Kafka by batches. Might increase performance, but considered an experimental feature. + +`KAFKA_UI_PASSWORD` + +Password for kafka-UI. Change for production deployment! + +--- + +### Mongo settings + +`MONGO_CRAWLER_URI` + +Address of Mongo db. Crawler utilizes mongo-db as a check-list for parsed block status. + +`MONGO_USER` + +Mongo username used by the crawler. + +`MONGO_PASSWORD` + +Mongo user password used by the crawler. Change for production deployment! + +`MAX_POOL_SIZE` + +Maximum pool size in Mongo. Do not change without a strict purpose. + +`MAX_CONNECTING` + +Maximum Mongo connections. Do not change without a strict purpose. + +--- + +### Clickhouse settings + +`CLICKHOUSE_PASSWORD` + +Password for clickhouse `default` user. Change for production deployment! Default user will require some privileges upgrade to allow it [new users creation](https://clickhouse.com/docs/en/operations/access-rights#enabling-sql-user-mode). + +--- + +### Health checker + +`HEALTHCHECK_ENABLED` + +If set to **true** the crawler will periodically check the health of the node endpoints, and behave accordingly. + +`HEALTHCHECK_FATAL_ON_CHECK` + +If set to **true** the crawler will crush if the health check has failed. That will result in a container restart. + +`HEALTHCHECK_MAX_LAST_BLOCK_LAG` + +Interval for the new block to appear in the RPC, to count endpoint healthy. + +`HEALTHCHECK_INTERVAL` + +Interval to perform a health check. + +`HEALTHCHECK_START_DELAY` + +Delay for the first health check after the start. diff --git a/docs/spacebox_grafana_sample.json b/docs/spacebox_grafana_sample.json new file mode 100644 index 0000000..9629276 --- /dev/null +++ b/docs/spacebox_grafana_sample.json @@ -0,0 +1,2966 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 151, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "builderOptions": { + "database": "", + "fields": [], + "limit": 100, + "metrics": [ + { + "aggregation": "count", + "field": "" + } + ], + "mode": "aggregate", + "table": "block final" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "format": 1, + "hide": true, + "meta": { + "builderOptions": { + "database": "", + "fields": [], + "limit": 100, + "metrics": [ + { + "aggregation": "count", + "field": "" + } + ], + "mode": "aggregate", + "table": "block final" + } + }, + "queryType": "sql", + "rawSql": "SELECT count() FROM block final LIMIT 100", + "refId": "A" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "SELECT count() FROM raw_block final", + "refId": "B" + } + ], + "title": "Total blocks in Clickhouse", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 5, + "y": 0 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "builderOptions": { + "fields": [ + "height" + ], + "filters": [], + "limit": 1, + "mode": "list", + "orderBy": [ + { + "dir": "DESC", + "name": "height" + } + ], + "table": "block" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "format": 1, + "hide": true, + "meta": { + "builderOptions": { + "fields": [ + "height" + ], + "filters": [], + "limit": 1, + "mode": "list", + "orderBy": [ + { + "dir": "DESC", + "name": "height" + } + ], + "table": "block" + } + }, + "queryType": "sql", + "rawSql": "SELECT height FROM block final ORDER BY height DESC LIMIT 1", + "refId": "A" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "SELECT height FROM raw_block final ORDER BY height DESC LIMIT 1", + "refId": "B" + } + ], + "title": "last block height in Clickhouse", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 10, + "y": 0 + }, + "id": 48, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "AIhBHDr4z" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "SELECT count(*) FROM (\n WITH blocks AS (\n SELECT\n toUInt64(min(height)) as st,\n toUInt64(max(height)) as ed\n FROM\n spacebox.raw_block_results\n )\n SELECT\n arrayJoin(range(st, ed)) as _height FROM blocks\n) AS _blocks\nLEFT JOIN (SELECT toUInt64(height) as height FROM spacebox.raw_block_results) AS blocks ON _blocks._height = blocks.height\nWHERE height = 0;", + "refId": "A" + } + ], + "title": "count of skipped block_results", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 13, + "y": 0 + }, + "id": 49, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "AIhBHDr4z" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "SELECT count(*) FROM (\n WITH blocks AS (\n SELECT\n toUInt64(min(height)) as st,\n toUInt64(max(height)) as ed\n FROM\n spacebox.raw_block\n )\n SELECT\n arrayJoin(range(st, ed)) as _height FROM blocks\n) AS _blocks\nLEFT JOIN (SELECT toUInt64(height) as height FROM spacebox.raw_block) AS blocks ON _blocks._height = blocks.height\nWHERE height = 0;", + "refId": "A" + } + ], + "title": "count of skipped blocks", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 15, + "y": 0 + }, + "id": 50, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "AIhBHDr4z" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "SELECT count(*) FROM (\n\tSELECT height, msgs.height as _height FROM (\n\t\tSELECT height FROM spacebox.raw_block_results\n\t\tWHERE begin_block_events != ''\n\t\tGROUP BY height \n\t) as txs\n\tLEFT JOIN (select height from spacebox.begin_block_events GROUP BY height ) as msgs ON txs.height = msgs.height\n)\nWHERE _height = 0", + "refId": "A" + } + ], + "title": "skipped blocks in begin_block_events", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 51, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "AIhBHDr4z" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "SELECT count(*) FROM (\n\tSELECT height, msgs.height as _height FROM (\n\t\tSELECT height FROM spacebox.raw_block_results\n\t\tWHERE end_block_events != ''\n\t\tGROUP BY height \n\t) as txs\n\tLEFT JOIN (select height from spacebox.end_block_events GROUP BY height ) as msgs ON txs.height = msgs.height\n)\nWHERE _height = 0", + "refId": "A" + } + ], + "title": "skipped blocks in end_block_events", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "b5da250c-0e29-42ea-8316-3a49866bc58e" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 52, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "b5da250c-0e29-42ea-8316-3a49866bc58e" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "SELECT count(*) FROM (\n\tSELECT height, msgs.height as _height FROM (\n\t\tSELECT height FROM spacebox.raw_block_results\n\t\tWHERE JSONExtractString(JSONExtractString(arrayJoin(JSONExtractArrayRaw(JSONExtractString(txs_results)))), 'events') != '[]'\n\t\tGROUP BY height \n\t) as txs\n\tLEFT JOIN (select height from spacebox.txs_events GROUP BY height ) as msgs ON txs.height = msgs.height\n)\nWHERE _height = 0", + "refId": "A" + } + ], + "title": "skipped blocks in txs_events", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 0, + "y": 6 + }, + "id": 53, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "WITH ch AS (\n\tSELECT height, count(height) as c FROM spacebox.raw_block_results\n\tGROUP BY height\n)\nSELECT count(*) as count FROM ch WHERE c != 1", + "refId": "A" + } + ], + "title": "Double block_results", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 5, + "y": 6 + }, + "id": 54, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "WITH ch AS (\n\tSELECT txhash, count(txhash) as c FROM spacebox.raw_transaction\n\tGROUP BY txhash\n)\nSELECT count(*) FROM ch where c != 1", + "refId": "A" + } + ], + "title": "Double transaction", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 10, + "y": 6 + }, + "id": 55, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${clickhouse}" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "WITH ch AS (\n\tSELECT height, count(height) as c FROM spacebox.raw_block\n\tGROUP BY height\n)\nSELECT count(*) as count FROM ch WHERE c != 1", + "refId": "A" + } + ], + "title": "Double blocks", + "transparent": true, + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "711GLNhVz" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "711GLNhVz" + }, + "refId": "A" + } + ], + "title": "Crawler", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#F2495C", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 43, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^version$/", + "values": false + }, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": false, + "expr": "spacebox_crawler_version", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Version", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 7, + "y": 13 + }, + "id": 30, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": true, + "expr": "spacebox_crawler_start_height{job=\"spacebox-crawler\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Start height", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 12, + "y": 13 + }, + "id": 31, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": true, + "expr": "spacebox_crawler_stop_height{job=\"spacebox-crawler\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Stop height", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 17, + "y": 13 + }, + "id": 32, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": true, + "expr": "spacebox_crawler_total_workers{job=\"spacebox-crawler\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Workers count", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "index": 0, + "text": "False" + }, + "1": { + "index": 1, + "text": "True" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 21, + "y": 13 + }, + "id": 45, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": false, + "expr": "spacebox_crawler_worker_recovery_mode", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Recovery mode", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 19 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": true, + "expr": "sum by(exported_instance) (spacebox_crawler_cache_length)", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Cache length", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 6, + "y": 19 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(spacebox_crawler_global_cache{action=\"hit\"}[$__rate_interval])) / (sum(rate(spacebox_crawler_global_cache{action=\"hit\"}[$__rate_interval])) + sum(rate(spacebox_crawler_global_cache{action=\"miss\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A", + "useBackend": false + } + ], + "title": "Cache HIT ratio", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 11, + "y": 19 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": true, + "expr": "sum by(action) (rate(spacebox_crawler_global_cache[$__rate_interval]))", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Cache HIT/MISS global", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 17, + "y": 19 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by(exported_instance,action) (rate(spacebox_crawler_special_cache[$__rate_interval]))", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Cache HIT/MISS by instance", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 0, + "y": 25 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": true, + "expr": "spacebox_crawler_last_processed_block_height{job=\"spacebox-crawler\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Last processed height", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 5, + "y": 25 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.95, sum by(le) (rate(request_duration_seconds_bucket[$__rate_interval])))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "HTTP duration 95", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 15, + "y": 25 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.95, sum by(le, command) (rate(mongo_commands_bucket[$__rate_interval])))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Mongo commands duration 95", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 0, + "y": 31 + }, + "id": 47, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": true, + "expr": "spacebox_crawler_total_error_messages{job=\"spacebox-crawler\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total error messages", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 3, + "y": 31 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "spacebox_crawler_total_error_txs{job=\"spacebox-crawler\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Total error tx", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 19, + "x": 5, + "y": 31 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.95, sum by(le, grpc_method) (rate(grpc_client_handling_seconds_bucket[$__rate_interval])))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "GRPC duration 95", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 19, + "x": 5, + "y": 37 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": true, + "expr": "sum by (status) (spacebox_crawler_blocks_by_status)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Block by status", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.95, sum by(le, type) (rate(spacebox_crawler_process_duration_bucket[$__rate_interval])))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Processing duration 95", + "transparent": true, + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "711GLNhVz" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 25, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "711GLNhVz" + }, + "refId": "A" + } + ], + "title": "go", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 21, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": true, + "expr": "time() - process_start_time_seconds{job=\"spacebox-crawler\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Up Time", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": true, + "expr": "rate(process_cpu_seconds_total{job=\"spacebox-crawler\"}[30s])", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": true, + "expr": "go_memstats_heap_alloc_bytes{job=\"spacebox-crawler\"}", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Heap alloc", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 65 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": true, + "expr": "go_goroutines{job=\"spacebox-crawler\"}", + "format": "time_series", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Goroutines", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 65 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "go_threads{job=\"spacebox-crawler\"}", + "format": "time_series", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Threads", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 65 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${space}" + }, + "exemplar": true, + "expr": "sum(rate(go_gc_duration_seconds_sum{job=\"spacebox-crawler\"}[1m]))", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "GC Time Seconds", + "transparent": true, + "type": "timeseries" + } + ], + "refresh": "15m", + "schemaVersion": 38, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "space", + "multi": false, + "name": "space", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "ClickHouse index demo", + "value": "YfpWCcUVz" + }, + "hide": 0, + "includeAll": false, + "label": "database", + "multi": false, + "name": "clickhouse", + "options": [], + "query": "grafana-clickhouse-datasource", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "spacebox", + "uid": "spacebox_stats", + "version": 39, + "weekStart": "" + } \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index f53a7ab..3ea114d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,6 +1,7 @@ site_name: Spacebox docs nav: - Home: index.md - - Getting started: getting_started.md + - Getting started: setup.md + - Monitoring: monitoring.md theme: name: readthedocs