From 931729d0c0f333d342a82439c272f8296485178a Mon Sep 17 00:00:00 2001
From: bro-n-bro-0 <bro-n-bro-0@users.noreply.github.com>
Date: Tue, 17 Sep 2024 09:56:29 +0000
Subject: [PATCH] deploy: 62255bb9ffe27710529dadcceb165761d4095d2b

---
 404.html                     |    6 +-
 getting_started/index.html   |   15 +-
 index.html                   |   14 +-
 monitoring/index.html        |  315 ++++
 search.html                  |    6 +-
 search/search_index.json     |    2 +-
 setup/index.html             |  410 +++++
 sitemap.xml.gz               |  Bin 127 -> 127 bytes
 spacebox_grafana_sample.json | 2966 ++++++++++++++++++++++++++++++++++
 9 files changed, 3717 insertions(+), 17 deletions(-)
 create mode 100644 monitoring/index.html
 create mode 100644 setup/index.html
 create mode 100644 spacebox_grafana_sample.json

diff --git a/404.html b/404.html
index e7e8237..cc3bc96 100644
--- a/404.html
+++ b/404.html
@@ -37,7 +37,11 @@
                 </li>
               </ul>
               <ul>
-                <li class="toctree-l1"><a class="reference internal" href="/getting_started/">Getting started</a>
+                <li class="toctree-l1"><a class="reference internal" href="/setup/">Getting started</a>
+                </li>
+              </ul>
+              <ul>
+                <li class="toctree-l1"><a class="reference internal" href="/monitoring/">Monitoring</a>
                 </li>
               </ul>
       </div>
diff --git a/getting_started/index.html b/getting_started/index.html
index f736520..6b7c469 100644
--- a/getting_started/index.html
+++ b/getting_started/index.html
@@ -43,10 +43,12 @@
                 <li class="toctree-l1"><a class="reference internal" href="..">Home</a>
                 </li>
               </ul>
-              <ul class="current">
-                <li class="toctree-l1 current"><a class="reference internal current" href="#">Getting started</a>
-    <ul class="current">
-    </ul>
+              <ul>
+                <li class="toctree-l1"><a class="reference internal" href="../setup/">Getting started</a>
+                </li>
+              </ul>
+              <ul>
+                <li class="toctree-l1"><a class="reference internal" href="../monitoring/">Monitoring</a>
                 </li>
               </ul>
       </div>
@@ -76,9 +78,6 @@ <h1 id="getting-started">Getting started</h1>
               
             </div>
           </div><footer>
-    <div class="rst-footer-buttons" role="navigation" aria-label="Footer Navigation">
-        <a href=".." class="btn btn-neutral float-left" title="Home"><span class="icon icon-circle-arrow-left"></span> Previous</a>
-    </div>
 
   <hr/>
 
@@ -100,8 +99,6 @@ <h1 id="getting-started">Getting started</h1>
   <span class="rst-current-version" data-toggle="rst-current-version">
     
     
-      <span><a href=".." style="color: #fcfcfc">&laquo; Previous</a></span>
-    
     
   </span>
 </div>
diff --git a/index.html b/index.html
index 787c3a6..99147fa 100644
--- a/index.html
+++ b/index.html
@@ -46,7 +46,11 @@
                 </li>
               </ul>
               <ul>
-                <li class="toctree-l1"><a class="reference internal" href="getting_started/">Getting started</a>
+                <li class="toctree-l1"><a class="reference internal" href="setup/">Getting started</a>
+                </li>
+              </ul>
+              <ul>
+                <li class="toctree-l1"><a class="reference internal" href="monitoring/">Monitoring</a>
                 </li>
               </ul>
       </div>
@@ -77,7 +81,7 @@ <h1 id="welcome-spacebox-documentation">Welcome Spacebox documentation</h1>
             </div>
           </div><footer>
     <div class="rst-footer-buttons" role="navigation" aria-label="Footer Navigation">
-        <a href="getting_started/" class="btn btn-neutral float-right" title="Getting started">Next <span class="icon icon-circle-arrow-right"></span></a>
+        <a href="setup/" class="btn btn-neutral float-right" title="Getting started">Next <span class="icon icon-circle-arrow-right"></span></a>
     </div>
 
   <hr/>
@@ -101,7 +105,7 @@ <h1 id="welcome-spacebox-documentation">Welcome Spacebox documentation</h1>
     
     
     
-      <span><a href="getting_started/" style="color: #fcfcfc">Next &raquo;</a></span>
+      <span><a href="setup/" style="color: #fcfcfc">Next &raquo;</a></span>
     
   </span>
 </div>
@@ -120,6 +124,6 @@ <h1 id="welcome-spacebox-documentation">Welcome Spacebox documentation</h1>
 </html>
 
 <!--
-MkDocs version : 1.6.0
-Build Date UTC : 2024-08-06 07:26:32.300331+00:00
+MkDocs version : 1.6.1
+Build Date UTC : 2024-09-17 09:56:29.203737+00:00
 -->
diff --git a/monitoring/index.html b/monitoring/index.html
new file mode 100644
index 0000000..58d2bbd
--- /dev/null
+++ b/monitoring/index.html
@@ -0,0 +1,315 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+    <meta charset="utf-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+      <link rel="shortcut icon" href="../img/favicon.ico" />
+    <title>Monitoring - Spacebox docs</title>
+    <link rel="stylesheet" href="../css/theme.css" />
+    <link rel="stylesheet" href="../css/theme_extra.css" />
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/github.min.css" />
+    
+      <script>
+        // Current page data
+        var mkdocs_page_name = "Monitoring";
+        var mkdocs_page_input_path = "monitoring.md";
+        var mkdocs_page_url = null;
+      </script>
+    
+    <!--[if lt IE 9]>
+      <script src="../js/html5shiv.min.js"></script>
+    <![endif]-->
+      <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+      <script>hljs.highlightAll();</script> 
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
+    <div class="wy-side-scroll">
+      <div class="wy-side-nav-search">
+          <a href=".." class="icon icon-home"> Spacebox docs
+        </a><div role="search">
+  <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get">
+      <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" title="Type search term here" />
+  </form>
+</div>
+      </div>
+
+      <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <ul>
+                <li class="toctree-l1"><a class="reference internal" href="..">Home</a>
+                </li>
+              </ul>
+              <ul>
+                <li class="toctree-l1"><a class="reference internal" href="../setup/">Getting started</a>
+                </li>
+              </ul>
+              <ul class="current">
+                <li class="toctree-l1 current"><a class="reference internal current" href="#">Monitoring</a>
+    <ul class="current">
+    </ul>
+                </li>
+              </ul>
+      </div>
+    </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+      <nav class="wy-nav-top" role="navigation" aria-label="Mobile navigation menu">
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="..">Spacebox docs</a>
+        
+      </nav>
+      <div class="wy-nav-content">
+        <div class="rst-content"><div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href=".." class="icon icon-home" aria-label="Docs"></a></li>
+      <li class="breadcrumb-item active">Monitoring</li>
+    <li class="wy-breadcrumbs-aside">
+    </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+            <div class="section" itemprop="articleBody">
+              
+                <h1 id="spacebox-monitoring">Spacebox monitoring</h1>
+<p>Spacebox supports text-based metrics, compatible with <a href="https://github.com/prometheus/prometheus">Prometheus</a>. Port is mapped to the localhost in default docker-compose configuration, and could be configured in <code>.env</code>:</p>
+<pre><code class="language-bash"># Metrics settings
+METRICS_ENABLED=true # Metrics enabled
+SERVER_PORT=2112 # Metrics server port
+</code></pre>
+<p>To make it appear in Prometheus, launch crawler add a new target to your prometheus.yml, and restart Prometheus:</p>
+<pre><code class="language-bash">  - job_name: 'spacebox-crawler'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['localhost:2112']
+</code></pre>
+<p>An example of the Grafana dashboard can be found <a href="../spacebox_grafana_sample.json">here</a>. To make this board fully functional it's necessary to install <a href="https://grafana.com/grafana/plugins/grafana-clickhouse-datasource/">ClickHouse plugin</a> to Grafana.</p>
+<p>List of the supported metrics with some sample values:</p>
+<pre><code class="language-bash"># HELP client_api_requests_total A counter for requests from the wrapped client.
+# TYPE client_api_requests_total counter
+client_api_requests_total{code=&quot;200&quot;,method=&quot;post&quot;} 235915
+# HELP client_in_flight_requests A gauge of in-flight requests for the wrapped client.
+# TYPE client_in_flight_requests gauge
+client_in_flight_requests 0
+# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.
+# TYPE go_gc_duration_seconds summary
+go_gc_duration_seconds{quantile=&quot;*&quot;} 0.00013202
+go_gc_duration_seconds_sum 569.665748093
+go_gc_duration_seconds_count 320180
+# HELP go_goroutines Number of goroutines that currently exist.
+# TYPE go_goroutines gauge
+go_goroutines 144
+# HELP go_info Information about the Go environment.
+# TYPE go_info gauge
+go_info{version=&quot;go1.22.4&quot;} 1
+# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
+# TYPE go_memstats_alloc_bytes gauge
+go_memstats_alloc_bytes 3.5313344e+08
+# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
+# TYPE go_memstats_alloc_bytes_total counter
+go_memstats_alloc_bytes_total 1.49838467006912e+14
+# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table.
+# TYPE go_memstats_buck_hash_sys_bytes gauge
+go_memstats_buck_hash_sys_bytes 7.119855e+06
+# HELP go_memstats_frees_total Total number of frees.
+# TYPE go_memstats_frees_total counter
+go_memstats_frees_total 2.228701493759e+12
+# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata.
+# TYPE go_memstats_gc_sys_bytes gauge
+go_memstats_gc_sys_bytes 4.607688e+07
+# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use.
+# TYPE go_memstats_heap_alloc_bytes gauge
+go_memstats_heap_alloc_bytes 3.5313344e+08
+# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used.
+# TYPE go_memstats_heap_idle_bytes gauge
+go_memstats_heap_idle_bytes 3.821551616e+09
+# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use.
+# TYPE go_memstats_heap_inuse_bytes gauge
+go_memstats_heap_inuse_bytes 3.74980608e+08
+# HELP go_memstats_heap_objects Number of allocated objects.
+# TYPE go_memstats_heap_objects gauge
+go_memstats_heap_objects 4.765757e+06
+# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS.
+# TYPE go_memstats_heap_released_bytes gauge
+go_memstats_heap_released_bytes 3.750273024e+09
+# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system.
+# TYPE go_memstats_heap_sys_bytes gauge
+go_memstats_heap_sys_bytes 4.196532224e+09
+# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection.
+# TYPE go_memstats_last_gc_time_seconds gauge
+go_memstats_last_gc_time_seconds 1.726229858651255e+09
+# HELP go_memstats_lookups_total Total number of pointer lookups.
+# TYPE go_memstats_lookups_total counter
+go_memstats_lookups_total 0
+# HELP go_memstats_mallocs_total Total number of mallocs.
+# TYPE go_memstats_mallocs_total counter
+go_memstats_mallocs_total 2.228706259516e+12
+# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures.
+# TYPE go_memstats_mcache_inuse_bytes gauge
+go_memstats_mcache_inuse_bytes 105600
+# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system.
+# TYPE go_memstats_mcache_sys_bytes gauge
+go_memstats_mcache_sys_bytes 109200
+# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures.
+# TYPE go_memstats_mspan_inuse_bytes gauge
+go_memstats_mspan_inuse_bytes 5.52096e+06
+# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system.
+# TYPE go_memstats_mspan_sys_bytes gauge
+go_memstats_mspan_sys_bytes 6.190176e+07
+# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place.
+# TYPE go_memstats_next_gc_bytes gauge
+go_memstats_next_gc_bytes 4.86296432e+08
+# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations.
+# TYPE go_memstats_other_sys_bytes gauge
+go_memstats_other_sys_bytes 1.5349049e+07
+# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator.
+# TYPE go_memstats_stack_inuse_bytes gauge
+go_memstats_stack_inuse_bytes 1.0354688e+07
+# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator.
+# TYPE go_memstats_stack_sys_bytes gauge
+go_memstats_stack_sys_bytes 1.0354688e+07
+# HELP go_memstats_sys_bytes Number of bytes obtained from system.
+# TYPE go_memstats_sys_bytes gauge
+go_memstats_sys_bytes 4.337443656e+09
+# HELP go_threads Number of OS threads created.
+# TYPE go_threads gauge
+go_threads 100
+# HELP grpc_client_handled_total Total number of RPCs completed by the client, regardless of success or failure.
+# TYPE grpc_client_handled_total counter
+grpc_client_handled_total{grpc_code=&quot;OK&quot;,grpc_method=&quot;*&quot;,grpc_service=&quot;*&quot;,grpc_type=&quot;unary&quot;} 117957
+# HELP grpc_client_handling_seconds Histogram of response latency (seconds) of the gRPC until it is finished by the application.
+# TYPE grpc_client_handling_seconds histogram
+grpc_client_handling_seconds_bucket{grpc_method=&quot;*&quot;,grpc_service=&quot;*&quot;,grpc_type=&quot;unary&quot;,le=&quot;0.005&quot;} 0
+grpc_client_handling_seconds_sum{grpc_method=&quot;*&quot;,grpc_service=&quot;*&quot;,grpc_type=&quot;unary&quot;} 1704.9106808000047
+grpc_client_handling_seconds_count{grpc_method=&quot;*&quot;,grpc_service=&quot;*&quot;,grpc_type=&quot;unary&quot;} 117957
+# HELP grpc_client_msg_received_total Total number of RPC stream messages received by the client.
+# TYPE grpc_client_msg_received_total counter
+grpc_client_msg_received_total{grpc_method=&quot;*&quot;,grpc_service=&quot;*&quot;,grpc_type=&quot;unary&quot;} 117957
+# HELP grpc_client_msg_sent_total Total number of gRPC stream messages sent by the client.
+# TYPE grpc_client_msg_sent_total counter
+grpc_client_msg_sent_total{grpc_method=&quot;*&quot;,grpc_service=&quot;*&quot;,grpc_type=&quot;unary&quot;} 117957
+# HELP grpc_client_started_total Total number of RPCs started on the client.
+# TYPE grpc_client_started_total counter
+grpc_client_started_total{grpc_method=&quot;*&quot;,grpc_service=&quot;*&quot;,grpc_type=&quot;unary&quot;} 117957
+# HELP mongo_commands Histogram of MongoDB commands
+# TYPE mongo_commands histogram
+mongo_commands_bucket{command=&quot;*&quot;,instance=&quot;*&quot;,le=&quot;*&quot;} 6706
+mongo_commands_sum{command=&quot;*&quot;,instance=&quot;*&quot;} 46649.20434609119
+mongo_commands_count{command=&quot;*&quot;,instance=&quot;*&quot;} 13632
+# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
+# TYPE process_cpu_seconds_total counter
+process_cpu_seconds_total 3.77212593e+06
+# HELP process_max_fds Maximum number of open file descriptors.
+# TYPE process_max_fds gauge
+process_max_fds 1.048576e+06
+# HELP process_open_fds Number of open file descriptors.
+# TYPE process_open_fds gauge
+process_open_fds 36
+# HELP process_resident_memory_bytes Resident memory size in bytes.
+# TYPE process_resident_memory_bytes gauge
+process_resident_memory_bytes 5.58522368e+08
+# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
+# TYPE process_start_time_seconds gauge
+process_start_time_seconds 1.72552562346e+09
+# HELP process_virtual_memory_bytes Virtual memory size in bytes.
+# TYPE process_virtual_memory_bytes gauge
+process_virtual_memory_bytes 5.791121408e+09
+# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes.
+# TYPE process_virtual_memory_max_bytes gauge
+process_virtual_memory_max_bytes 1.8446744073709552e+19
+# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served.
+# TYPE promhttp_metric_handler_requests_in_flight gauge
+promhttp_metric_handler_requests_in_flight 1
+# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code.
+# TYPE promhttp_metric_handler_requests_total counter
+promhttp_metric_handler_requests_total{code=&quot;*&quot;} 140841
+# HELP request_duration_seconds A histogram of request latencies.
+# TYPE request_duration_seconds histogram
+request_duration_seconds_bucket{method=&quot;post&quot;,le=&quot;*&quot;} 1
+request_duration_seconds_sum{method=&quot;post&quot;} 5616.343836235997
+request_duration_seconds_count{method=&quot;post&quot;} 235915
+# HELP spacebox_crawler_blocks_by_status Total blocks for each status
+# TYPE spacebox_crawler_blocks_by_status gauge
+spacebox_crawler_blocks_by_status{status=&quot;error&quot;} 1.046201e+06
+spacebox_crawler_blocks_by_status{status=&quot;processed&quot;} 1.6964531e+07
+spacebox_crawler_blocks_by_status{status=&quot;processing&quot;} 0
+# HELP spacebox_crawler_last_processed_block_height Last processed block height
+# TYPE spacebox_crawler_last_processed_block_height gauge
+spacebox_crawler_last_processed_block_height 2.216532e+07
+# HELP spacebox_crawler_process_duration Duration of parsed blockchain objects
+# TYPE spacebox_crawler_process_duration histogram
+spacebox_crawler_process_duration_bucket{type=&quot;*&quot;,le=&quot;*&quot;} 117957
+spacebox_crawler_process_duration_sum{type=&quot;*&quot;} 0.48113770700000247
+spacebox_crawler_process_duration_count{type=&quot;*&quot;} 117957
+# HELP spacebox_crawler_start_height Start height for processing
+# TYPE spacebox_crawler_start_height gauge
+spacebox_crawler_start_height 1.8e+07
+# HELP spacebox_crawler_stop_height Stop height for processing
+# TYPE spacebox_crawler_stop_height gauge
+spacebox_crawler_stop_height 2.2047366e+07
+# HELP spacebox_crawler_total_error_messages Total error messages
+# TYPE spacebox_crawler_total_error_messages gauge
+spacebox_crawler_total_error_messages 0
+# HELP spacebox_crawler_total_error_txs Total error txs
+# TYPE spacebox_crawler_total_error_txs gauge
+spacebox_crawler_total_error_txs 6.719612e+06
+# HELP spacebox_crawler_total_workers Count of workers
+# TYPE spacebox_crawler_total_workers gauge
+spacebox_crawler_total_workers 5
+# HELP spacebox_crawler_version Crawler version
+# TYPE spacebox_crawler_version gauge
+spacebox_crawler_version{version=&quot;v2.0.0&quot;} 1
+# HELP spacebox_crawler_worker_recovery_mode Is worker recovery mode enabled
+# TYPE spacebox_crawler_worker_recovery_mode gauge
+spacebox_crawler_worker_recovery_mode 0
+</code></pre>
+              
+            </div>
+          </div><footer>
+    <div class="rst-footer-buttons" role="navigation" aria-label="Footer Navigation">
+        <a href="../setup/" class="btn btn-neutral float-left" title="Getting started"><span class="icon icon-circle-arrow-left"></span> Previous</a>
+    </div>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <!-- Copyright etc -->
+  </div>
+
+  Built with <a href="https://www.mkdocs.org/">MkDocs</a> using a <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+</footer>
+          
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+
+  <div class="rst-versions" role="note" aria-label="Versions">
+  <span class="rst-current-version" data-toggle="rst-current-version">
+    
+    
+      <span><a href="../setup/" style="color: #fcfcfc">&laquo; Previous</a></span>
+    
+    
+  </span>
+</div>
+    <script src="../js/jquery-3.6.0.min.js"></script>
+    <script>var base_url = "..";</script>
+    <script src="../js/theme_extra.js"></script>
+    <script src="../js/theme.js"></script>
+      <script src="../search/main.js"></script>
+    <script>
+        jQuery(function () {
+            SphinxRtdTheme.Navigation.enable(true);
+        });
+    </script>
+
+</body>
+</html>
diff --git a/search.html b/search.html
index 15670ff..9f45931 100644
--- a/search.html
+++ b/search.html
@@ -37,7 +37,11 @@
                 </li>
               </ul>
               <ul>
-                <li class="toctree-l1"><a class="reference internal" href="./getting_started/">Getting started</a>
+                <li class="toctree-l1"><a class="reference internal" href="./setup/">Getting started</a>
+                </li>
+              </ul>
+              <ul>
+                <li class="toctree-l1"><a class="reference internal" href="./monitoring/">Monitoring</a>
                 </li>
               </ul>
       </div>
diff --git a/search/search_index.json b/search/search_index.json
index fd2aafc..302751f 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"indexing":"full","lang":["en"],"min_search_length":3,"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"Welcome Spacebox documentation","title":"Home"},{"location":"#welcome-spacebox-documentation","text":"","title":"Welcome Spacebox documentation"},{"location":"getting_started/","text":"Getting started","title":"Getting started"},{"location":"getting_started/#getting-started","text":"","title":"Getting started"}]}
\ No newline at end of file
+{"config":{"indexing":"full","lang":["en"],"min_search_length":3,"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"Welcome Spacebox documentation","title":"Home"},{"location":"#welcome-spacebox-documentation","text":"","title":"Welcome Spacebox documentation"},{"location":"getting_started/","text":"Getting started","title":"Getting started"},{"location":"getting_started/#getting-started","text":"","title":"Getting started"},{"location":"monitoring/","text":"Spacebox monitoring Spacebox supports text-based metrics, compatible with Prometheus . Port is mapped to the localhost in default docker-compose configuration, and could be configured in .env : # Metrics settings METRICS_ENABLED=true # Metrics enabled SERVER_PORT=2112 # Metrics server port To make it appear in Prometheus, launch crawler add a new target to your prometheus.yml, and restart Prometheus: - job_name: 'spacebox-crawler' scrape_interval: 5s static_configs: - targets: ['localhost:2112'] An example of the Grafana dashboard can be found here . To make this board fully functional it's necessary to install ClickHouse plugin to Grafana. List of the supported metrics with some sample values: # HELP client_api_requests_total A counter for requests from the wrapped client. # TYPE client_api_requests_total counter client_api_requests_total{code=\"200\",method=\"post\"} 235915 # HELP client_in_flight_requests A gauge of in-flight requests for the wrapped client. # TYPE client_in_flight_requests gauge client_in_flight_requests 0 # HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles. # TYPE go_gc_duration_seconds summary go_gc_duration_seconds{quantile=\"*\"} 0.00013202 go_gc_duration_seconds_sum 569.665748093 go_gc_duration_seconds_count 320180 # HELP go_goroutines Number of goroutines that currently exist. # TYPE go_goroutines gauge go_goroutines 144 # HELP go_info Information about the Go environment. # TYPE go_info gauge go_info{version=\"go1.22.4\"} 1 # HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. # TYPE go_memstats_alloc_bytes gauge go_memstats_alloc_bytes 3.5313344e+08 # HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. # TYPE go_memstats_alloc_bytes_total counter go_memstats_alloc_bytes_total 1.49838467006912e+14 # HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. # TYPE go_memstats_buck_hash_sys_bytes gauge go_memstats_buck_hash_sys_bytes 7.119855e+06 # HELP go_memstats_frees_total Total number of frees. # TYPE go_memstats_frees_total counter go_memstats_frees_total 2.228701493759e+12 # HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. # TYPE go_memstats_gc_sys_bytes gauge go_memstats_gc_sys_bytes 4.607688e+07 # HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. # TYPE go_memstats_heap_alloc_bytes gauge go_memstats_heap_alloc_bytes 3.5313344e+08 # HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. # TYPE go_memstats_heap_idle_bytes gauge go_memstats_heap_idle_bytes 3.821551616e+09 # HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. # TYPE go_memstats_heap_inuse_bytes gauge go_memstats_heap_inuse_bytes 3.74980608e+08 # HELP go_memstats_heap_objects Number of allocated objects. # TYPE go_memstats_heap_objects gauge go_memstats_heap_objects 4.765757e+06 # HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. # TYPE go_memstats_heap_released_bytes gauge go_memstats_heap_released_bytes 3.750273024e+09 # HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. # TYPE go_memstats_heap_sys_bytes gauge go_memstats_heap_sys_bytes 4.196532224e+09 # HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. # TYPE go_memstats_last_gc_time_seconds gauge go_memstats_last_gc_time_seconds 1.726229858651255e+09 # HELP go_memstats_lookups_total Total number of pointer lookups. # TYPE go_memstats_lookups_total counter go_memstats_lookups_total 0 # HELP go_memstats_mallocs_total Total number of mallocs. # TYPE go_memstats_mallocs_total counter go_memstats_mallocs_total 2.228706259516e+12 # HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. # TYPE go_memstats_mcache_inuse_bytes gauge go_memstats_mcache_inuse_bytes 105600 # HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. # TYPE go_memstats_mcache_sys_bytes gauge go_memstats_mcache_sys_bytes 109200 # HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. # TYPE go_memstats_mspan_inuse_bytes gauge go_memstats_mspan_inuse_bytes 5.52096e+06 # HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. # TYPE go_memstats_mspan_sys_bytes gauge go_memstats_mspan_sys_bytes 6.190176e+07 # HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. # TYPE go_memstats_next_gc_bytes gauge go_memstats_next_gc_bytes 4.86296432e+08 # HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. # TYPE go_memstats_other_sys_bytes gauge go_memstats_other_sys_bytes 1.5349049e+07 # HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. # TYPE go_memstats_stack_inuse_bytes gauge go_memstats_stack_inuse_bytes 1.0354688e+07 # HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. # TYPE go_memstats_stack_sys_bytes gauge go_memstats_stack_sys_bytes 1.0354688e+07 # HELP go_memstats_sys_bytes Number of bytes obtained from system. # TYPE go_memstats_sys_bytes gauge go_memstats_sys_bytes 4.337443656e+09 # HELP go_threads Number of OS threads created. # TYPE go_threads gauge go_threads 100 # HELP grpc_client_handled_total Total number of RPCs completed by the client, regardless of success or failure. # TYPE grpc_client_handled_total counter grpc_client_handled_total{grpc_code=\"OK\",grpc_method=\"*\",grpc_service=\"*\",grpc_type=\"unary\"} 117957 # HELP grpc_client_handling_seconds Histogram of response latency (seconds) of the gRPC until it is finished by the application. # TYPE grpc_client_handling_seconds histogram grpc_client_handling_seconds_bucket{grpc_method=\"*\",grpc_service=\"*\",grpc_type=\"unary\",le=\"0.005\"} 0 grpc_client_handling_seconds_sum{grpc_method=\"*\",grpc_service=\"*\",grpc_type=\"unary\"} 1704.9106808000047 grpc_client_handling_seconds_count{grpc_method=\"*\",grpc_service=\"*\",grpc_type=\"unary\"} 117957 # HELP grpc_client_msg_received_total Total number of RPC stream messages received by the client. # TYPE grpc_client_msg_received_total counter grpc_client_msg_received_total{grpc_method=\"*\",grpc_service=\"*\",grpc_type=\"unary\"} 117957 # HELP grpc_client_msg_sent_total Total number of gRPC stream messages sent by the client. # TYPE grpc_client_msg_sent_total counter grpc_client_msg_sent_total{grpc_method=\"*\",grpc_service=\"*\",grpc_type=\"unary\"} 117957 # HELP grpc_client_started_total Total number of RPCs started on the client. # TYPE grpc_client_started_total counter grpc_client_started_total{grpc_method=\"*\",grpc_service=\"*\",grpc_type=\"unary\"} 117957 # HELP mongo_commands Histogram of MongoDB commands # TYPE mongo_commands histogram mongo_commands_bucket{command=\"*\",instance=\"*\",le=\"*\"} 6706 mongo_commands_sum{command=\"*\",instance=\"*\"} 46649.20434609119 mongo_commands_count{command=\"*\",instance=\"*\"} 13632 # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. # TYPE process_cpu_seconds_total counter process_cpu_seconds_total 3.77212593e+06 # HELP process_max_fds Maximum number of open file descriptors. # TYPE process_max_fds gauge process_max_fds 1.048576e+06 # HELP process_open_fds Number of open file descriptors. # TYPE process_open_fds gauge process_open_fds 36 # HELP process_resident_memory_bytes Resident memory size in bytes. # TYPE process_resident_memory_bytes gauge process_resident_memory_bytes 5.58522368e+08 # HELP process_start_time_seconds Start time of the process since unix epoch in seconds. # TYPE process_start_time_seconds gauge process_start_time_seconds 1.72552562346e+09 # HELP process_virtual_memory_bytes Virtual memory size in bytes. # TYPE process_virtual_memory_bytes gauge process_virtual_memory_bytes 5.791121408e+09 # HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. # TYPE process_virtual_memory_max_bytes gauge process_virtual_memory_max_bytes 1.8446744073709552e+19 # HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served. # TYPE promhttp_metric_handler_requests_in_flight gauge promhttp_metric_handler_requests_in_flight 1 # HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code. # TYPE promhttp_metric_handler_requests_total counter promhttp_metric_handler_requests_total{code=\"*\"} 140841 # HELP request_duration_seconds A histogram of request latencies. # TYPE request_duration_seconds histogram request_duration_seconds_bucket{method=\"post\",le=\"*\"} 1 request_duration_seconds_sum{method=\"post\"} 5616.343836235997 request_duration_seconds_count{method=\"post\"} 235915 # HELP spacebox_crawler_blocks_by_status Total blocks for each status # TYPE spacebox_crawler_blocks_by_status gauge spacebox_crawler_blocks_by_status{status=\"error\"} 1.046201e+06 spacebox_crawler_blocks_by_status{status=\"processed\"} 1.6964531e+07 spacebox_crawler_blocks_by_status{status=\"processing\"} 0 # HELP spacebox_crawler_last_processed_block_height Last processed block height # TYPE spacebox_crawler_last_processed_block_height gauge spacebox_crawler_last_processed_block_height 2.216532e+07 # HELP spacebox_crawler_process_duration Duration of parsed blockchain objects # TYPE spacebox_crawler_process_duration histogram spacebox_crawler_process_duration_bucket{type=\"*\",le=\"*\"} 117957 spacebox_crawler_process_duration_sum{type=\"*\"} 0.48113770700000247 spacebox_crawler_process_duration_count{type=\"*\"} 117957 # HELP spacebox_crawler_start_height Start height for processing # TYPE spacebox_crawler_start_height gauge spacebox_crawler_start_height 1.8e+07 # HELP spacebox_crawler_stop_height Stop height for processing # TYPE spacebox_crawler_stop_height gauge spacebox_crawler_stop_height 2.2047366e+07 # HELP spacebox_crawler_total_error_messages Total error messages # TYPE spacebox_crawler_total_error_messages gauge spacebox_crawler_total_error_messages 0 # HELP spacebox_crawler_total_error_txs Total error txs # TYPE spacebox_crawler_total_error_txs gauge spacebox_crawler_total_error_txs 6.719612e+06 # HELP spacebox_crawler_total_workers Count of workers # TYPE spacebox_crawler_total_workers gauge spacebox_crawler_total_workers 5 # HELP spacebox_crawler_version Crawler version # TYPE spacebox_crawler_version gauge spacebox_crawler_version{version=\"v2.0.0\"} 1 # HELP spacebox_crawler_worker_recovery_mode Is worker recovery mode enabled # TYPE spacebox_crawler_worker_recovery_mode gauge spacebox_crawler_worker_recovery_mode 0","title":"Monitoring"},{"location":"monitoring/#spacebox-monitoring","text":"Spacebox supports text-based metrics, compatible with Prometheus . Port is mapped to the localhost in default docker-compose configuration, and could be configured in .env : # Metrics settings METRICS_ENABLED=true # Metrics enabled SERVER_PORT=2112 # Metrics server port To make it appear in Prometheus, launch crawler add a new target to your prometheus.yml, and restart Prometheus: - job_name: 'spacebox-crawler' scrape_interval: 5s static_configs: - targets: ['localhost:2112'] An example of the Grafana dashboard can be found here . To make this board fully functional it's necessary to install ClickHouse plugin to Grafana. List of the supported metrics with some sample values: # HELP client_api_requests_total A counter for requests from the wrapped client. # TYPE client_api_requests_total counter client_api_requests_total{code=\"200\",method=\"post\"} 235915 # HELP client_in_flight_requests A gauge of in-flight requests for the wrapped client. # TYPE client_in_flight_requests gauge client_in_flight_requests 0 # HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles. # TYPE go_gc_duration_seconds summary go_gc_duration_seconds{quantile=\"*\"} 0.00013202 go_gc_duration_seconds_sum 569.665748093 go_gc_duration_seconds_count 320180 # HELP go_goroutines Number of goroutines that currently exist. # TYPE go_goroutines gauge go_goroutines 144 # HELP go_info Information about the Go environment. # TYPE go_info gauge go_info{version=\"go1.22.4\"} 1 # HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. # TYPE go_memstats_alloc_bytes gauge go_memstats_alloc_bytes 3.5313344e+08 # HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. # TYPE go_memstats_alloc_bytes_total counter go_memstats_alloc_bytes_total 1.49838467006912e+14 # HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. # TYPE go_memstats_buck_hash_sys_bytes gauge go_memstats_buck_hash_sys_bytes 7.119855e+06 # HELP go_memstats_frees_total Total number of frees. # TYPE go_memstats_frees_total counter go_memstats_frees_total 2.228701493759e+12 # HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. # TYPE go_memstats_gc_sys_bytes gauge go_memstats_gc_sys_bytes 4.607688e+07 # HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. # TYPE go_memstats_heap_alloc_bytes gauge go_memstats_heap_alloc_bytes 3.5313344e+08 # HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. # TYPE go_memstats_heap_idle_bytes gauge go_memstats_heap_idle_bytes 3.821551616e+09 # HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. # TYPE go_memstats_heap_inuse_bytes gauge go_memstats_heap_inuse_bytes 3.74980608e+08 # HELP go_memstats_heap_objects Number of allocated objects. # TYPE go_memstats_heap_objects gauge go_memstats_heap_objects 4.765757e+06 # HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. # TYPE go_memstats_heap_released_bytes gauge go_memstats_heap_released_bytes 3.750273024e+09 # HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. # TYPE go_memstats_heap_sys_bytes gauge go_memstats_heap_sys_bytes 4.196532224e+09 # HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. # TYPE go_memstats_last_gc_time_seconds gauge go_memstats_last_gc_time_seconds 1.726229858651255e+09 # HELP go_memstats_lookups_total Total number of pointer lookups. # TYPE go_memstats_lookups_total counter go_memstats_lookups_total 0 # HELP go_memstats_mallocs_total Total number of mallocs. # TYPE go_memstats_mallocs_total counter go_memstats_mallocs_total 2.228706259516e+12 # HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. # TYPE go_memstats_mcache_inuse_bytes gauge go_memstats_mcache_inuse_bytes 105600 # HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. # TYPE go_memstats_mcache_sys_bytes gauge go_memstats_mcache_sys_bytes 109200 # HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. # TYPE go_memstats_mspan_inuse_bytes gauge go_memstats_mspan_inuse_bytes 5.52096e+06 # HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. # TYPE go_memstats_mspan_sys_bytes gauge go_memstats_mspan_sys_bytes 6.190176e+07 # HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. # TYPE go_memstats_next_gc_bytes gauge go_memstats_next_gc_bytes 4.86296432e+08 # HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. # TYPE go_memstats_other_sys_bytes gauge go_memstats_other_sys_bytes 1.5349049e+07 # HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. # TYPE go_memstats_stack_inuse_bytes gauge go_memstats_stack_inuse_bytes 1.0354688e+07 # HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. # TYPE go_memstats_stack_sys_bytes gauge go_memstats_stack_sys_bytes 1.0354688e+07 # HELP go_memstats_sys_bytes Number of bytes obtained from system. # TYPE go_memstats_sys_bytes gauge go_memstats_sys_bytes 4.337443656e+09 # HELP go_threads Number of OS threads created. # TYPE go_threads gauge go_threads 100 # HELP grpc_client_handled_total Total number of RPCs completed by the client, regardless of success or failure. # TYPE grpc_client_handled_total counter grpc_client_handled_total{grpc_code=\"OK\",grpc_method=\"*\",grpc_service=\"*\",grpc_type=\"unary\"} 117957 # HELP grpc_client_handling_seconds Histogram of response latency (seconds) of the gRPC until it is finished by the application. # TYPE grpc_client_handling_seconds histogram grpc_client_handling_seconds_bucket{grpc_method=\"*\",grpc_service=\"*\",grpc_type=\"unary\",le=\"0.005\"} 0 grpc_client_handling_seconds_sum{grpc_method=\"*\",grpc_service=\"*\",grpc_type=\"unary\"} 1704.9106808000047 grpc_client_handling_seconds_count{grpc_method=\"*\",grpc_service=\"*\",grpc_type=\"unary\"} 117957 # HELP grpc_client_msg_received_total Total number of RPC stream messages received by the client. # TYPE grpc_client_msg_received_total counter grpc_client_msg_received_total{grpc_method=\"*\",grpc_service=\"*\",grpc_type=\"unary\"} 117957 # HELP grpc_client_msg_sent_total Total number of gRPC stream messages sent by the client. # TYPE grpc_client_msg_sent_total counter grpc_client_msg_sent_total{grpc_method=\"*\",grpc_service=\"*\",grpc_type=\"unary\"} 117957 # HELP grpc_client_started_total Total number of RPCs started on the client. # TYPE grpc_client_started_total counter grpc_client_started_total{grpc_method=\"*\",grpc_service=\"*\",grpc_type=\"unary\"} 117957 # HELP mongo_commands Histogram of MongoDB commands # TYPE mongo_commands histogram mongo_commands_bucket{command=\"*\",instance=\"*\",le=\"*\"} 6706 mongo_commands_sum{command=\"*\",instance=\"*\"} 46649.20434609119 mongo_commands_count{command=\"*\",instance=\"*\"} 13632 # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. # TYPE process_cpu_seconds_total counter process_cpu_seconds_total 3.77212593e+06 # HELP process_max_fds Maximum number of open file descriptors. # TYPE process_max_fds gauge process_max_fds 1.048576e+06 # HELP process_open_fds Number of open file descriptors. # TYPE process_open_fds gauge process_open_fds 36 # HELP process_resident_memory_bytes Resident memory size in bytes. # TYPE process_resident_memory_bytes gauge process_resident_memory_bytes 5.58522368e+08 # HELP process_start_time_seconds Start time of the process since unix epoch in seconds. # TYPE process_start_time_seconds gauge process_start_time_seconds 1.72552562346e+09 # HELP process_virtual_memory_bytes Virtual memory size in bytes. # TYPE process_virtual_memory_bytes gauge process_virtual_memory_bytes 5.791121408e+09 # HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. # TYPE process_virtual_memory_max_bytes gauge process_virtual_memory_max_bytes 1.8446744073709552e+19 # HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served. # TYPE promhttp_metric_handler_requests_in_flight gauge promhttp_metric_handler_requests_in_flight 1 # HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code. # TYPE promhttp_metric_handler_requests_total counter promhttp_metric_handler_requests_total{code=\"*\"} 140841 # HELP request_duration_seconds A histogram of request latencies. # TYPE request_duration_seconds histogram request_duration_seconds_bucket{method=\"post\",le=\"*\"} 1 request_duration_seconds_sum{method=\"post\"} 5616.343836235997 request_duration_seconds_count{method=\"post\"} 235915 # HELP spacebox_crawler_blocks_by_status Total blocks for each status # TYPE spacebox_crawler_blocks_by_status gauge spacebox_crawler_blocks_by_status{status=\"error\"} 1.046201e+06 spacebox_crawler_blocks_by_status{status=\"processed\"} 1.6964531e+07 spacebox_crawler_blocks_by_status{status=\"processing\"} 0 # HELP spacebox_crawler_last_processed_block_height Last processed block height # TYPE spacebox_crawler_last_processed_block_height gauge spacebox_crawler_last_processed_block_height 2.216532e+07 # HELP spacebox_crawler_process_duration Duration of parsed blockchain objects # TYPE spacebox_crawler_process_duration histogram spacebox_crawler_process_duration_bucket{type=\"*\",le=\"*\"} 117957 spacebox_crawler_process_duration_sum{type=\"*\"} 0.48113770700000247 spacebox_crawler_process_duration_count{type=\"*\"} 117957 # HELP spacebox_crawler_start_height Start height for processing # TYPE spacebox_crawler_start_height gauge spacebox_crawler_start_height 1.8e+07 # HELP spacebox_crawler_stop_height Stop height for processing # TYPE spacebox_crawler_stop_height gauge spacebox_crawler_stop_height 2.2047366e+07 # HELP spacebox_crawler_total_error_messages Total error messages # TYPE spacebox_crawler_total_error_messages gauge spacebox_crawler_total_error_messages 0 # HELP spacebox_crawler_total_error_txs Total error txs # TYPE spacebox_crawler_total_error_txs gauge spacebox_crawler_total_error_txs 6.719612e+06 # HELP spacebox_crawler_total_workers Count of workers # TYPE spacebox_crawler_total_workers gauge spacebox_crawler_total_workers 5 # HELP spacebox_crawler_version Crawler version # TYPE spacebox_crawler_version gauge spacebox_crawler_version{version=\"v2.0.0\"} 1 # HELP spacebox_crawler_worker_recovery_mode Is worker recovery mode enabled # TYPE spacebox_crawler_worker_recovery_mode gauge spacebox_crawler_worker_recovery_mode 0","title":"Spacebox monitoring"},{"location":"setup/","text":"Setup Requirements To operate the Spacebox you'll need an x86-64/arm machine with >8GB of RAM, and >4 CPU cores. For storage it is better to use faster SSD or NVME-like drives, size depends on the chain and TX volume(cosmoshub-4 index utilizes ~1 TB). Spacebox runs in Docker with help of Docker-compose . Also, you will need the RPC and GRPC endpoints of the chain, which you will index with the appropriate historical state. Quick start Get Clone the Spacebox repo, it contains everything to get started. console $ git clone https://github.com/bro-n-bro/spacebox.git Cloning into 'spacebox'... remote: Enumerating objects: 1111, done. remote: Counting objects: 100% (59/59), done. remote: Compressing objects: 100% (41/41), done. remote: Total 1111 (delta 17), reused 23 (delta 16), pack-reused 1052 Receiving objects: 100% (1111/1111), 2.82 MiB | 10.48 MiB/s, done. Resolving deltas: 100% (601/601), done. $ ls spacebox/ config docker-compose-local.yaml docker-compose.yaml docs go.mod LICENSE migrations mkdocs.yml README.md Set Fill .env with chain settings like chain prefix, node RPC & GRPC, and start \\ stop height: .env START_HEIGHT=5048767 # Start block height STOP_HEIGHT=0 # Stop block height, 0 for actual height WORKERS_COUNT=15 # Go workers to pull data in async mode SUBSCRIBE_NEW_BLOCKS=true # pull actual blocks # Chain settings CHAIN_PREFIX=cosmos # Prefix of indexing chain RPC_URL=http://0.0.0.0:26657 # RPC API GRPC_URL=0.0.0.0:9090 # GRPC API, no HTTP\\S prefix GRPC_SECURE_CONNECTION=false # GRPC secure connection Launch When .env is filled to start all containers run: console $ docker-compose up -d Creating network \"spacebox_default\" with the default driver Creating spacebox_hasura_1 ... done Creating spacebox_mongo-crawler_1 ... done Creating spacebox_ndc-clickhouse_1 ... done Creating spacebox_clickhouse_1 ... done Creating spacebox-zookeeper ... done Creating spacebox_postgres_1 ... done Creating spacebox_kafka_1 ... done Creating spacebox-kafka-ui ... done Creating spacebox_migration_1 ... done If everything is set correctly you will see the following in spacebox_crawler_1 : console $ docker logs spacebox_crawler_1 -f INF starting app cmp=app version=v1.2.0 INF module registered cmp=app name=raw version=v1.2.0 INF all modules registered cmp=app count=1 version=v1.2.0 INF starting cmp=app service=storage version=v1.2.0 INF started cmp=app service=storage version=v1.2.0 INF starting cmp=app service=grpc_client version=v1.2.0 INF started cmp=app service=grpc_client version=v1.2.0 INF starting cmp=app service=rpc_client version=v1.2.0 INF started cmp=app service=rpc_client version=v1.2.0 INF starting cmp=app service=broker version=v1.2.0 INF started cmp=app service=broker version=v1.2.0 INF starting cmp=app service=worker version=v1.2.0 INF started cmp=app service=worker version=v1.2.0 INF starting cmp=app service=server version=v1.2.0 INF listening for new block events cmp=worker version=v1.2.0 INF started cmp=app service=server version=v1.2.0 INF starting cmp=app service=health_checker version=v1.2.0 INF exit not needed cmp=worker version=v1.2.0 INF started cmp=app service=health_checker version=v1.2.0 INF start metrics scraper cmp=server version=v1.2.0 INF application started cmp=app version=v1.2.0 INF parse block cmp=worker height=1 version=v1.2.0 worker_number=1 INF parse block cmp=worker height=2 version=v1.2.0 worker_number=11 INF parse block cmp=worker height=4 version=v1.2.0 worker_number=19 INF parse block cmp=worker height=5 version=v1.2.0 worker_number=17 INF parse block cmp=worker height=3 version=v1.2.0 worker_number=6 To stop and remove all containers use the following: console docker-compose down Issues? If after docker-compose up you see the error: console ERROR: for crawler Container \"d0d8ccc463b8\" is unhealthy. ERROR: Encountered errors while bringing up the project. Then it is necessary to grant the ownership of the volumes folder to the docker user (uid 1001): console $ docker-compose down $ chown -R 1001:1001 volumes/ $ docker-compose up -d It usually requires around ~2 minutes to start everything. Check the log of each container with docker logs <container_name> to see what's going under the hood. It is possible to adjust some parameters on the fly, edit .env , and restart the appropriate container. Upgrading Spacebox main part spacebox-crawler follows semantic versioning: MAJOR.MINOR.PATCH + some releases are chain-specific and contain chain name in the tag, e.g. v2.0.0-neutron . As long as spacebox is considered as a beta software, every major version upgrade will require re-sync, check the release details for more info on that. We recommend using a specified version for the crawler in docker-compose.yml , instead of :latest that will ease troubleshooting and upgrades: docker-compose.yaml version: \"3.9\" services: crawler: image: bronbro/spacebox-crawler:v1.2.0 For upgrade with full resync stop everything remove the volmes folder and start over with the newer version of the crawler: console $ docker-compose down $ rm -rf volumes/ $ docker-compose up -d If an upgrade is happening alongside the chain upgrade, then it is possible to relaunch only the crawler: console $ docker stop spacebox_crawler-1 $ docker rm spacebox_crawler-1 Set new version in docker-compose.yaml services: crawler: - image: bronbro/spacebox-crawler:v1.2.0 + image: bronbro/spacebox-crawler:v1.3.0 Start crawler container console docker-compose up -d crawler In such case, indexation with the new version will start since the block you will upgrade the crawler's container. To avoid data inconsistency it might be a good idea to set the upgrade height in the .env as STOP_HEIGHT= and perform the container upgrade safely. Config options Spacebox is configured over environment variables, set in the .env file. [!TIP] To apply .env changes just restart the spacebox_crawler-1 container. However same does not apply to the rest of the containers(aka kafka or clickhouse), and they would need to be recreated. Crawler START_TIMEOUT Timeout to start application. Do not change without a strict purpose. STOP_TIMEOUT Timeout to stop application. Do not change without a strict purpose. LOG_LEVEL Supported values: info , debug , error . Debug level will produce more logs, including checks of the previously parsed blocks. Error log will show only errors if there's any. RECOVERY_MODE Default value: false Intended for debugging purposes mostly. In case of panic will output a full error log, without crashing the crawler. Significantly(!) lowers the perfomance. START_HEIGHT Default value: 0 Height to start crawling from. Ex: for cosmoshub-4 must be set to 5200792 as it was the first block in this chain. If set to 0 genesis will be parced automatically. STOP_HEIGHT Default value: 0 Height to stop indexing at. Might be useful when necessary to index only certain parts of the chain. When set to 0 crawler will pull the latest block from the RPC at the moment of launch and use it as stop height. WORKERS_COUNT A number of the asynchronous Go workers inside the crawler. Recommended value: <= CPU cores in your system. A huge number of workers may exhaust the RPC node, resulting in missed blocks. Usually, more workers don't mean faster crawling, best performing number is 30-40 workers (even on 88-core machine). SUBSCRIBE_NEW_BLOCKS Default value: true If set to false , the crawler will stop indexation on STOP_HEIGHT without an app crash. If set to true , the crawler will subscribe to the new coming blocks over the websocket and will parce them alongside to historical blocks. [Warning!] If SUBSCRIBE_NEW_BLOCKS=true spacebox_crawler_last_processed_block_height metric will display the highest indexed block, even though historical blocks might still be processed. PROCESS_ERROR_BLOCKS If set to true crawler will attempt to re-index the blocks that had errors during previous processing(based on info stored in Mongo). Re-indexation would be attempted every PROCESS_ERROR_BLOCKS_INTERVAL . PROCESS_GENESIS If set to true crawler will parce genesis to have genesis initial data in db. If START_HEIGHT set to 0 genesis would be parced as well. Text metrics METRICS_ENABLED Enable or disable crawler text metrics endpoint. Compatible with Prometheus. SERVER_PORT Specify the port for text metrics. Chain settings CHAIN_PREFIX Specify the account prefix for the chain you going to index. Typically found in the beginning of every address, e.g. for cosmos106yp7zw35wftheyyv9f9pe69t8rteumjxjql7m chain prefix is cosmos . RPC_URL & GRPC_URL GRPC_TIMEOUT Timeout for the GRPC requests. Default value: 15s. RPC_TIMEOUT Timeout for the RPC requests. Default value: 15s. Node RPC and GRPC endpoints. By default served on port 26657 and 9090. [TIP!] For better performance, we advise running the indexer as \"close\" to the chain node as possible, ideally on the same machine. If you running both crawler and chain node on the same host you need to add the following line to the crawler's docker-compose.yaml section: docker-compose.yaml ports: - '2112:2112' + extra_hosts: + - \"host.docker.internal:host-gateway\" And set RPC and GRPC addresses in the .env accordingly: RPC_URL=http://host.docker.internal:26657 GRPC_URL=host.docker.internal:9090 Also, depending on your firewall setup, it might be required to allow connections from the docker subnet to the chain endpoint. GRPC_SECURE_CONNECTION Set to true if your GRPC endpoint running with SSL encryption enabled. Leave as false if connecting directly to node port or within the same localhost. GRPC_MAX_RECEIVE_MESSAGE_SIZE_BYTES Defines the maximum size of a single message crawler will process. If you see something similar to ...failed to get block: rpc error: code = ResourceExhausted desc = grpc: received message larger than max... in the crawler log - increase this parameter and restart the container with spacebox_cralwer. Chains might have very heavy blocks, for ex. neutron's testnet pion-1 have some block_results of the size ~170mb. Broker settings BROKER_SERVER Address of Kafka broker. Do not change without a strict purpose. BROKER_ENABLED Enable\\disable message publishing messages to the broker. Do not change without a strict purpose. PARTITIONS_COUNT Count of the partitions in the broker. Do not change without a strict purpose. MAX_MESSAGE_MAX_BYTES Define the maximum size of the message in Kafka. Requires Kafka container recreation. Default value: 5242880 (5Mb). BATCH_PRODUCER Enable batch producer to process messages in Kafka by batches. Might increase performance, but considered an experimental feature. KAFKA_UI_PASSWORD Password for kafka-UI. Change for production deployment! Mongo settings MONGO_CRAWLER_URI Address of Mongo db. Crawler utilizes mongo-db as a check-list for parsed block status. MONGO_USER Mongo username used by the crawler. MONGO_PASSWORD Mongo user password used by the crawler. Change for production deployment! MAX_POOL_SIZE Maximum pool size in Mongo. Do not change without a strict purpose. MAX_CONNECTING Maximum Mongo connections. Do not change without a strict purpose. Clickhouse settings CLICKHOUSE_PASSWORD Password for clickhouse default user. Change for production deployment! Default user will require some privileges upgrade to allow it new users creation . Health checker HEALTHCHECK_ENABLED If set to true the crawler will periodically check the health of the node endpoints, and behave accordingly. HEALTHCHECK_FATAL_ON_CHECK If set to true the crawler will crush if the health check has failed. That will result in a container restart. HEALTHCHECK_MAX_LAST_BLOCK_LAG Interval for the new block to appear in the RPC, to count endpoint healthy. HEALTHCHECK_INTERVAL Interval to perform a health check. HEALTHCHECK_START_DELAY Delay for the first health check after the start.","title":"Getting started"},{"location":"setup/#setup","text":"","title":"Setup"},{"location":"setup/#requirements","text":"To operate the Spacebox you'll need an x86-64/arm machine with >8GB of RAM, and >4 CPU cores. For storage it is better to use faster SSD or NVME-like drives, size depends on the chain and TX volume(cosmoshub-4 index utilizes ~1 TB). Spacebox runs in Docker with help of Docker-compose . Also, you will need the RPC and GRPC endpoints of the chain, which you will index with the appropriate historical state.","title":"Requirements"},{"location":"setup/#quick-start","text":"","title":"Quick start"},{"location":"setup/#get","text":"Clone the Spacebox repo, it contains everything to get started. console $ git clone https://github.com/bro-n-bro/spacebox.git Cloning into 'spacebox'... remote: Enumerating objects: 1111, done. remote: Counting objects: 100% (59/59), done. remote: Compressing objects: 100% (41/41), done. remote: Total 1111 (delta 17), reused 23 (delta 16), pack-reused 1052 Receiving objects: 100% (1111/1111), 2.82 MiB | 10.48 MiB/s, done. Resolving deltas: 100% (601/601), done. $ ls spacebox/ config docker-compose-local.yaml docker-compose.yaml docs go.mod LICENSE migrations mkdocs.yml README.md","title":"Get"},{"location":"setup/#set","text":"Fill .env with chain settings like chain prefix, node RPC & GRPC, and start \\ stop height: .env START_HEIGHT=5048767 # Start block height STOP_HEIGHT=0 # Stop block height, 0 for actual height WORKERS_COUNT=15 # Go workers to pull data in async mode SUBSCRIBE_NEW_BLOCKS=true # pull actual blocks # Chain settings CHAIN_PREFIX=cosmos # Prefix of indexing chain RPC_URL=http://0.0.0.0:26657 # RPC API GRPC_URL=0.0.0.0:9090 # GRPC API, no HTTP\\S prefix GRPC_SECURE_CONNECTION=false # GRPC secure connection","title":"Set"},{"location":"setup/#launch","text":"When .env is filled to start all containers run: console $ docker-compose up -d Creating network \"spacebox_default\" with the default driver Creating spacebox_hasura_1 ... done Creating spacebox_mongo-crawler_1 ... done Creating spacebox_ndc-clickhouse_1 ... done Creating spacebox_clickhouse_1 ... done Creating spacebox-zookeeper ... done Creating spacebox_postgres_1 ... done Creating spacebox_kafka_1 ... done Creating spacebox-kafka-ui ... done Creating spacebox_migration_1 ... done If everything is set correctly you will see the following in spacebox_crawler_1 : console $ docker logs spacebox_crawler_1 -f INF starting app cmp=app version=v1.2.0 INF module registered cmp=app name=raw version=v1.2.0 INF all modules registered cmp=app count=1 version=v1.2.0 INF starting cmp=app service=storage version=v1.2.0 INF started cmp=app service=storage version=v1.2.0 INF starting cmp=app service=grpc_client version=v1.2.0 INF started cmp=app service=grpc_client version=v1.2.0 INF starting cmp=app service=rpc_client version=v1.2.0 INF started cmp=app service=rpc_client version=v1.2.0 INF starting cmp=app service=broker version=v1.2.0 INF started cmp=app service=broker version=v1.2.0 INF starting cmp=app service=worker version=v1.2.0 INF started cmp=app service=worker version=v1.2.0 INF starting cmp=app service=server version=v1.2.0 INF listening for new block events cmp=worker version=v1.2.0 INF started cmp=app service=server version=v1.2.0 INF starting cmp=app service=health_checker version=v1.2.0 INF exit not needed cmp=worker version=v1.2.0 INF started cmp=app service=health_checker version=v1.2.0 INF start metrics scraper cmp=server version=v1.2.0 INF application started cmp=app version=v1.2.0 INF parse block cmp=worker height=1 version=v1.2.0 worker_number=1 INF parse block cmp=worker height=2 version=v1.2.0 worker_number=11 INF parse block cmp=worker height=4 version=v1.2.0 worker_number=19 INF parse block cmp=worker height=5 version=v1.2.0 worker_number=17 INF parse block cmp=worker height=3 version=v1.2.0 worker_number=6 To stop and remove all containers use the following: console docker-compose down","title":"Launch"},{"location":"setup/#issues","text":"If after docker-compose up you see the error: console ERROR: for crawler Container \"d0d8ccc463b8\" is unhealthy. ERROR: Encountered errors while bringing up the project. Then it is necessary to grant the ownership of the volumes folder to the docker user (uid 1001): console $ docker-compose down $ chown -R 1001:1001 volumes/ $ docker-compose up -d It usually requires around ~2 minutes to start everything. Check the log of each container with docker logs <container_name> to see what's going under the hood. It is possible to adjust some parameters on the fly, edit .env , and restart the appropriate container.","title":"Issues?"},{"location":"setup/#upgrading","text":"Spacebox main part spacebox-crawler follows semantic versioning: MAJOR.MINOR.PATCH + some releases are chain-specific and contain chain name in the tag, e.g. v2.0.0-neutron . As long as spacebox is considered as a beta software, every major version upgrade will require re-sync, check the release details for more info on that. We recommend using a specified version for the crawler in docker-compose.yml , instead of :latest that will ease troubleshooting and upgrades: docker-compose.yaml version: \"3.9\" services: crawler: image: bronbro/spacebox-crawler:v1.2.0 For upgrade with full resync stop everything remove the volmes folder and start over with the newer version of the crawler: console $ docker-compose down $ rm -rf volumes/ $ docker-compose up -d If an upgrade is happening alongside the chain upgrade, then it is possible to relaunch only the crawler: console $ docker stop spacebox_crawler-1 $ docker rm spacebox_crawler-1 Set new version in docker-compose.yaml services: crawler: - image: bronbro/spacebox-crawler:v1.2.0 + image: bronbro/spacebox-crawler:v1.3.0 Start crawler container console docker-compose up -d crawler In such case, indexation with the new version will start since the block you will upgrade the crawler's container. To avoid data inconsistency it might be a good idea to set the upgrade height in the .env as STOP_HEIGHT= and perform the container upgrade safely.","title":"Upgrading"},{"location":"setup/#config-options","text":"Spacebox is configured over environment variables, set in the .env file. [!TIP] To apply .env changes just restart the spacebox_crawler-1 container. However same does not apply to the rest of the containers(aka kafka or clickhouse), and they would need to be recreated.","title":"Config options"},{"location":"setup/#crawler","text":"START_TIMEOUT Timeout to start application. Do not change without a strict purpose. STOP_TIMEOUT Timeout to stop application. Do not change without a strict purpose. LOG_LEVEL Supported values: info , debug , error . Debug level will produce more logs, including checks of the previously parsed blocks. Error log will show only errors if there's any. RECOVERY_MODE Default value: false Intended for debugging purposes mostly. In case of panic will output a full error log, without crashing the crawler. Significantly(!) lowers the perfomance. START_HEIGHT Default value: 0 Height to start crawling from. Ex: for cosmoshub-4 must be set to 5200792 as it was the first block in this chain. If set to 0 genesis will be parced automatically. STOP_HEIGHT Default value: 0 Height to stop indexing at. Might be useful when necessary to index only certain parts of the chain. When set to 0 crawler will pull the latest block from the RPC at the moment of launch and use it as stop height. WORKERS_COUNT A number of the asynchronous Go workers inside the crawler. Recommended value: <= CPU cores in your system. A huge number of workers may exhaust the RPC node, resulting in missed blocks. Usually, more workers don't mean faster crawling, best performing number is 30-40 workers (even on 88-core machine). SUBSCRIBE_NEW_BLOCKS Default value: true If set to false , the crawler will stop indexation on STOP_HEIGHT without an app crash. If set to true , the crawler will subscribe to the new coming blocks over the websocket and will parce them alongside to historical blocks. [Warning!] If SUBSCRIBE_NEW_BLOCKS=true spacebox_crawler_last_processed_block_height metric will display the highest indexed block, even though historical blocks might still be processed. PROCESS_ERROR_BLOCKS If set to true crawler will attempt to re-index the blocks that had errors during previous processing(based on info stored in Mongo). Re-indexation would be attempted every PROCESS_ERROR_BLOCKS_INTERVAL . PROCESS_GENESIS If set to true crawler will parce genesis to have genesis initial data in db. If START_HEIGHT set to 0 genesis would be parced as well.","title":"Crawler"},{"location":"setup/#text-metrics","text":"METRICS_ENABLED Enable or disable crawler text metrics endpoint. Compatible with Prometheus. SERVER_PORT Specify the port for text metrics.","title":"Text metrics"},{"location":"setup/#chain-settings","text":"CHAIN_PREFIX Specify the account prefix for the chain you going to index. Typically found in the beginning of every address, e.g. for cosmos106yp7zw35wftheyyv9f9pe69t8rteumjxjql7m chain prefix is cosmos . RPC_URL & GRPC_URL GRPC_TIMEOUT Timeout for the GRPC requests. Default value: 15s. RPC_TIMEOUT Timeout for the RPC requests. Default value: 15s. Node RPC and GRPC endpoints. By default served on port 26657 and 9090. [TIP!] For better performance, we advise running the indexer as \"close\" to the chain node as possible, ideally on the same machine. If you running both crawler and chain node on the same host you need to add the following line to the crawler's docker-compose.yaml section: docker-compose.yaml ports: - '2112:2112' + extra_hosts: + - \"host.docker.internal:host-gateway\" And set RPC and GRPC addresses in the .env accordingly: RPC_URL=http://host.docker.internal:26657 GRPC_URL=host.docker.internal:9090 Also, depending on your firewall setup, it might be required to allow connections from the docker subnet to the chain endpoint. GRPC_SECURE_CONNECTION Set to true if your GRPC endpoint running with SSL encryption enabled. Leave as false if connecting directly to node port or within the same localhost. GRPC_MAX_RECEIVE_MESSAGE_SIZE_BYTES Defines the maximum size of a single message crawler will process. If you see something similar to ...failed to get block: rpc error: code = ResourceExhausted desc = grpc: received message larger than max... in the crawler log - increase this parameter and restart the container with spacebox_cralwer. Chains might have very heavy blocks, for ex. neutron's testnet pion-1 have some block_results of the size ~170mb.","title":"Chain settings"},{"location":"setup/#broker-settings","text":"BROKER_SERVER Address of Kafka broker. Do not change without a strict purpose. BROKER_ENABLED Enable\\disable message publishing messages to the broker. Do not change without a strict purpose. PARTITIONS_COUNT Count of the partitions in the broker. Do not change without a strict purpose. MAX_MESSAGE_MAX_BYTES Define the maximum size of the message in Kafka. Requires Kafka container recreation. Default value: 5242880 (5Mb). BATCH_PRODUCER Enable batch producer to process messages in Kafka by batches. Might increase performance, but considered an experimental feature. KAFKA_UI_PASSWORD Password for kafka-UI. Change for production deployment!","title":"Broker settings"},{"location":"setup/#mongo-settings","text":"MONGO_CRAWLER_URI Address of Mongo db. Crawler utilizes mongo-db as a check-list for parsed block status. MONGO_USER Mongo username used by the crawler. MONGO_PASSWORD Mongo user password used by the crawler. Change for production deployment! MAX_POOL_SIZE Maximum pool size in Mongo. Do not change without a strict purpose. MAX_CONNECTING Maximum Mongo connections. Do not change without a strict purpose.","title":"Mongo settings"},{"location":"setup/#clickhouse-settings","text":"CLICKHOUSE_PASSWORD Password for clickhouse default user. Change for production deployment! Default user will require some privileges upgrade to allow it new users creation .","title":"Clickhouse settings"},{"location":"setup/#health-checker","text":"HEALTHCHECK_ENABLED If set to true the crawler will periodically check the health of the node endpoints, and behave accordingly. HEALTHCHECK_FATAL_ON_CHECK If set to true the crawler will crush if the health check has failed. That will result in a container restart. HEALTHCHECK_MAX_LAST_BLOCK_LAG Interval for the new block to appear in the RPC, to count endpoint healthy. HEALTHCHECK_INTERVAL Interval to perform a health check. HEALTHCHECK_START_DELAY Delay for the first health check after the start.","title":"Health checker"}]}
\ No newline at end of file
diff --git a/setup/index.html b/setup/index.html
new file mode 100644
index 0000000..5bf8379
--- /dev/null
+++ b/setup/index.html
@@ -0,0 +1,410 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+    <meta charset="utf-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+      <link rel="shortcut icon" href="../img/favicon.ico" />
+    <title>Getting started - Spacebox docs</title>
+    <link rel="stylesheet" href="../css/theme.css" />
+    <link rel="stylesheet" href="../css/theme_extra.css" />
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/github.min.css" />
+    
+      <script>
+        // Current page data
+        var mkdocs_page_name = "Getting started";
+        var mkdocs_page_input_path = "setup.md";
+        var mkdocs_page_url = null;
+      </script>
+    
+    <!--[if lt IE 9]>
+      <script src="../js/html5shiv.min.js"></script>
+    <![endif]-->
+      <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+      <script>hljs.highlightAll();</script> 
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
+    <div class="wy-side-scroll">
+      <div class="wy-side-nav-search">
+          <a href=".." class="icon icon-home"> Spacebox docs
+        </a><div role="search">
+  <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get">
+      <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" title="Type search term here" />
+  </form>
+</div>
+      </div>
+
+      <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <ul>
+                <li class="toctree-l1"><a class="reference internal" href="..">Home</a>
+                </li>
+              </ul>
+              <ul class="current">
+                <li class="toctree-l1 current"><a class="reference internal current" href="#">Getting started</a>
+    <ul class="current">
+    <li class="toctree-l2"><a class="reference internal" href="#requirements">Requirements</a>
+    </li>
+    <li class="toctree-l2"><a class="reference internal" href="#quick-start">Quick start</a>
+        <ul>
+    <li class="toctree-l3"><a class="reference internal" href="#get">Get</a>
+    </li>
+    <li class="toctree-l3"><a class="reference internal" href="#set">Set</a>
+    </li>
+    <li class="toctree-l3"><a class="reference internal" href="#launch">Launch</a>
+    </li>
+    <li class="toctree-l3"><a class="reference internal" href="#issues">Issues?</a>
+    </li>
+        </ul>
+    </li>
+    <li class="toctree-l2"><a class="reference internal" href="#upgrading">Upgrading</a>
+    </li>
+    <li class="toctree-l2"><a class="reference internal" href="#config-options">Config options</a>
+        <ul>
+    <li class="toctree-l3"><a class="reference internal" href="#crawler">Crawler</a>
+    </li>
+    <li class="toctree-l3"><a class="reference internal" href="#text-metrics">Text metrics</a>
+    </li>
+    <li class="toctree-l3"><a class="reference internal" href="#chain-settings">Chain settings</a>
+    </li>
+    <li class="toctree-l3"><a class="reference internal" href="#broker-settings">Broker settings</a>
+    </li>
+    <li class="toctree-l3"><a class="reference internal" href="#mongo-settings">Mongo settings</a>
+    </li>
+    <li class="toctree-l3"><a class="reference internal" href="#clickhouse-settings">Clickhouse settings</a>
+    </li>
+    <li class="toctree-l3"><a class="reference internal" href="#health-checker">Health checker</a>
+    </li>
+        </ul>
+    </li>
+    </ul>
+                </li>
+              </ul>
+              <ul>
+                <li class="toctree-l1"><a class="reference internal" href="../monitoring/">Monitoring</a>
+                </li>
+              </ul>
+      </div>
+    </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+      <nav class="wy-nav-top" role="navigation" aria-label="Mobile navigation menu">
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="..">Spacebox docs</a>
+        
+      </nav>
+      <div class="wy-nav-content">
+        <div class="rst-content"><div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href=".." class="icon icon-home" aria-label="Docs"></a></li>
+      <li class="breadcrumb-item active">Getting started</li>
+    <li class="wy-breadcrumbs-aside">
+    </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+            <div class="section" itemprop="articleBody">
+              
+                <h1 id="setup">Setup</h1>
+<h2 id="requirements">Requirements</h2>
+<p>To operate the Spacebox you'll need an x86-64/arm machine with &gt;8GB of RAM, and &gt;4 CPU cores. For storage it is better to use faster SSD or NVME-like drives, size depends on the chain and TX volume(cosmoshub-4 index utilizes ~1 TB). Spacebox runs in <a href="https://docs.docker.com/engine/install/">Docker</a> with help of<a href="https://docs.docker.com/compose/install/">Docker-compose</a>. Also, you will need the RPC and GRPC endpoints of the chain, which you will index with the appropriate historical state.</p>
+<h2 id="quick-start">Quick start</h2>
+<h3 id="get">Get</h3>
+<p>Clone the <a href="https://github.com/bro-n-bro/spacebox.git">Spacebox</a> repo, it contains everything to get started.</p>
+<p><code>console</code></p>
+<pre><code class="language-console">$ git clone https://github.com/bro-n-bro/spacebox.git
+Cloning into 'spacebox'...
+remote: Enumerating objects: 1111, done.
+remote: Counting objects: 100% (59/59), done.
+remote: Compressing objects: 100% (41/41), done.
+remote: Total 1111 (delta 17), reused 23 (delta 16), pack-reused 1052
+Receiving objects: 100% (1111/1111), 2.82 MiB | 10.48 MiB/s, done.
+Resolving deltas: 100% (601/601), done.
+$ ls spacebox/
+config  docker-compose-local.yaml  docker-compose.yaml  docs  go.mod  LICENSE  migrations  mkdocs.yml  README.md
+</code></pre>
+<h3 id="set">Set</h3>
+<p>Fill .env with chain settings like chain prefix, node RPC &amp; GRPC, and start \ stop height:</p>
+<p><code>.env</code></p>
+<pre><code class="language-bash">START_HEIGHT=5048767  # Start block height
+STOP_HEIGHT=0 # Stop block height, 0 for actual height
+WORKERS_COUNT=15 # Go workers to pull data in async mode
+SUBSCRIBE_NEW_BLOCKS=true # pull actual blocks
+
+
+# Chain settings
+CHAIN_PREFIX=cosmos # Prefix of indexing chain
+RPC_URL=http://0.0.0.0:26657 # RPC API
+GRPC_URL=0.0.0.0:9090 # GRPC API, no HTTP\S prefix
+GRPC_SECURE_CONNECTION=false # GRPC secure connection
+</code></pre>
+<h3 id="launch">Launch</h3>
+<p>When <code>.env</code> is filled to start all containers run:</p>
+<p><code>console</code></p>
+<pre><code class="language-bash">$ docker-compose up -d
+Creating network &quot;spacebox_default&quot; with the default driver
+Creating spacebox_hasura_1         ... done
+Creating spacebox_mongo-crawler_1  ... done
+Creating spacebox_ndc-clickhouse_1 ... done
+Creating spacebox_clickhouse_1     ... done
+Creating spacebox-zookeeper        ... done
+Creating spacebox_postgres_1       ... done
+Creating spacebox_kafka_1          ... done
+Creating spacebox-kafka-ui         ... done
+Creating spacebox_migration_1      ... done
+</code></pre>
+<p>If everything is set correctly you will see the following in <code>spacebox_crawler_1</code>:</p>
+<p><code>console</code></p>
+<pre><code class="language-console">$ docker logs spacebox_crawler_1 -f 
+INF starting app cmp=app version=v1.2.0
+INF module registered cmp=app name=raw version=v1.2.0
+INF all modules registered cmp=app count=1 version=v1.2.0
+INF starting cmp=app service=storage version=v1.2.0
+INF started cmp=app service=storage version=v1.2.0
+INF starting cmp=app service=grpc_client version=v1.2.0
+INF started cmp=app service=grpc_client version=v1.2.0
+INF starting cmp=app service=rpc_client version=v1.2.0
+INF started cmp=app service=rpc_client version=v1.2.0
+INF starting cmp=app service=broker version=v1.2.0
+INF started cmp=app service=broker version=v1.2.0
+INF starting cmp=app service=worker version=v1.2.0
+INF started cmp=app service=worker version=v1.2.0
+INF starting cmp=app service=server version=v1.2.0
+INF listening for new block events cmp=worker version=v1.2.0
+INF started cmp=app service=server version=v1.2.0
+INF starting cmp=app service=health_checker version=v1.2.0
+INF exit not needed cmp=worker version=v1.2.0
+INF started cmp=app service=health_checker version=v1.2.0
+INF start metrics scraper cmp=server version=v1.2.0
+INF application started cmp=app version=v1.2.0
+INF parse block cmp=worker height=1 version=v1.2.0 worker_number=1
+INF parse block cmp=worker height=2 version=v1.2.0 worker_number=11
+INF parse block cmp=worker height=4 version=v1.2.0 worker_number=19
+INF parse block cmp=worker height=5 version=v1.2.0 worker_number=17
+INF parse block cmp=worker height=3 version=v1.2.0 worker_number=6
+</code></pre>
+<p>To stop and remove all containers use the following:</p>
+<p><code>console</code></p>
+<pre><code class="language-bash">docker-compose down
+</code></pre>
+<h3 id="issues">Issues?</h3>
+<p>If after <code>docker-compose up</code> you see the error:</p>
+<p><code>console</code></p>
+<pre><code class="language-console">ERROR: for crawler  Container &quot;d0d8ccc463b8&quot; is unhealthy.
+ERROR: Encountered errors while bringing up the project.
+</code></pre>
+<p>Then it is necessary to grant the ownership of the <code>volumes</code> folder to the docker user (uid 1001):</p>
+<p><code>console</code></p>
+<pre><code class="language-console">$ docker-compose down
+$ chown -R 1001:1001 volumes/
+$ docker-compose up -d
+</code></pre>
+<p>It usually requires around ~2 minutes to start everything. Check the log of each container with <code>docker logs &lt;container_name&gt;</code> to see what's going under the hood. It is possible to adjust some parameters on the fly, edit <code>.env</code>, and restart the appropriate container.</p>
+<h2 id="upgrading">Upgrading</h2>
+<p>Spacebox main part <a href="https://github.com/bro-n-bro/spacebox-crawler/releases">spacebox-crawler</a> follows <a href="https://semver.org/">semantic versioning:</a> <code>MAJOR.MINOR.PATCH</code> + some releases are chain-specific and contain chain name in the tag, e.g. <code>v2.0.0-neutron</code>.</p>
+<p>As long as spacebox is considered as a beta software, every major version upgrade will require re-sync, check the release details for more info on that.</p>
+<p>We recommend using a specified version for the crawler in <code>docker-compose.yml</code>, instead of <code>:latest</code> that will ease troubleshooting and upgrades:</p>
+<p><code>docker-compose.yaml</code></p>
+<pre><code class="language-console">version: &quot;3.9&quot;
+
+services:
+ crawler:
+ image: bronbro/spacebox-crawler:v1.2.0
+</code></pre>
+<p>For upgrade with full resync stop everything remove the volmes folder and start over with the newer version of the crawler:</p>
+<p><code>console</code></p>
+<pre><code class="language-console">$ docker-compose down
+$ rm -rf  volumes/
+$ docker-compose up -d
+</code></pre>
+<p>If an upgrade is happening alongside the chain upgrade, then it is possible to relaunch only the crawler:</p>
+<p><code>console</code></p>
+<pre><code class="language-bash">$ docker stop spacebox_crawler-1
+$ docker rm spacebox_crawler-1
+</code></pre>
+<p>Set new version in <code>docker-compose.yaml</code></p>
+<pre><code class="language-diff">services:
+ crawler:
+-    image: bronbro/spacebox-crawler:v1.2.0
++    image: bronbro/spacebox-crawler:v1.3.0
+</code></pre>
+<p>Start crawler container</p>
+<p><code>console</code></p>
+<pre><code class="language-bash">docker-compose up -d crawler
+</code></pre>
+<p>In such case, indexation with the new version will start since the block you will upgrade the crawler's container. To avoid data inconsistency it might be a good idea to set the upgrade height in the .env as <code>STOP_HEIGHT=</code> and perform the container upgrade safely.</p>
+<h2 id="config-options">Config options</h2>
+<p>Spacebox is configured over environment variables, set in the <code>.env</code> file.</p>
+<blockquote>
+<p>[!TIP]
+To apply .env changes just restart the <code>spacebox_crawler-1</code> container. However same does not apply to the rest of the containers(aka kafka or clickhouse), and they would need to be <strong>recreated.</strong></p>
+</blockquote>
+<h3 id="crawler">Crawler</h3>
+<hr />
+<p><code>START_TIMEOUT</code></p>
+<p>Timeout to start application. Do not change without a strict purpose.</p>
+<hr />
+<p><code>STOP_TIMEOUT</code></p>
+<p>Timeout to stop application. Do not change without a strict purpose.</p>
+<hr />
+<p><code>LOG_LEVEL</code></p>
+<p>Supported values: <code>info</code>, <code>debug</code>,  <code>error</code>.</p>
+<p>Debug level will produce more logs, including checks of the previously parsed blocks. Error log will show only errors if there's any.</p>
+<hr />
+<p><code>RECOVERY_MODE</code></p>
+<p>Default value: false</p>
+<p>Intended for debugging purposes mostly. In case of <em><code>panic</code></em> will output a full error log, without crashing the crawler. Significantly(!) lowers the perfomance.</p>
+<hr />
+<p><code>START_HEIGHT</code></p>
+<p>Default value: 0</p>
+<p>Height to start crawling from. Ex: for <code>cosmoshub-4</code> must be set to 5200792 as it was the first block in this chain.
+If set to 0 genesis will be parced automatically.</p>
+<hr />
+<p><code>STOP_HEIGHT</code></p>
+<p>Default value: 0</p>
+<p>Height to stop indexing at. Might be useful when necessary to index only certain parts of the chain. When set to 0 crawler will pull the latest block from the RPC at the moment of launch and use it as stop height.</p>
+<hr />
+<p><code>WORKERS_COUNT</code></p>
+<p>A number of the asynchronous Go workers inside the crawler. Recommended value: <strong>&lt;=</strong> <em>CPU cores</em> in your system. A huge number of workers may exhaust the RPC node, resulting in missed blocks. Usually, more workers don't mean faster crawling, best performing number is 30-40 workers (even on 88-core machine).</p>
+<hr />
+<p><code>SUBSCRIBE_NEW_BLOCKS</code></p>
+<p>Default value: true</p>
+<p>If set to <strong>false</strong>, the crawler will stop indexation on <code>STOP_HEIGHT</code> without an app crash. If set to <strong>true</strong>, the crawler will subscribe to the new coming blocks over the websocket and will parce them alongside to historical blocks.</p>
+<blockquote>
+<p>[Warning!] If SUBSCRIBE_NEW_BLOCKS=true <code>spacebox_crawler_last_processed_block_height</code> metric will display the highest indexed block, even though historical blocks might still be processed.</p>
+</blockquote>
+<hr />
+<p><code>PROCESS_ERROR_BLOCKS</code></p>
+<p>If set to <strong>true</strong> crawler will attempt to re-index the blocks that had errors during previous processing(based on info stored in Mongo). Re-indexation would be attempted every <code>PROCESS_ERROR_BLOCKS_INTERVAL</code>.</p>
+<hr />
+<p><code>PROCESS_GENESIS</code></p>
+<p>If set to <strong>true</strong> crawler will parce genesis to have genesis initial data in db. If <code>START_HEIGHT</code> set to 0 genesis would be parced as well.</p>
+<h3 id="text-metrics">Text metrics</h3>
+<p><code>METRICS_ENABLED</code></p>
+<p>Enable or disable crawler text metrics endpoint. Compatible with Prometheus.</p>
+<p><code>SERVER_PORT</code></p>
+<p>Specify the port for text metrics.</p>
+<hr />
+<h3 id="chain-settings">Chain settings</h3>
+<p><code>CHAIN_PREFIX</code></p>
+<p>Specify the account prefix for the chain you going to index. Typically found in the beginning of every address, e.g. for <code>cosmos106yp7zw35wftheyyv9f9pe69t8rteumjxjql7m</code> chain prefix is <code>cosmos</code>.</p>
+<p><code>RPC_URL</code> &amp; <code>GRPC_URL</code></p>
+<p><code>GRPC_TIMEOUT</code></p>
+<p>Timeout for the GRPC requests. Default value: 15s.</p>
+<p><code>RPC_TIMEOUT</code></p>
+<p>Timeout for the RPC requests. Default value: 15s.</p>
+<p>Node RPC and GRPC endpoints. By default served on port 26657 and 9090.</p>
+<blockquote>
+<p>[TIP!] For better performance, we advise running the indexer as "close" to the chain node as possible, ideally on the same machine.</p>
+</blockquote>
+<p>If you running both crawler and chain node on the same host you need to add the following line to the crawler's <code>docker-compose.yaml</code> section:</p>
+<p><code>docker-compose.yaml</code></p>
+<pre><code class="language-diff"> ports:
+ - '2112:2112'
++    extra_hosts:
++     - &quot;host.docker.internal:host-gateway&quot;
+</code></pre>
+<p>And set RPC and GRPC addresses in the <code>.env</code> accordingly:</p>
+<pre><code class="language-bash">RPC_URL=http://host.docker.internal:26657
+GRPC_URL=host.docker.internal:9090 
+</code></pre>
+<p>Also, depending on your firewall setup, it might be required to allow connections from the docker subnet to the chain endpoint.</p>
+<p><code>GRPC_SECURE_CONNECTION</code></p>
+<p>Set to <strong>true</strong> if your GRPC endpoint running with SSL encryption enabled. Leave as <strong>false</strong> if connecting directly to node port or within the same localhost.</p>
+<p><code>GRPC_MAX_RECEIVE_MESSAGE_SIZE_BYTES</code></p>
+<p>Defines the maximum size of a single message crawler will process. If you see something similar to <code>...failed to get block: rpc error: code = ResourceExhausted desc = grpc: received message larger than max...</code> in the crawler log - increase this parameter and restart the container with spacebox_cralwer. Chains might have very heavy blocks, for ex. neutron's testnet pion-1 have some block_results of the size ~170mb.</p>
+<hr />
+<h3 id="broker-settings">Broker settings</h3>
+<p><code>BROKER_SERVER</code></p>
+<p>Address of Kafka broker. Do not change without a strict purpose.</p>
+<p><code>BROKER_ENABLED</code></p>
+<p>Enable\disable message publishing messages to the broker. Do not change without a strict purpose.</p>
+<p><code>PARTITIONS_COUNT</code></p>
+<p>Count of the partitions in the broker. Do not change without a strict purpose.</p>
+<p><code>MAX_MESSAGE_MAX_BYTES</code></p>
+<p>Define the maximum size of the message in Kafka. Requires Kafka container recreation. Default value: 5242880 (5Mb).</p>
+<p><code>BATCH_PRODUCER</code></p>
+<p>Enable batch producer to process messages in Kafka by batches. Might increase performance, but considered an experimental feature.</p>
+<p><code>KAFKA_UI_PASSWORD</code></p>
+<p>Password for kafka-UI. Change for production deployment!</p>
+<hr />
+<h3 id="mongo-settings">Mongo settings</h3>
+<p><code>MONGO_CRAWLER_URI</code></p>
+<p>Address of Mongo db. Crawler utilizes mongo-db as a check-list for parsed block status.</p>
+<p><code>MONGO_USER</code></p>
+<p>Mongo username used by the crawler.</p>
+<p><code>MONGO_PASSWORD</code></p>
+<p>Mongo user password used by the crawler. Change for production deployment!</p>
+<p><code>MAX_POOL_SIZE</code></p>
+<p>Maximum pool size in Mongo. Do not change without a strict purpose.</p>
+<p><code>MAX_CONNECTING</code></p>
+<p>Maximum Mongo connections. Do not change without a strict purpose.</p>
+<hr />
+<h3 id="clickhouse-settings">Clickhouse settings</h3>
+<p><code>CLICKHOUSE_PASSWORD</code></p>
+<p>Password for clickhouse <code>default</code> user.  Change for production deployment! Default user will require some privileges upgrade to allow it <a href="https://clickhouse.com/docs/en/operations/access-rights#enabling-sql-user-mode">new users creation</a>.</p>
+<hr />
+<h3 id="health-checker">Health checker</h3>
+<p><code>HEALTHCHECK_ENABLED</code></p>
+<p>If set to <strong>true</strong> the crawler will periodically check the health of the node endpoints, and behave accordingly.</p>
+<p><code>HEALTHCHECK_FATAL_ON_CHECK</code></p>
+<p>If set to <strong>true</strong> the crawler will crush if the health check has failed. That will result in a container restart.</p>
+<p><code>HEALTHCHECK_MAX_LAST_BLOCK_LAG</code></p>
+<p>Interval for the new block to appear in the RPC, to count endpoint healthy.</p>
+<p><code>HEALTHCHECK_INTERVAL</code></p>
+<p>Interval to perform a health check.</p>
+<p><code>HEALTHCHECK_START_DELAY</code></p>
+<p>Delay for the first health check after the start.</p>
+              
+            </div>
+          </div><footer>
+    <div class="rst-footer-buttons" role="navigation" aria-label="Footer Navigation">
+        <a href=".." class="btn btn-neutral float-left" title="Home"><span class="icon icon-circle-arrow-left"></span> Previous</a>
+        <a href="../monitoring/" class="btn btn-neutral float-right" title="Monitoring">Next <span class="icon icon-circle-arrow-right"></span></a>
+    </div>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <!-- Copyright etc -->
+  </div>
+
+  Built with <a href="https://www.mkdocs.org/">MkDocs</a> using a <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+</footer>
+          
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+
+  <div class="rst-versions" role="note" aria-label="Versions">
+  <span class="rst-current-version" data-toggle="rst-current-version">
+    
+    
+      <span><a href=".." style="color: #fcfcfc">&laquo; Previous</a></span>
+    
+    
+      <span><a href="../monitoring/" style="color: #fcfcfc">Next &raquo;</a></span>
+    
+  </span>
+</div>
+    <script src="../js/jquery-3.6.0.min.js"></script>
+    <script>var base_url = "..";</script>
+    <script src="../js/theme_extra.js"></script>
+    <script src="../js/theme.js"></script>
+      <script src="../search/main.js"></script>
+    <script>
+        jQuery(function () {
+            SphinxRtdTheme.Navigation.enable(true);
+        });
+    </script>
+
+</body>
+</html>
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index 8ea9fb7ced55a954c6bed8acfbca2fd3c6bdfeeb..ccb0c40b53da4f9b251f4bbb1da4da6c7891c180 100644
GIT binary patch
delta 13
Ucmb=gXP58h;AlAZVj_D503Znj{r~^~

delta 13
Ucmb=gXP58h;AlwSIFY>q03A{UXaE2J

diff --git a/spacebox_grafana_sample.json b/spacebox_grafana_sample.json
new file mode 100644
index 0000000..9629276
--- /dev/null
+++ b/spacebox_grafana_sample.json
@@ -0,0 +1,2966 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "datasource",
+            "uid": "grafana"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "target": {
+            "limit": 100,
+            "matchAny": false,
+            "tags": [],
+            "type": "dashboard"
+          },
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "id": 151,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+      {
+        "datasource": {
+          "type": "grafana-clickhouse-datasource",
+          "uid": "${clickhouse}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 5,
+          "x": 0,
+          "y": 0
+        },
+        "id": 20,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "builderOptions": {
+              "database": "",
+              "fields": [],
+              "limit": 100,
+              "metrics": [
+                {
+                  "aggregation": "count",
+                  "field": ""
+                }
+              ],
+              "mode": "aggregate",
+              "table": "block final"
+            },
+            "datasource": {
+              "type": "grafana-clickhouse-datasource",
+              "uid": "${clickhouse}"
+            },
+            "format": 1,
+            "hide": true,
+            "meta": {
+              "builderOptions": {
+                "database": "",
+                "fields": [],
+                "limit": 100,
+                "metrics": [
+                  {
+                    "aggregation": "count",
+                    "field": ""
+                  }
+                ],
+                "mode": "aggregate",
+                "table": "block final"
+              }
+            },
+            "queryType": "sql",
+            "rawSql": "SELECT count() FROM block final LIMIT 100",
+            "refId": "A"
+          },
+          {
+            "datasource": {
+              "type": "grafana-clickhouse-datasource",
+              "uid": "${clickhouse}"
+            },
+            "format": 1,
+            "hide": false,
+            "meta": {
+              "builderOptions": {
+                "fields": [],
+                "limit": 100,
+                "mode": "list"
+              }
+            },
+            "queryType": "sql",
+            "rawSql": "SELECT count() FROM raw_block final",
+            "refId": "B"
+          }
+        ],
+        "title": "Total blocks in Clickhouse",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "grafana-clickhouse-datasource",
+          "uid": "${clickhouse}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 5,
+          "x": 5,
+          "y": 0
+        },
+        "id": 15,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "none",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "builderOptions": {
+              "fields": [
+                "height"
+              ],
+              "filters": [],
+              "limit": 1,
+              "mode": "list",
+              "orderBy": [
+                {
+                  "dir": "DESC",
+                  "name": "height"
+                }
+              ],
+              "table": "block"
+            },
+            "datasource": {
+              "type": "grafana-clickhouse-datasource",
+              "uid": "${clickhouse}"
+            },
+            "format": 1,
+            "hide": true,
+            "meta": {
+              "builderOptions": {
+                "fields": [
+                  "height"
+                ],
+                "filters": [],
+                "limit": 1,
+                "mode": "list",
+                "orderBy": [
+                  {
+                    "dir": "DESC",
+                    "name": "height"
+                  }
+                ],
+                "table": "block"
+              }
+            },
+            "queryType": "sql",
+            "rawSql": "SELECT height FROM block final ORDER BY height DESC LIMIT 1",
+            "refId": "A"
+          },
+          {
+            "datasource": {
+              "type": "grafana-clickhouse-datasource",
+              "uid": "${clickhouse}"
+            },
+            "format": 1,
+            "hide": false,
+            "meta": {
+              "builderOptions": {
+                "fields": [],
+                "limit": 100,
+                "mode": "list"
+              }
+            },
+            "queryType": "sql",
+            "rawSql": "SELECT height FROM raw_block final ORDER BY height DESC LIMIT 1",
+            "refId": "B"
+          }
+        ],
+        "title": "last block height in Clickhouse",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "grafana-clickhouse-datasource",
+          "uid": "${clickhouse}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 3,
+          "x": 10,
+          "y": 0
+        },
+        "id": 48,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "grafana-clickhouse-datasource",
+              "uid": "AIhBHDr4z"
+            },
+            "format": 1,
+            "meta": {
+              "builderOptions": {
+                "fields": [],
+                "limit": 100,
+                "mode": "list"
+              }
+            },
+            "queryType": "sql",
+            "rawSql": "SELECT count(*) FROM (\n WITH blocks AS (\n SELECT\n  toUInt64(min(height)) as st,\n  toUInt64(max(height)) as ed\n FROM\n  spacebox.raw_block_results\n )\n SELECT\n  arrayJoin(range(st, ed)) as _height FROM blocks\n) AS _blocks\nLEFT JOIN (SELECT toUInt64(height) as height FROM spacebox.raw_block_results) AS blocks ON _blocks._height = blocks.height\nWHERE height = 0;",
+            "refId": "A"
+          }
+        ],
+        "title": "count of skipped block_results",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "grafana-clickhouse-datasource",
+          "uid": "${clickhouse}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 2,
+          "x": 13,
+          "y": 0
+        },
+        "id": 49,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "grafana-clickhouse-datasource",
+              "uid": "AIhBHDr4z"
+            },
+            "format": 1,
+            "meta": {
+              "builderOptions": {
+                "fields": [],
+                "limit": 100,
+                "mode": "list"
+              }
+            },
+            "queryType": "sql",
+            "rawSql": "SELECT count(*) FROM (\n WITH blocks AS (\n SELECT\n  toUInt64(min(height)) as st,\n  toUInt64(max(height)) as ed\n FROM\n  spacebox.raw_block\n )\n SELECT\n  arrayJoin(range(st, ed)) as _height FROM blocks\n) AS _blocks\nLEFT JOIN (SELECT toUInt64(height) as height FROM spacebox.raw_block) AS blocks ON _blocks._height = blocks.height\nWHERE height = 0;",
+            "refId": "A"
+          }
+        ],
+        "title": "count of skipped blocks",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "grafana-clickhouse-datasource",
+          "uid": "${clickhouse}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 3,
+          "x": 15,
+          "y": 0
+        },
+        "id": 50,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "grafana-clickhouse-datasource",
+              "uid": "AIhBHDr4z"
+            },
+            "format": 1,
+            "meta": {
+              "builderOptions": {
+                "fields": [],
+                "limit": 100,
+                "mode": "list"
+              }
+            },
+            "queryType": "sql",
+            "rawSql": "SELECT count(*) FROM (\n\tSELECT height, msgs.height as _height FROM (\n\t\tSELECT height FROM spacebox.raw_block_results\n\t\tWHERE begin_block_events != ''\n\t\tGROUP BY height \n\t) as txs\n\tLEFT JOIN (select height from spacebox.begin_block_events GROUP BY height ) as msgs ON txs.height = msgs.height\n)\nWHERE _height = 0",
+            "refId": "A"
+          }
+        ],
+        "title": "skipped blocks in begin_block_events",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "grafana-clickhouse-datasource",
+          "uid": "${clickhouse}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 3,
+          "x": 18,
+          "y": 0
+        },
+        "id": 51,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "grafana-clickhouse-datasource",
+              "uid": "AIhBHDr4z"
+            },
+            "format": 1,
+            "meta": {
+              "builderOptions": {
+                "fields": [],
+                "limit": 100,
+                "mode": "list"
+              }
+            },
+            "queryType": "sql",
+            "rawSql": "SELECT count(*) FROM (\n\tSELECT height, msgs.height as _height FROM (\n\t\tSELECT height FROM spacebox.raw_block_results\n\t\tWHERE end_block_events != ''\n\t\tGROUP BY height \n\t) as txs\n\tLEFT JOIN (select height from spacebox.end_block_events GROUP BY height ) as msgs ON txs.height = msgs.height\n)\nWHERE _height = 0",
+            "refId": "A"
+          }
+        ],
+        "title": "skipped blocks in end_block_events",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "grafana-clickhouse-datasource",
+          "uid": "b5da250c-0e29-42ea-8316-3a49866bc58e"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 3,
+          "x": 21,
+          "y": 0
+        },
+        "id": 52,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "grafana-clickhouse-datasource",
+              "uid": "b5da250c-0e29-42ea-8316-3a49866bc58e"
+            },
+            "format": 1,
+            "meta": {
+              "builderOptions": {
+                "fields": [],
+                "limit": 100,
+                "mode": "list"
+              }
+            },
+            "queryType": "sql",
+            "rawSql": "SELECT count(*) FROM (\n\tSELECT height, msgs.height as _height FROM (\n\t\tSELECT height FROM spacebox.raw_block_results\n\t\tWHERE JSONExtractString(JSONExtractString(arrayJoin(JSONExtractArrayRaw(JSONExtractString(txs_results)))), 'events') != '[]'\n\t\tGROUP BY height \n\t) as txs\n\tLEFT JOIN (select height from spacebox.txs_events GROUP BY height ) as msgs ON txs.height = msgs.height\n)\nWHERE _height = 0",
+            "refId": "A"
+          }
+        ],
+        "title": "skipped blocks in txs_events",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "grafana-clickhouse-datasource",
+          "uid": "${clickhouse}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 5,
+          "x": 0,
+          "y": 6
+        },
+        "id": 53,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "grafana-clickhouse-datasource",
+              "uid": "${clickhouse}"
+            },
+            "format": 1,
+            "meta": {
+              "builderOptions": {
+                "fields": [],
+                "limit": 100,
+                "mode": "list"
+              }
+            },
+            "queryType": "sql",
+            "rawSql": "WITH ch AS (\n\tSELECT height, count(height) as c FROM spacebox.raw_block_results\n\tGROUP BY height\n)\nSELECT count(*) as count FROM ch WHERE c != 1",
+            "refId": "A"
+          }
+        ],
+        "title": "Double block_results",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "grafana-clickhouse-datasource",
+          "uid": "${clickhouse}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 5,
+          "x": 5,
+          "y": 6
+        },
+        "id": 54,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "grafana-clickhouse-datasource",
+              "uid": "${clickhouse}"
+            },
+            "format": 1,
+            "meta": {
+              "builderOptions": {
+                "fields": [],
+                "limit": 100,
+                "mode": "list"
+              }
+            },
+            "queryType": "sql",
+            "rawSql": "WITH ch AS (\n\tSELECT txhash, count(txhash) as c FROM spacebox.raw_transaction\n\tGROUP BY txhash\n)\nSELECT count(*) FROM ch where c != 1",
+            "refId": "A"
+          }
+        ],
+        "title": "Double transaction",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "grafana-clickhouse-datasource",
+          "uid": "${clickhouse}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 5,
+          "x": 10,
+          "y": 6
+        },
+        "id": 55,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "grafana-clickhouse-datasource",
+              "uid": "${clickhouse}"
+            },
+            "format": 1,
+            "meta": {
+              "builderOptions": {
+                "fields": [],
+                "limit": 100,
+                "mode": "list"
+              }
+            },
+            "queryType": "sql",
+            "rawSql": "WITH ch AS (\n\tSELECT height, count(height) as c FROM spacebox.raw_block\n\tGROUP BY height\n)\nSELECT count(*) as count FROM ch WHERE c != 1",
+            "refId": "A"
+          }
+        ],
+        "title": "Double blocks",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "collapsed": false,
+        "datasource": {
+          "type": "prometheus",
+          "uid": "711GLNhVz"
+        },
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 12
+        },
+        "id": 10,
+        "panels": [],
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "711GLNhVz"
+            },
+            "refId": "A"
+          }
+        ],
+        "title": "Crawler",
+        "type": "row"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "#F2495C",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 7,
+          "x": 0,
+          "y": 13
+        },
+        "id": 43,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "/^version$/",
+            "values": false
+          },
+          "text": {},
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": false,
+            "expr": "spacebox_crawler_version",
+            "format": "table",
+            "instant": true,
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Version",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "green",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 5,
+          "x": 7,
+          "y": 13
+        },
+        "id": 30,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "none",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": true,
+            "expr": "spacebox_crawler_start_height{job=\"spacebox-crawler\"}",
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Start height",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "green",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 5,
+          "x": 12,
+          "y": 13
+        },
+        "id": 31,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "none",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": true,
+            "expr": "spacebox_crawler_stop_height{job=\"spacebox-crawler\"}",
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Stop height",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 4,
+          "x": 17,
+          "y": 13
+        },
+        "id": 32,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "none",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": true,
+            "expr": "spacebox_crawler_total_workers{job=\"spacebox-crawler\"}",
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Workers count",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [
+              {
+                "options": {
+                  "0": {
+                    "index": 0,
+                    "text": "False"
+                  },
+                  "1": {
+                    "index": 1,
+                    "text": "True"
+                  }
+                },
+                "type": "value"
+              }
+            ],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 3,
+          "x": 21,
+          "y": 13
+        },
+        "id": 45,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": false,
+            "expr": "spacebox_crawler_worker_recovery_mode",
+            "instant": true,
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Recovery mode",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 6,
+          "x": 0,
+          "y": 19
+        },
+        "id": 2,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": true,
+            "expr": "sum by(exported_instance) (spacebox_crawler_cache_length)",
+            "instant": false,
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Cache length",
+        "transparent": true,
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 5,
+          "x": 6,
+          "y": 19
+        },
+        "id": 46,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": false
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "disableTextWrap": false,
+            "editorMode": "code",
+            "exemplar": true,
+            "expr": "sum(rate(spacebox_crawler_global_cache{action=\"hit\"}[$__rate_interval])) / (sum(rate(spacebox_crawler_global_cache{action=\"hit\"}[$__rate_interval])) + sum(rate(spacebox_crawler_global_cache{action=\"miss\"}[$__rate_interval])))",
+            "fullMetaSearch": false,
+            "includeNullMetadata": true,
+            "instant": false,
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A",
+            "useBackend": false
+          }
+        ],
+        "title": "Cache HIT ratio",
+        "transparent": true,
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 6,
+          "x": 11,
+          "y": 19
+        },
+        "id": 34,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": true,
+            "expr": "sum by(action) (rate(spacebox_crawler_global_cache[$__rate_interval]))",
+            "instant": false,
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Cache HIT/MISS global",
+        "transparent": true,
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 7,
+          "x": 17,
+          "y": 19
+        },
+        "id": 35,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "editorMode": "code",
+            "exemplar": true,
+            "expr": "sum by(exported_instance,action) (rate(spacebox_crawler_special_cache[$__rate_interval]))",
+            "hide": false,
+            "instant": false,
+            "interval": "",
+            "intervalFactor": 1,
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Cache HIT/MISS by instance",
+        "transparent": true,
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 5,
+          "x": 0,
+          "y": 25
+        },
+        "id": 4,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "none",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": true,
+            "expr": "spacebox_crawler_last_processed_block_height{job=\"spacebox-crawler\"}",
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Last processed height",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 10,
+          "x": 5,
+          "y": 25
+        },
+        "id": 56,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": false
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "editorMode": "code",
+            "exemplar": false,
+            "expr": "histogram_quantile(0.95, sum by(le) (rate(request_duration_seconds_bucket[$__rate_interval])))",
+            "format": "time_series",
+            "instant": false,
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "HTTP duration 95",
+        "transparent": true,
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 9,
+          "x": 15,
+          "y": 25
+        },
+        "id": 8,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": true,
+            "expr": "histogram_quantile(0.95, sum by(le, command) (rate(mongo_commands_bucket[$__rate_interval])))",
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Mongo commands duration 95",
+        "transparent": true,
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 2,
+          "x": 0,
+          "y": 31
+        },
+        "id": 47,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": true,
+            "expr": "spacebox_crawler_total_error_messages{job=\"spacebox-crawler\"}",
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Total error messages",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 2,
+          "x": 3,
+          "y": 31
+        },
+        "id": 21,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto",
+          "wideLayout": true
+        },
+        "pluginVersion": "10.2.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "editorMode": "code",
+            "exemplar": true,
+            "expr": "spacebox_crawler_total_error_txs{job=\"spacebox-crawler\"}",
+            "interval": "",
+            "legendFormat": "",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Total error tx",
+        "transparent": true,
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 19,
+          "x": 5,
+          "y": 31
+        },
+        "id": 6,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "editorMode": "code",
+            "exemplar": false,
+            "expr": "histogram_quantile(0.95, sum by(le, grpc_method) (rate(grpc_client_handling_seconds_bucket[$__rate_interval])))",
+            "format": "time_series",
+            "instant": false,
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "GRPC duration 95",
+        "transparent": true,
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 19,
+          "x": 5,
+          "y": 37
+        },
+        "id": 33,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": true,
+            "expr": "sum by (status) (spacebox_crawler_blocks_by_status)",
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Block by status",
+        "transparent": true,
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 24,
+          "x": 0,
+          "y": 43
+        },
+        "id": 23,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": true,
+            "expr": "histogram_quantile(0.95, sum by(le, type) (rate(spacebox_crawler_process_duration_bucket[$__rate_interval])))",
+            "format": "time_series",
+            "instant": false,
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Processing duration 95",
+        "transparent": true,
+        "type": "timeseries"
+      },
+      {
+        "collapsed": false,
+        "datasource": {
+          "type": "prometheus",
+          "uid": "711GLNhVz"
+        },
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 51
+        },
+        "id": 25,
+        "panels": [],
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "711GLNhVz"
+            },
+            "refId": "A"
+          }
+        ],
+        "title": "go",
+        "type": "row"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 21,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 2,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "s"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 24,
+          "x": 0,
+          "y": 52
+        },
+        "id": 29,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": false
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "pluginVersion": "8.4.2",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": true,
+            "expr": "time() - process_start_time_seconds{job=\"spacebox-crawler\"}",
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Up Time",
+        "transparent": true,
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "s"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 7,
+          "w": 12,
+          "x": 0,
+          "y": 58
+        },
+        "id": 36,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": false
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": true,
+            "expr": "rate(process_cpu_seconds_total{job=\"spacebox-crawler\"}[30s])",
+            "instant": false,
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "CPU Usage",
+        "transparent": true,
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "decbytes"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 7,
+          "w": 12,
+          "x": 12,
+          "y": 58
+        },
+        "id": 39,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": false
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": true,
+            "expr": "go_memstats_heap_alloc_bytes{job=\"spacebox-crawler\"}",
+            "instant": false,
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Heap alloc",
+        "transparent": true,
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 6,
+          "x": 0,
+          "y": 65
+        },
+        "id": 38,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": false
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": true,
+            "expr": "go_goroutines{job=\"spacebox-crawler\"}",
+            "format": "time_series",
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "Goroutines",
+        "transparent": true,
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 6,
+          "x": 6,
+          "y": 65
+        },
+        "id": 57,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": false
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "editorMode": "code",
+            "exemplar": true,
+            "expr": "go_threads{job=\"spacebox-crawler\"}",
+            "format": "time_series",
+            "interval": "",
+            "legendFormat": "",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Threads",
+        "transparent": true,
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${space}"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "s"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 12,
+          "x": 12,
+          "y": 65
+        },
+        "id": 27,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": false
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "${space}"
+            },
+            "exemplar": true,
+            "expr": "sum(rate(go_gc_duration_seconds_sum{job=\"spacebox-crawler\"}[1m]))",
+            "instant": false,
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "title": "GC Time Seconds",
+        "transparent": true,
+        "type": "timeseries"
+      }
+    ],
+    "refresh": "15m",
+    "schemaVersion": 38,
+    "tags": [],
+    "templating": {
+      "list": [
+        {
+          "current": {
+            "selected": true,
+            "text": "default",
+            "value": "default"
+          },
+          "hide": 0,
+          "includeAll": false,
+          "label": "space",
+          "multi": false,
+          "name": "space",
+          "options": [],
+          "query": "prometheus",
+          "queryValue": "",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "type": "datasource"
+        },
+        {
+          "current": {
+            "selected": true,
+            "text": "ClickHouse index demo",
+            "value": "YfpWCcUVz"
+          },
+          "hide": 0,
+          "includeAll": false,
+          "label": "database",
+          "multi": false,
+          "name": "clickhouse",
+          "options": [],
+          "query": "grafana-clickhouse-datasource",
+          "queryValue": "",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "type": "datasource"
+        }
+      ]
+    },
+    "time": {
+      "from": "now-6h",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "spacebox",
+    "uid": "spacebox_stats",
+    "version": 39,
+    "weekStart": ""
+  }
\ No newline at end of file