Add support for Jinja config files (kotekan#1015)

This PR adds an optional config file type with the extension `.j2`. Files with this type are written like normal YAML but with Jinja commands processed by the Jinja template engine converting them into pure YAML. The process looks like "Jinga -> YAML -> JSON". This added two new kotekan command line options: `--variables (-e) [json]` Pass extra variables to the jinja engine on the command line. `--print-yaml (-y)` Print the YAML being used (works for both .yaml and .j2 files. Co-authored-by: ketiltrout <[email protected]>
CHIMEFRB · Apr 7, 2022 · 758120f · 758120f
1 parent ede7534
commit 758120f
Show file tree

Hide file tree

Showing 6 changed files with 467 additions and 527 deletions.
diff --git a/config/dpdk_test_32_ports_2_numa_nodes.j2 b/config/dpdk_test_32_ports_2_numa_nodes.j2
@@ -0,0 +1,98 @@
+---
+type: config
+# Logging level can be one of:
+# OFF, ERROR, WARN, INFO, DEBUG, DEBUG2 (case insensitive)
+# Note DEBUG and DEBUG2 require a build with (-DCMAKE_BUILD_TYPE=Debug)
+log_level: debug
+
+buffer_depth: 8
+samples_per_data_set: 49152
+num_elements: 16
+num_local_freq: 128
+
+cpu_affinity: [4, 20, 5, 21, 12, 18, 13, 29]
+
+# Pool
+main_pool:
+    kotekan_metadata_pool: chimeMetadata
+    num_metadata_objects: 128 * buffer_depth
+
+# Buffers
+gpu_input_buffers:
+    num_frames: buffer_depth
+    frame_size: samples_per_data_set * num_elements * num_local_freq
+    metadata_pool: main_pool
+    numa_0:
+        numa_node: 0
+{%- for id in range(16) %}
+        gpu_input_buffer_{{ id }}:
+            kotekan_buffer: standard
+{%- endfor %}
+    numa_1:
+        numa_node: 1
+{%- for id in range(16,32) %}
+        gpu_input_buffer_{{ id }}:
+            kotekan_buffer: standard
+{%- endfor %}
+
+lost_samples_buffers:
+    num_frames: buffer_depth
+    frame_size: samples_per_data_set
+    metadata_pool: main_pool
+    numa_0:
+        numa_node: 0
+{%- for id in range(16) %}
+        lost_samples_buffer_{{ id }}:
+            kotekan_buffer: standard
+{%- endfor %}
+    numa_1:
+        numa_node: 1
+{%- for id in range(16,32) %}
+        lost_samples_buffer_{{ id }}:
+            kotekan_buffer: standard
+{%- endfor %}
+
+dpdk:
+    kotekan_stage: dpdkCore
+    # Format is index = lcore, value = cpu core
+    lcore_cpu_map: [0,16,1,17,2,18,3,19,8,24,9,25,10,26,11,27]
+    master_lcore_cpu: 2
+    fpga_packet_size: 4680
+    alignment: samples_per_data_set
+    mbuf_cache_size: 250
+    # Format is index = lcore, value = array of port IDs
+    # so [[0,1],[2,3]] maps lcore 0 to service ports 0 and 1,
+    # and lcore 1 to service ports 2 and 3.
+    lcore_port_map:
+        - [0,1]
+        - [2,3]
+        - [4,5]
+        - [6,7]
+        - [8,9]
+        - [10,11]
+        - [12,13]
+        - [14,15]
+        - [16,17]
+        - [18,19]
+        - [20,21]
+        - [22,23]
+        - [24,25]
+        - [26,27]
+        - [28,29]
+        - [30,31]
+    # One handler must be given per port on the system.
+    handlers:
+{%- for id in range(32) %}
+        - dpdk_handler: iceBoardStandard
+          out_buf: gpu_input_buffer_{{ id }}
+          lost_samples_buf: lost_samples_buffer_{{ id }}
+{%- endfor %}
+
+zero_samples:
+{%- for id in range(32) %}
+    zero_{{ id }}:
+        kotekan_stage: zeroSamples
+        out_buf: gpu_input_buffer_{{ id }}
+        lost_samples_buf: lost_samples_buffer_{{ id }}
+{%- endfor %}
+