Merge pull request kotekan#1001 from kotekan/develop

Merge develop to master for 2021.10 release

config/chime_science_run_gpu.yaml

@@ -70,6 +70,10 @@ frb_scaling: 0.05 #1.0
 # Tracking beams global options
 feed_sep_NS: 0.3048
 feed_sep_EW: 22.0
+# Latitude and longitude converted from final geocentric (XYZ) position as calculated
+# in https://bao.chimenet.ca/doc/documents/1327 via NGS NOAA website.
+inst_lat: 49.32070922
+inst_long: -119.62367743
 num_beams: 12
 num_pol: 2
 
@@ -793,6 +797,7 @@ gpu:
   buffer_depth: 4
   # How many sub frames to devide the N2 products into.
   n_intg: 24576 / num_sub_frames
+  profiling: true
   frame_arrival_period: samples_per_data_set / 390625
   updatable_config:
     tracking_pt: /updatable_config/tracking_pointing

config/chime_science_run_gpu_no_output.yaml

@@ -70,6 +70,10 @@ frb_scaling: 0.05 #1.0
 # Tracking beams global options
 feed_sep_NS: 0.3048
 feed_sep_EW: 22.0
+# Latitude and longitude converted from final geocentric (XYZ) position as calculated
+# in https://bao.chimenet.ca/doc/documents/1327 via NGS NOAA website.
+inst_lat: 49.32070922
+inst_long: -119.62367743
 num_beams: 12
 num_pol: 2
 
@@ -792,6 +796,7 @@ gpu:
   buffer_depth: 4
   # How many sub frames to devide the N2 products into.
   n_intg: 24576 / num_sub_frames
+  profiling: true
   frame_arrival_period: samples_per_data_set / 390625
   updatable_config:
     tracking_pt: /updatable_config/tracking_pointing

config/examples/basic_pipeline.yaml

@@ -3,6 +3,14 @@ type: config
 log_level: DEBUG
 cpu_affinity: [0,1,2,3]
 
+trackers:
+  enable_crash_dump: true
+  dump_path: ./
+
+cpu_monitor:
+  enabled: false
+  track_length: 2  # save last 2 mins cpu usage.
+
 main_pool:
   kotekan_metadata_pool: chimeMetadata
   num_metadata_objects: 30

config/hartrao_reduced_links.yaml

@@ -0,0 +1,329 @@
+---
+type: config
+# Logging level can be one of:
+# OFF, ERROR, WARN, INFO, DEBUG, DEBUG2 (case insensitive)
+# Note DEBUG and DEBUG2 require a build with (-DCMAKE_BUILD_TYPE=Debug)
+log_level: debug
+
+buffer_depth: 6
+samples_per_data_set: 65536
+num_elements: 16
+num_links: 6
+num_freq: 1024
+num_local_freq: num_freq / 8
+num_freq_in_frame: num_freq / 8
+block_size: 16
+num_blocks: (num_elements / block_size) * (num_elements / block_size + 1) / 2
+instrument_name: HartRAO
+num_ev: 0
+
+telescope:
+  name: ICETelescope
+  require_gps: false
+  allow_default_frequency_map: true # Don't fall back on default map
+
+dataset_manager:
+  use_dataset_broker: False
+
+sizeof_int: 4
+sizeof_float: 4
+numa_node: 0
+
+# Pool
+main_pool:
+  kotekan_metadata_pool: chimeMetadata
+  num_metadata_objects: 15 * buffer_depth
+# Buffers
+gpu_input_buffers:
+  num_frames: buffer_depth
+  frame_size: samples_per_data_set * num_elements * num_local_freq
+  metadata_pool: main_pool
+  gpu_input_buffer_0:
+    kotekan_buffer: standard
+  gpu_input_buffer_1:
+    kotekan_buffer: standard
+  gpu_input_buffer_2:
+    kotekan_buffer: standard
+  gpu_input_buffer_3:
+    kotekan_buffer: standard
+  gpu_input_buffer_4:
+    kotekan_buffer: standard
+  gpu_input_buffer_5:
+    kotekan_buffer: standard
+#  gpu_input_buffer_6:
+#    kotekan_buffer: standard
+#  gpu_input_buffer_7:
+#    kotekan_buffer: standard
+
+lost_samples_buffers:
+  num_frames: buffer_depth
+  frame_size: samples_per_data_set
+  metadata_pool: main_pool
+  lost_samples_buffer_0:
+    kotekan_buffer: standard
+  lost_samples_buffer_1:
+    kotekan_buffer: standard
+  lost_samples_buffer_2:
+    kotekan_buffer: standard
+  lost_samples_buffer_3:
+    kotekan_buffer: standard
+  lost_samples_buffer_4:
+    kotekan_buffer: standard
+  lost_samples_buffer_5:
+    kotekan_buffer: standard
+#  lost_samples_buffer_6:
+#    kotekan_buffer: standard
+#  lost_samples_buffer_7:
+#    kotekan_buffer: standard
+
+merged_network_buf:
+  kotekan_buffer: standard
+  num_frames: buffer_depth * num_links
+  frame_size: samples_per_data_set * num_elements * num_local_freq
+  metadata_pool: main_pool
+
+gpu_output_buffers:
+  num_frames: buffer_depth * num_links
+  frame_size: num_local_freq * num_blocks * (block_size*block_size) * 2  * sizeof_int
+  metadata_pool: main_pool
+  gpu_output_buffer:
+    kotekan_buffer: standard
+
+gpu_output_split_buffer:
+  num_frames: buffer_depth
+  frame_size: num_local_freq * num_blocks * (block_size*block_size) * 2  * sizeof_int
+  metadata_pool: main_pool
+  gpu_output_split_buffer_0:
+    kotekan_buffer: standard
+  gpu_output_split_buffer_1:
+    kotekan_buffer: standard
+  gpu_output_split_buffer_2:
+    kotekan_buffer: standard
+  gpu_output_split_buffer_3:
+    kotekan_buffer: standard
+  gpu_output_split_buffer_4:
+    kotekan_buffer: standard
+  gpu_output_split_buffer_5:
+    kotekan_buffer: standard
+#  gpu_output_split_buffer_6:
+#    kotekan_buffer: standard
+#  gpu_output_split_buffer_7:
+#    kotekan_buffer: standard
+
+cpu_affinity: [2,3,6,7] # other cores are on DPDK
+dpdk:
+  kotekan_stage: dpdkCore
+  # Format is index = lcore, value = cpu core
+  lcore_cpu_map: [0,1,4,5]
+  master_lcore_cpu: 3
+  num_mem_channels: 3
+  fpga_packet_size: 4680
+  alignment: samples_per_data_set
+  # Format is index = lcore, value = array of port IDs
+  # so [[0,1],[2,3]] maps lcore 0 to service ports 0 and 1,
+  # and lcore 1 to service ports 2 and 3.
+  lcore_port_map:
+    - [0,1]
+    - [2,3]
+    - [4,5]
+    - [6,7]
+  # One handler must be given per port on the system.
+  handlers:
+    - dpdk_handler: iceBoardStandard
+      out_buf: gpu_input_buffer_0
+      lost_samples_buf: lost_samples_buffer_0
+    - dpdk_handler: iceBoardStandard
+      out_buf: gpu_input_buffer_1
+      lost_samples_buf: lost_samples_buffer_1
+    - dpdk_handler: iceBoardStandard
+      out_buf: gpu_input_buffer_2
+      lost_samples_buf: lost_samples_buffer_2
+    - dpdk_handler: iceBoardStandard
+      out_buf: gpu_input_buffer_3
+      lost_samples_buf: lost_samples_buffer_3
+    - dpdk_handler: iceBoardStandard
+      out_buf: gpu_input_buffer_4
+      lost_samples_buf: lost_samples_buffer_4
+    - dpdk_handler: iceBoardStandard
+      out_buf: gpu_input_buffer_5
+      lost_samples_buf: lost_samples_buffer_5
+    - dpdk_handler: none
+    - dpdk_handler: none
+#    - dpdk_handler: iceBoardStandard
+#      out_buf: gpu_input_buffer_6
+#      lost_samples_buf: lost_samples_buffer_6
+#    - dpdk_handler: iceBoardStandard
+#      out_buf: gpu_input_buffer_7
+#      lost_samples_buf: lost_samples_buffer_7
+
+zero_samples:
+  zero_0:
+    kotekan_stage: zeroSamples
+    out_buf: gpu_input_buffer_0
+    lost_samples_buf: lost_samples_buffer_0
+  zero_1:
+    kotekan_stage: zeroSamples
+    out_buf: gpu_input_buffer_1
+    lost_samples_buf: lost_samples_buffer_1
+  zero_2:
+    kotekan_stage: zeroSamples
+    out_buf: gpu_input_buffer_2
+    lost_samples_buf: lost_samples_buffer_2
+  zero_3:
+    kotekan_stage: zeroSamples
+    out_buf: gpu_input_buffer_3
+    lost_samples_buf: lost_samples_buffer_3
+  zero_4:
+    kotekan_stage: zeroSamples
+    out_buf: gpu_input_buffer_4
+    lost_samples_buf: lost_samples_buffer_4
+  zero_5:
+    kotekan_stage: zeroSamples
+    out_buf: gpu_input_buffer_5
+    lost_samples_buf: lost_samples_buffer_5
+#  zero_6:
+#    kotekan_stage: zeroSamples
+#    out_buf: gpu_input_buffer_6
+#    lost_samples_buf: lost_samples_buffer_6
+#  zero_7:
+#    kotekan_stage: zeroSamples
+#    out_buf: gpu_input_buffer_7
+#    lost_samples_buf: lost_samples_buffer_7
+
+merge:
+  kotekan_stage: bufferMerge
+  in_bufs:
+    - gpu_input_buffer_0
+    - gpu_input_buffer_1
+    - gpu_input_buffer_2
+    - gpu_input_buffer_3
+    - gpu_input_buffer_4
+    - gpu_input_buffer_5
+#    - gpu_input_buffer_6
+#    - gpu_input_buffer_7
+  out_buf: merged_network_buf
+
+gpu:
+  num_data_sets: 1
+  profiling: true
+  legacy_opencl: true
+  kernel_path: "/var/lib/kotekan/opencl/kernels/"
+  log_level: WARN
+  frame_arrival_period: samples_per_data_set / 390625 / num_links
+  commands: &command_list
+    - name: clInputData
+    - name: clPresumZero
+    - name: clOutputDataZero
+    - name: clPresumKernel
+    - name: clKVCorr
+    - name: clOutputData
+  gpu_0:
+    kotekan_stage: clProcess
+    gpu_id: 0
+    commands: *command_list
+    in_buffers:
+      network_buf: merged_network_buf
+    out_buffers:
+      output_buf: gpu_output_buffer
+
+split:
+  kotekan_stage: BufferSplit
+  in_buf: gpu_output_buffer
+  out_bufs:
+    - gpu_output_split_buffer_0
+    - gpu_output_split_buffer_1
+    - gpu_output_split_buffer_2
+    - gpu_output_split_buffer_3
+    - gpu_output_split_buffer_4
+    - gpu_output_split_buffer_5
+#    - gpu_output_split_buffer_6
+#    - gpu_output_split_buffer_7
+
+# Metadata pool
+vis_pool:
+  kotekan_metadata_pool: VisMetadata
+  num_metadata_objects: 20 * buffer_depth * num_freq_in_frame
+
+# Buffers
+vis_buffers:
+  metadata_pool: vis_pool
+  num_frames: buffer_depth * num_freq_in_frame
+  visbuf_5s_merged:
+    kotekan_buffer: vis
+  visbuf_5s_0:
+    kotekan_buffer: vis
+  visbuf_5s_1:
+    kotekan_buffer: vis
+  visbuf_5s_2:
+    kotekan_buffer: vis
+  visbuf_5s_3:
+    kotekan_buffer: vis
+  visbuf_5s_4:
+    kotekan_buffer: vis
+  visbuf_5s_5:
+    kotekan_buffer: vis
+#  visbuf_5s_6:
+#    kotekan_buffer: vis
+#  visbuf_5s_7:
+#    kotekan_buffer: vis
+
+vis_accumulate_processes:
+  log_level: WARN
+  integration_time: 5.0
+  acc_0:
+    kotekan_stage: visAccumulate
+    in_buf: gpu_output_split_buffer_0
+    out_buf: visbuf_5s_0
+    log_level: INFO
+  acc_1:
+    kotekan_stage: visAccumulate
+    in_buf: gpu_output_split_buffer_1
+    out_buf: visbuf_5s_1
+  acc_2:
+    kotekan_stage: visAccumulate
+    in_buf: gpu_output_split_buffer_2
+    out_buf: visbuf_5s_2
+  acc_3:
+    kotekan_stage: visAccumulate
+    in_buf: gpu_output_split_buffer_3
+    out_buf: visbuf_5s_3
+  acc_4:
+    kotekan_stage: visAccumulate
+    in_buf: gpu_output_split_buffer_4
+    out_buf: visbuf_5s_4
+  acc_5:
+    kotekan_stage: visAccumulate
+    in_buf: gpu_output_split_buffer_5
+    out_buf: visbuf_5s_5
+#  acc_6:
+#    kotekan_stage: visAccumulate
+#    in_buf: gpu_output_split_buffer_6
+#    out_buf: visbuf_5s_6
+#  acc_7:
+#    kotekan_stage: visAccumulate
+#    in_buf: gpu_output_split_buffer_7
+#    out_buf: visbuf_5s_7
+
+merge_acc:
+  kotekan_stage: bufferMerge
+  in_bufs:
+    - visbuf_5s_0
+    - visbuf_5s_1
+    - visbuf_5s_2
+    - visbuf_5s_3
+    - visbuf_5s_4
+    - visbuf_5s_5
+#    - visbuf_5s_6
+#    - visbuf_5s_7
+  out_buf: visbuf_5s_merged
+
+vis_writer:
+  kotekan_stage: VisWriter
+  in_buf: visbuf_5s_merged
+  node_mode: true
+  file_type: hdf5
+  log_level: INFO
+  root_path: /data/archive/
+  freq_ids: []
+  write_ev: False
+  file_length: 512

config/single_iceboard_n2.yaml

@@ -210,7 +210,7 @@ gpu:
   kernel_path: "/var/lib/kotekan/opencl/kernels/"
   log_level: WARN
   frame_arrival_period: samples_per_data_set / 390625 / num_links
-  commands:
+  commands: &command_list
     - name: clInputData
     - name: clPresumZero
     - name: clOutputDataZero
@@ -220,6 +220,7 @@ gpu:
   gpu_0:
     kotekan_stage: clProcess
     gpu_id: 0
+    commands: *command_list
     in_buffers:
       network_buf: merged_network_buf
     out_buffers: