From 5d51134848f9ee8a2d20008861967759a0c4b8d9 Mon Sep 17 00:00:00 2001 From: james-s-willis <36906682+james-s-willis@users.noreply.github.com> Date: Tue, 15 Dec 2020 14:15:50 -0500 Subject: [PATCH 01/15] Created RawReader base class (#910) * Abstracted the visRawReader stage into RawReader and created HFBRawReader to inherit from it. --- lib/stages/CMakeLists.txt | 4 +- lib/stages/HFBRawReader.cpp | 76 +++++ lib/stages/HFBRawReader.hpp | 59 ++++ lib/stages/RawReader.cpp | 197 +++++++++++ lib/stages/RawReader.hpp | 629 ++++++++++++++++++++++++++++++++++++ lib/stages/VisRawReader.cpp | 87 +++++ lib/stages/VisRawReader.hpp | 77 +++++ lib/stages/visRawReader.cpp | 580 --------------------------------- lib/stages/visRawReader.hpp | 287 ---------------- lib/utils/FrameView.hpp | 1 + lib/utils/HFBFrameView.cpp | 16 + lib/utils/HFBFrameView.hpp | 2 + lib/utils/visBuffer.cpp | 17 + lib/utils/visBuffer.hpp | 2 + python/kotekan/runner.py | 2 +- 15 files changed, 1167 insertions(+), 869 deletions(-) create mode 100644 lib/stages/HFBRawReader.cpp create mode 100644 lib/stages/HFBRawReader.hpp create mode 100644 lib/stages/RawReader.cpp create mode 100644 lib/stages/RawReader.hpp create mode 100644 lib/stages/VisRawReader.cpp create mode 100644 lib/stages/VisRawReader.hpp delete mode 100644 lib/stages/visRawReader.cpp delete mode 100644 lib/stages/visRawReader.hpp diff --git a/lib/stages/CMakeLists.txt b/lib/stages/CMakeLists.txt index 9ed4875ee..ff117d2b2 100644 --- a/lib/stages/CMakeLists.txt +++ b/lib/stages/CMakeLists.txt @@ -38,7 +38,9 @@ add_library( BaseWriter.cpp VisWriter.cpp HFBWriter.cpp - visRawReader.cpp + RawReader.cpp + VisRawReader.cpp + HFBRawReader.cpp restInspectFrame.cpp frbNetworkProcess.cpp pulsarNetworkProcess.cpp diff --git a/lib/stages/HFBRawReader.cpp b/lib/stages/HFBRawReader.cpp new file mode 100644 index 000000000..b58c12486 --- /dev/null +++ b/lib/stages/HFBRawReader.cpp @@ -0,0 +1,76 @@ +#include "HFBRawReader.hpp" + +#include "Config.hpp" // for Config +#include "HFBFrameView.hpp" // for HFBFrameView +#include "HFBMetadata.hpp" // for HFBMetadata +#include "Stage.hpp" // for Stage +#include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate +#include "bufferContainer.hpp" // for bufferContainer +#include "datasetManager.hpp" // for state_id_t, datasetManager, dset_id_t +#include "datasetState.hpp" // for beamState, subfreqState +#include "kotekanLogging.hpp" // for DEBUG, WARN +#include "visUtil.hpp" // for frameID + +#include "fmt.hpp" // for format, fmt +#include "json.hpp" // for basic_json<>::object_t, json, basic_json, basic_json<>::v... + +#include // for fill, max +#include // for uint32_t +#include // for runtime_error +#include // for pair + +using kotekan::bufferContainer; +using kotekan::Config; +using kotekan::Stage; +using nlohmann::json; + +REGISTER_KOTEKAN_STAGE(HFBRawReader); + +HFBRawReader::HFBRawReader(Config& config, const std::string& unique_name, + bufferContainer& buffer_container) : + RawReader(config, unique_name, buffer_container) { + + // Extract data specific indices + _beams = metadata_json["index_map"]["beam"].get>(); + _subfreqs = metadata_json["index_map"]["subfreq"].get>(); + + // Check metadata is the correct size + if (sizeof(HFBMetadata) != metadata_size) { + std::string msg = fmt::format(fmt("Metadata in file {:s} is larger ({:d} bytes) than " + "HFBMetadata ({:d} bytes)."), + filename, metadata_size, sizeof(HFBMetadata)); + throw std::runtime_error(msg); + } + + // Register a state for the time axis if using comet, or register the replacement dataset ID if + // using + if (update_dataset_id) { + + datasetManager& dm = datasetManager::instance(); + + if (!use_comet) { + // Add data specific states + states.push_back(dm.create_state(_beams).first); + states.push_back(dm.create_state(_subfreqs).first); + + // register it as root dataset + static_out_dset_id = dm.add_dataset(states); + + WARN("Updating the dataset IDs without comet is not recommended " + "as it will not preserve dataset ID changes."); + } + } +} + +HFBRawReader::~HFBRawReader() {} + +void HFBRawReader::create_empty_frame(frameID frame_id) { + + // Create frame and set structural metadata + auto frame = + HFBFrameView::create_frame_view(out_buf, frame_id, _beams.size(), _subfreqs.size()); + + frame.zero_frame(); + + DEBUG("HFBRawReader: Reading empty frame: {:d}", frame_id); +} diff --git a/lib/stages/HFBRawReader.hpp b/lib/stages/HFBRawReader.hpp new file mode 100644 index 000000000..3b50b2951 --- /dev/null +++ b/lib/stages/HFBRawReader.hpp @@ -0,0 +1,59 @@ +/***************************************** +@file +@brief Read HFBFileRaw data. +- HFBRawReader : public RawReader +*****************************************/ +#ifndef _HFB_RAW_READER_HPP +#define _HFB_RAW_READER_HPP + +#include "Config.hpp" // for Config +#include "HFBFrameView.hpp" +#include "RawReader.hpp" // for RawReader +#include "bufferContainer.hpp" // for bufferContainer +#include "visUtil.hpp" // for frameID + +#include // for uint32_t +#include // for string +#include // for vector + +/** + * @class HFBRawReader + * @brief Read and stream a raw 21cm absorber file. + * + * This class inherits from the RawReader base class and reads raw 21cm absorber data + * @author James Willis + */ +class HFBRawReader : public RawReader { + +public: + /// default constructor + HFBRawReader(kotekan::Config& config, const std::string& unique_name, + kotekan::bufferContainer& buffer_container); + + ~HFBRawReader(); + + /** + * @brief Get the beams in the file. + **/ + const std::vector& beams() { + return _beams; + } + + /** + * @brief Get the sub-frequencies in the file. + **/ + const std::vector& subfreqs() { + return _subfreqs; + } + +protected: + // Create an empty frame + void create_empty_frame(frameID frame_id) override; + +private: + // The metadata + std::vector _beams; + std::vector _subfreqs; +}; + +#endif diff --git a/lib/stages/RawReader.cpp b/lib/stages/RawReader.cpp new file mode 100644 index 000000000..1ee0729bb --- /dev/null +++ b/lib/stages/RawReader.cpp @@ -0,0 +1,197 @@ +#include "RawReader.hpp" + +#include "Config.hpp" // for Config +#include "Hash.hpp" // for Hash, operator<, operator== +#include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate +#include "buffer.h" // for mark_frame_full, wait_for_empty_frame, mark_frame_empty +#include "bufferContainer.hpp" // for bufferContainer +#include "datasetManager.hpp" // for state_id_t, dset_id_t, datasetManager, DS_UNIQUE_NAME +#include "datasetState.hpp" // for freqState, timeState, metadataState +#include "kotekanLogging.hpp" // for INFO, FATAL_ERROR, DEBUG, WARN, ERROR +#include "visBuffer.hpp" // for VisFrameView +#include "visUtil.hpp" // for time_ctype, freq_ctype, frameID, modulo, current_time + +#include "json.hpp" // for basic_json<>::object_t, json, basic_json, basic_json<>::v... + +#include // for atomic_bool +#include // for __forced_unwind +#include // for exception +#include // for _Bind_helper<>::type, bind, function +#include // for async, future +#include // for allocator_traits<>::value_type +#include // for match_results<>::_Base_type +#include // for runtime_error, invalid_argument, out_of_range +#include // for system_error +#include // for get + +using kotekan::bufferContainer; +using kotekan::Config; +using kotekan::Stage; +using nlohmann::json; + +REGISTER_KOTEKAN_STAGE(ensureOrdered); + +ensureOrdered::ensureOrdered(Config& config, const std::string& unique_name, + bufferContainer& buffer_container) : + Stage(config, unique_name, buffer_container, std::bind(&ensureOrdered::main_thread, this)) { + + max_waiting = config.get_default(unique_name, "max_waiting", 100); + + chunked = config.exists(unique_name, "chunk_size"); + if (chunked) { + chunk_size = config.get>(unique_name, "chunk_size"); + if (chunk_size.size() != 3) { + FATAL_ERROR("Chunk size needs exactly three elements (got {:d}).", chunk_size.size()); + return; + } + chunk_t = chunk_size[2]; + chunk_f = chunk_size[0]; + if (chunk_size[0] < 1 || chunk_size[1] < 1 || chunk_size[2] < 1) { + FATAL_ERROR("Chunk dimensions need to be >= 1 (got ({:d}, {:d}, {:d}).", chunk_size[0], + chunk_size[1], chunk_size[2]); + return; + } + } + + // Get the list of buffers that this stage should connect to + out_buf = get_buffer("out_buf"); + register_producer(out_buf, unique_name.c_str()); + in_buf = get_buffer("in_buf"); + register_consumer(in_buf, unique_name.c_str()); +} + +bool ensureOrdered::get_dataset_state(dset_id_t ds_id) { + + datasetManager& dm = datasetManager::instance(); + + // Get the states synchronously. + auto tstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); + auto fstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); + + const timeState* tstate = tstate_fut.get(); + const freqState* fstate = fstate_fut.get(); + + if (tstate == nullptr || fstate == nullptr) { + ERROR("One of time or freq dataset states is null for dataset {}.", ds_id); + return false; + } + + auto times = tstate->get_times(); + auto freq_pairs = fstate->get_freqs(); + + // construct map of times to axis index + for (size_t i = 0; i < times.size(); i++) { + time_map.insert({times.at(i), i}); + } + // construct map of freq_ind to axis index + for (size_t i = 0; i < freq_pairs.size(); i++) { + freq_map.insert({freq_pairs.at(i).first, i}); + } + + return true; +} + +void ensureOrdered::main_thread() { + + // The index to the current buffer frame + frameID frame_id(in_buf); + frameID output_frame_id(out_buf); + // The index of the current frame relative to the first frame + size_t output_ind = 0; + + // Frequency and time indices + size_t fi, ti; + time_ctype t; + + // The dataset ID we read from the frame + dset_id_t ds_id; + + // Get axes from dataset state + uint32_t first_ind = 0; + while (true) { + // Wait for a frame in the input buffer in order to get the dataset ID + if ((wait_for_full_frame(in_buf, unique_name.c_str(), first_ind)) == nullptr) { + return; + } + auto frame = VisFrameView(in_buf, first_ind); + if (frame.fpga_seq_length == 0) { + INFO("Got empty frame ({:d}).", first_ind); + first_ind++; + } else { + ds_id = frame.dataset_id; + break; + } + } + + auto future_ds_state = std::async(&ensureOrdered::get_dataset_state, this, ds_id); + if (!future_ds_state.get()) { + FATAL_ERROR("Couldn't find ancestor of dataset {}. " + "Make sure there is a stage upstream in the config, that adds the dataset " + "states.\nExiting...", + ds_id); + return; + } + + // main loop: + while (!stop_thread) { + // Wait for a full frame in the input buffer + if ((wait_for_full_frame(in_buf, unique_name.c_str(), frame_id)) == nullptr) { + break; + } + auto frame = VisFrameView(in_buf, frame_id); + + // Figure out the ordered index of this frame + t = {std::get<0>(frame.time), ts_to_double(std::get<1>(frame.time))}; + ti = time_map.at(t); + fi = freq_map.at(frame.freq_id); + size_t ordered_ind = get_frame_ind(ti, fi); + + // Check if this is the index we are ready to send + if (ordered_ind == output_ind) { + // copy frame into output buffer + if (wait_for_empty_frame(out_buf, unique_name.c_str(), output_frame_id) == nullptr) { + return; + } + auto output_frame = + VisFrameView::copy_frame(in_buf, frame_id, out_buf, output_frame_id); + mark_frame_full(out_buf, unique_name.c_str(), output_frame_id++); + + // release input frame + mark_frame_empty(in_buf, unique_name.c_str(), frame_id++); + + // increment output index + output_ind++; + } else if (waiting.size() <= max_waiting) { + INFO("Frame {:d} arrived out of order. Expected {:d}. Adding it to waiting buffer.", + ordered_ind, output_ind); + // Add to waiting frames and move to next (without marking empty!) + waiting.insert({ordered_ind, (int)frame_id}); + frame_id++; + } else { + FATAL_ERROR("Number of frames arriving out of order exceeded maximum buffer size."); + return; + } + + // Check if any of the waiting frames are ready + auto ready = waiting.find(output_ind); + while (ready != waiting.end()) { + // remove this index from waiting map + uint32_t waiting_id = ready->second; + waiting.erase(ready); + INFO("Frame {:d} is ready to be sent. Releasing buffer.", output_ind); + // copy frame into output buffer + auto past_frame = VisFrameView(in_buf, waiting_id); + if (wait_for_empty_frame(out_buf, unique_name.c_str(), output_frame_id) == nullptr) { + return; + } + auto output_frame = + VisFrameView::copy_frame(in_buf, waiting_id, out_buf, output_frame_id); + mark_frame_full(out_buf, unique_name.c_str(), output_frame_id++); + + mark_frame_empty(in_buf, unique_name.c_str(), waiting_id); + output_ind++; + + ready = waiting.find(output_ind); + } + } +} diff --git a/lib/stages/RawReader.hpp b/lib/stages/RawReader.hpp new file mode 100644 index 000000000..f7fec0007 --- /dev/null +++ b/lib/stages/RawReader.hpp @@ -0,0 +1,629 @@ +/***************************************** +@file +@brief Base class for reading raw files. +- RawReader : public kotekan::Stage +*****************************************/ +#ifndef _RAW_READER_HPP +#define _RAW_READER_HPP + +#include "Config.hpp" +#include "Hash.hpp" // for Hash, operator<, operator== +#include "Stage.hpp" // for Stage +#include "Telescope.hpp" // for Telescope +#include "buffer.h" +#include "bufferContainer.hpp" +#include "datasetManager.hpp" // for dset_id_t +#include "datasetState.hpp" // for freqState, timeState, metadataState +#include "errors.h" // for exit_kotekan, CLEAN_EXIT, ReturnCode +#include "kotekanLogging.hpp" // for INFO, FATAL_ERROR, DEBUG, WARN, ERROR +#include "metadata.h" // for metadataContainer +#include "version.h" // for get_git_commit_hash +#include "visUtil.hpp" // for freq_ctype (ptr only), input_ctype, prod_ctype, rstack_ctype + +#include "fmt.hpp" // for format, fmt +#include "json.hpp" // for json + +#include // for strerror, memcpy +#include // for errno +#include // for exception +#include // for open, posix_fadvise, O_RDONLY, POSIX_FADV_DONTNEED +#include // for ifstream, ios_base::failure, ios_base, basic_ios, basic_i... +#include // for _Bind_helper<>::type, bind, function +#include // for map +#include // for match_results<>::_Base_type +#include // for size_t +#include // for runtime_error, invalid_argument, out_of_range +#include // for uint32_t, uint8_t +#include // for string +#include // for madvise, mmap, munmap, MADV_DONTNEED, MADV_WILLNEED, MAP_... +#include // for stat +#include // for nanosleep, timespec +#include // for close, off_t +#include // for pair +#include // for vector + +using kotekan::bufferContainer; +using kotekan::Config; +using kotekan::Stage; +using nlohmann::json; + +/** + * @class RawReader + * @brief Generic class to read and stream a raw file. + * + * All classes which inherit from this should provide the following API: + * + * create_empty_frame(frameID frame_id); + * get_dataset_id(frameID frame_id); + * + * This stage will divide the file up into time-frequency chunks of set size and + * stream out the frames with time as the *fastest* index. The dataset ID + * will be restored from the dataset broker if `use_comet` is set. Otherwise + * a new dataset will be created and the original ID stored in the frames + * will be lost. + * + * The chunking strategy aids the downstream Transpose stage that writes to a HDF5 + * file with a chunked layout. The file writing is most efficient when writing entire + * chunks on exact chunk boundaries. The reason for using chunking in the first place + * is to enable compression and try to optimise for certain IO patterns + * (i.e. read a few frequencies for many times). + * + * @par Buffers + * @buffer out_buf The data read from the raw file. + * @buffer_format Buffer structured + * @buffer_metadata Metadata + * + * @conf readahead_blocks Int. Number of blocks to advise OS to read ahead + * of current read. + * @conf chunk_size Array of [int, int, int]. Read chunk size (freq, + * prod, time). If not specified will read file + * contiguously. + * @conf infile String. Path to the (data-meta-pair of) files to + * read (e.g. "/path/to/0000_000", without .data or + * .meta). + * @conf max_read_rate Float. Maximum read rate for the process in MB/s. + * If the value is zero (default), then no rate + * limiting is applied. + * @conf sleep_time Float. After the data is read pause this long in + * seconds before sending shutdown. If < 0, never + * send a shutdown signal. Default is -1. + * @conf update_dataset_id Bool. Update the dataset ID with information about the + file, for example which time samples does it contain. + * @conf use_dataset_broker Bool. Restore dataset ID from dataset broker (i.e. comet). + * Should be disabled only for testing. Default is true. + * @conf use_local_dataset_man Bool. Instead of using comet, register metadata on top + * of dataset ID in the file. Should only be used for + * testing or if original dataset IDs can be lost. + * Default is False. + * + * @author Richard Shaw, Tristan Pinsonneault-Marotte, Rick Nitsche, James Willis + */ +template +class RawReader : public kotekan::Stage { + +public: + /// default constructor + RawReader(kotekan::Config& config, const std::string& unique_name, + kotekan::bufferContainer& buffer_container); + + ~RawReader(); + + /// Main loop over buffer frames + void main_thread() override; + + /** + * @brief Get the times in the file. + **/ + const std::vector& times() { + return _times; + } + + /** + * @brief Get the frequencies in the file. + **/ + const std::vector>& freqs() { + return _freqs; + } + + /** + * @brief Get the metadata saved into the file. + **/ + const nlohmann::json& metadata() { + return _metadata; + } + +protected: + // Dataset states constructed from metadata + std::vector states; + + // The input file + std::string filename; + + // Metadata size + size_t metadata_size; + + // whether to update the dataset ID with info about the file + bool update_dataset_id; + + // whether to use comet to track dataset IDs + bool use_comet; + + // Dataset ID to assign to output frames if not using comet + dset_id_t static_out_dset_id; + + // Metadata file in json format + json metadata_json; + + Buffer* out_buf; + +private: + // Create an empty frame + virtual void create_empty_frame(frameID frame_id) = 0; + + /** + * @brief Get the new dataset ID. + * + * If not using change the ID this just returns its input ID. If using the + * broker, this will simply append a timeState to the current state. + * Otherwise, we just use a static dataset_id constructed from the file + * metadata. + * + * @param ds_id The ID of the read frame. + * @returns The replacement ID + */ + dset_id_t get_dataset_state(dset_id_t ds_id); + + /** + * @brief Read the next frame. + * + * The exact frame read is dependent on whether the reads are time ordered + * or not. + * + * @param ind The frame index to read. + **/ + void read_ahead(int ind); + + /** + * @brief Map the index into a frame position in the file. + * + * The exact frame read dependent on whether the reads are time ordered + * or not. + * + * @param ind The frame index. + * @returns The frame index into the file. + **/ + int position_map(int ind); + + // The metadata + nlohmann::json _metadata; + std::vector _times; + std::vector> _freqs; + + // whether to read in chunks + bool chunked; + + // whether to use the local dataset manager + bool local_dm; + + // Read chunk size (freq, prod, time) + std::vector chunk_size; + // time chunk size + size_t chunk_t; + // freq chunk size + size_t chunk_f; + + // Number of elements in a chunked row + size_t row_size; + + // the input file + int fd; + uint8_t* mapped_file; + + size_t file_frame_size, data_size, nfreq, ntime; + + // Number of blocks to read ahead while reading from disk + size_t readahead_blocks; + + // the dataset state for the time axis + state_id_t tstate_id; + + // Map input to output dataset IDs for quick access + std::map ds_in_file; + + // The read rate + double max_read_rate; + + // Sleep time after reading + double sleep_time; +}; + +template +RawReader::RawReader(Config& config, const std::string& unique_name, + bufferContainer& buffer_container) : + Stage(config, unique_name, buffer_container, std::bind(&RawReader::main_thread, this)) { + + filename = config.get(unique_name, "infile"); + readahead_blocks = config.get(unique_name, "readahead_blocks"); + max_read_rate = config.get_default(unique_name, "max_read_rate", 0.0); + sleep_time = config.get_default(unique_name, "sleep_time", -1); + update_dataset_id = config.get_default(unique_name, "update_dataset_id", true); + use_comet = config.get_default(DS_UNIQUE_NAME, "use_dataset_broker", true); + local_dm = config.get_default(unique_name, "use_local_dataset_man", false); + if (local_dm && use_comet) + FATAL_ERROR("Cannot use local dataset manager and dataset broker together.") + + chunked = config.exists(unique_name, "chunk_size"); + if (chunked) { + chunk_size = config.get>(unique_name, "chunk_size"); + if (chunk_size.size() != 3) + throw std::invalid_argument("Chunk size needs exactly three " + "elements (has " + + std::to_string(chunk_size.size()) + ")."); + chunk_t = chunk_size[2]; + chunk_f = chunk_size[0]; + if (chunk_size[0] < 1 || chunk_size[1] < 1 || chunk_size[2] < 1) + throw std::invalid_argument("RawReader: config: Chunk size " + "needs to be greater or equal to (1,1,1) (is (" + + std::to_string(chunk_size[0]) + "," + + std::to_string(chunk_size[1]) + "," + + std::to_string(chunk_size[2]) + "))."); + } + + // Get the list of buffers that this stage should connect to + out_buf = get_buffer("out_buf"); + register_producer(out_buf, unique_name.c_str()); + + // Read the metadata + std::string md_filename = (filename + ".meta"); + INFO("Reading metadata file: {:s}", md_filename); + struct stat st; + if (stat(md_filename.c_str(), &st) == -1) + throw std::ios_base::failure( + fmt::format(fmt("RawReader: Error reading from metadata file: {:s}"), md_filename)); + size_t filesize = st.st_size; + std::vector packed_json(filesize); + + std::ifstream metadata_file(md_filename, std::ios::binary); + if (metadata_file) // only read if no error + metadata_file.read((char*)&packed_json[0], filesize); + if (!metadata_file) // check if open and read successful + throw std::ios_base::failure("RawReader: Error reading from " + "metadata file: " + + md_filename); + metadata_json = json::from_msgpack(packed_json); + metadata_file.close(); + + // Extract the attributes and index maps + _metadata = metadata_json["attributes"]; + _times = metadata_json["index_map"]["time"].template get>(); + auto freqs = metadata_json["index_map"]["freq"].template get>(); + + // Match frequencies to IDs in the Telescope... + auto& tel = Telescope::instance(); + std::map inv_freq_map; + + // ... first construct a map of central frequencies to IDs known by the + // telescope object + for (uint32_t id = 0; id < tel.num_freq(); id++) { + inv_freq_map[tel.to_freq(id)] = id; + } + + // ... then use this to match the central frequencies given in the file + for (auto f : freqs) { + + auto it = inv_freq_map.find(f.centre); + + if (it == inv_freq_map.end()) { + FATAL_ERROR("Could not match a frequency ID to channel in file at {} MHz. " + "Check you are specifying the correct telescope.", + f.centre); + return; + } + + DEBUG("restored freq_id for f_centre={:.2f} : {:d}", f.centre, it->second); + _freqs.push_back({it->second, f}); + } + + // check git version tag + // TODO: enforce that they match if build type == "release"? + if (_metadata.at("git_version_tag").template get() + != std::string(get_git_commit_hash())) + INFO("Git version tags don't match: dataset in file {:s} has tag {:s}, while the local git " + "version tag is {:s}", + filename, _metadata.at("git_version_tag").template get(), + get_git_commit_hash()); + + // Extract the structure + file_frame_size = metadata_json["structure"]["frame_size"].template get(); + metadata_size = metadata_json["structure"]["metadata_size"].template get(); + data_size = metadata_json["structure"]["data_size"].template get(); + nfreq = metadata_json["structure"]["nfreq"].template get(); + ntime = metadata_json["structure"]["ntime"].template get(); + + DEBUG("Metadata fields. frame_size: {}, metadata_size: {}, data_size: {}, nfreq: {}, ntime: {}", + file_frame_size, metadata_size, data_size, nfreq, ntime); + + if (chunked) { + // Special case if dimensions less than chunk size + chunk_f = std::min(chunk_f, nfreq); + chunk_t = std::min(chunk_t, ntime); + + // Number of elements in a chunked row + row_size = chunk_t * nfreq; + } + + // Check that buffer is large enough + if ((unsigned int)(out_buf->frame_size) < data_size || out_buf->frame_size < 0) { + std::string msg = + fmt::format(fmt("Data in file {:s} is larger ({:d} bytes) than buffer size " + "({:d} bytes)."), + filename, data_size, out_buf->frame_size); + throw std::runtime_error(msg); + } + + // Register a state for the time axis if using comet, or register the replacement dataset ID if + // using + if (update_dataset_id) { + + datasetManager& dm = datasetManager::instance(); + tstate_id = dm.create_state(_times).first; + + if (!use_comet) { + // Add the states: metadata, time, freq + states.push_back(tstate_id); + states.push_back(dm.create_state(_freqs).first); + states.push_back(dm.create_state(_metadata.at("weight_type"), + _metadata.at("instrument_name"), + _metadata.at("git_version_tag")) + .first); + } + } + + // Open up the data file and mmap it + INFO("Opening data file: {:s}.data", filename); + if ((fd = open((filename + ".data").c_str(), O_RDONLY)) == -1) { + throw std::runtime_error( + fmt::format(fmt("Failed to open file {:s}.data: {:s}."), filename, strerror(errno))); + } + mapped_file = + (uint8_t*)mmap(nullptr, ntime * nfreq * file_frame_size, PROT_READ, MAP_SHARED, fd, 0); + if (mapped_file == MAP_FAILED) + throw std::runtime_error(fmt::format(fmt("Failed to map file {:s}.data to memory: {:s}."), + filename, strerror(errno))); +} + +template +RawReader::~RawReader() { + if (munmap(mapped_file, ntime * nfreq * file_frame_size) == -1) { + // Make sure kotekan is exiting... + FATAL_ERROR("Failed to unmap file {:s}.data: {:s}.", filename, strerror(errno)); + } + + close(fd); +} + +template +void RawReader::main_thread() { + + double start_time, end_time; + frameID frame_id(out_buf); + uint8_t* frame; + + size_t ind = 0, read_ind = 0, file_ind; + + size_t nframe = nfreq * ntime; + + // Calculate the minimum time we should take to read the data to satisfy the + // rate limiting + double min_read_time = + (max_read_rate > 0 ? file_frame_size / (max_read_rate * 1024 * 1024) : 0.0); + DEBUG("Minimum read time per frame {}s", min_read_time); + + readahead_blocks = std::min(nframe, readahead_blocks); + // Initial readahead for frames + for (read_ind = 0; read_ind < readahead_blocks; read_ind++) { + read_ahead(read_ind); + } + + while (!stop_thread && ind < nframe) { + + // Get the start time of the loop for rate limiting + start_time = current_time(); + + // Wait for an empty frame in the output buffer + if ((frame = wait_for_empty_frame(out_buf, unique_name.c_str(), frame_id)) == nullptr) { + break; + } + + // Issue the read ahead request + if (read_ind < (ntime * nfreq)) { + read_ahead(read_ind); + } + + // Get the index into the file + file_ind = position_map(ind); + + // Allocate the metadata space + allocate_new_metadata_object(out_buf, frame_id); + + // Check first byte indicating empty frame + if (*(mapped_file + file_ind * file_frame_size) != 0) { + // Copy the metadata from the file + std::memcpy(out_buf->metadata[frame_id]->metadata, + mapped_file + file_ind * file_frame_size + 1, metadata_size); + + // Copy the data from the file + std::memcpy(frame, mapped_file + file_ind * file_frame_size + metadata_size + 1, + data_size); + } else { + // Create empty frame and set structural metadata + create_empty_frame(frame_id); + } + + // Set the dataset ID to the updated value + auto frame = T(out_buf, frame_id); + dset_id_t& ds_id = frame.dataset_id; + ds_id = get_dataset_state(ds_id); + + // Try and clear out the cached data from the memory map as we don't need it again + if (madvise(mapped_file + file_ind * file_frame_size, file_frame_size, MADV_DONTNEED) == -1) + WARN("madvise failed: {:s}", strerror(errno)); +#ifdef __linux__ + // Try and clear out the cached data from the page cache as we don't need it again + // NOTE: unless we do this in addition to the above madvise the kernel will try and keep as + // much of the file in the page cache as possible and it will fill all the available memory + if (posix_fadvise(fd, file_ind * file_frame_size, file_frame_size, POSIX_FADV_DONTNEED) + == -1) + WARN("fadvise failed: {:s}", strerror(errno)); +#endif + + // Release the frame and advance all the counters + mark_frame_full(out_buf, unique_name.c_str(), frame_id++); + read_ind++; + ind++; + + // Get the end time for the loop and sleep for long enough to satisfy + // the max rate + end_time = current_time(); + double sleep_time_this_frame = min_read_time - (end_time - start_time); + DEBUG("Sleep time {}", sleep_time_this_frame); + if (sleep_time > 0) { + auto ts = double_to_ts(sleep_time_this_frame); + nanosleep(&ts, nullptr); + } + } + + if (sleep_time > 0) { + INFO("Read all data. Sleeping and then exiting kotekan..."); + timespec ts = double_to_ts(sleep_time); + nanosleep(&ts, nullptr); + exit_kotekan(ReturnCode::CLEAN_EXIT); + } else { + INFO("Read all data. Exiting stage, but keeping kotekan alive."); + } +} + +template +dset_id_t RawReader::get_dataset_state(dset_id_t ds_id) { + + // See if we've already processed this ds_id... + auto got_it = ds_in_file.find(ds_id); + + // ... if we have, just return the corresponding output + if (got_it != ds_in_file.end()) + return got_it->second; + + dset_id_t new_id; + + if (!update_dataset_id || ds_id == dset_id_t::null) { + new_id = ds_id; + } else if (local_dm) { + INFO("Registering new dataset with local DM based on {}.", ds_id); + datasetManager& dm = datasetManager::instance(); + new_id = dm.add_dataset(states, ds_id); + } else if (use_comet) { + INFO("Registering new dataset with broker based on {}.", ds_id); + datasetManager& dm = datasetManager::instance(); + new_id = dm.add_dataset(tstate_id, ds_id); + } else { + new_id = static_out_dset_id; + } + + ds_in_file[ds_id] = new_id; + return new_id; +} + +template +void RawReader::read_ahead(int ind) { + + off_t offset = position_map(ind) * file_frame_size; + + if (madvise(mapped_file + offset, file_frame_size, MADV_WILLNEED) == -1) + DEBUG("madvise failed: {:s}", strerror(errno)); +} + +template +int RawReader::position_map(int ind) { + if (chunked) { + // chunked row index + int ri = ind / row_size; + // Special case at edges of time*freq array + int t_width = std::min(ntime - ri * chunk_t, chunk_t); + // chunked column index + int ci = (ind % row_size) / (t_width * chunk_f); + // edges case + int f_width = std::min(nfreq - ci * chunk_f, chunk_f); + // time and frequency indices + // frequency is fastest varying + int fi = ci * chunk_f + ((ind % row_size) % (t_width * chunk_f)) % f_width; + int ti = ri * chunk_t + ((ind % row_size) % (t_width * chunk_f)) / f_width; + + return ti * nfreq + fi; + } + return ind; +} + + +/** + * @class ensureOrdered + * @brief Check frames are coming through in order and reorder them otherwise. + * Not used presently. + */ +class ensureOrdered : public kotekan::Stage { + +public: + ensureOrdered(kotekan::Config& config, const std::string& unique_name, + kotekan::bufferContainer& buffer_container); + + ~ensureOrdered() = default; + + /// Main loop over buffer frames + void main_thread() override; + +private: + Buffer* in_buf; + Buffer* out_buf; + + // Map of buffer frames waiting for their turn + std::map waiting; + size_t max_waiting; + + // time and frequency axes + std::map time_map; + std::map freq_map; + size_t ntime; + size_t nfreq; + + // HDF5 chunk size + std::vector chunk_size; + // size of time dimension of chunk + size_t chunk_t; + // size of frequency dimension of chunk + size_t chunk_f; + bool chunked; + + bool get_dataset_state(dset_id_t ds_id); + + // map from time and freq index to frame index + // RawReader reads chunks with frequency as fastest varying index + // within a chunk, frequency is also the fastest varying index. + inline size_t get_frame_ind(size_t ti, size_t fi) { + size_t ind = 0; + // chunk row and column + size_t row = ti / chunk_t; + size_t col = fi / chunk_f; + // special dimension at array edges + size_t this_chunk_t = chunk_t ? row * chunk_t + chunk_t < ntime : ntime - row * chunk_t; + size_t this_chunk_f = chunk_f ? col * chunk_f + chunk_f < nfreq : nfreq - col * chunk_f; + // number of frames in previous rows + ind += nfreq * chunk_t * row; + // number of frames in chunks in this row + ind += (this_chunk_t * chunk_f) * col; + // within a chunk, frequency is fastest varying + ind += (ti % chunk_t) * this_chunk_f + (fi % chunk_f); + + return ind; + }; +}; + +#endif diff --git a/lib/stages/VisRawReader.cpp b/lib/stages/VisRawReader.cpp new file mode 100644 index 000000000..5d97cdb60 --- /dev/null +++ b/lib/stages/VisRawReader.cpp @@ -0,0 +1,87 @@ +#include "VisRawReader.hpp" + +#include "Config.hpp" // for Config +#include "Stage.hpp" // for Stage +#include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate +#include "bufferContainer.hpp" // for bufferContainer +#include "datasetManager.hpp" // for state_id_t, datasetManager, dset_id_t +#include "datasetState.hpp" // for eigenvalueState, inputState, prodState, stackState +#include "kotekanLogging.hpp" // for DEBUG, WARN +#include "visBuffer.hpp" // for VisFrameView, VisMetadata +#include "visUtil.hpp" // for prod_ctype, rstack_ctype, stack_ctype, input_ctype, frameID + +#include "fmt.hpp" // for format, fmt +#include "json.hpp" // for basic_json<>::object_t, basic_json, json, basic_json<>::v... + +#include // for fill, max +#include // for uint32_t +#include // for size_t +#include // for runtime_error +#include // for pair, move + +using kotekan::bufferContainer; +using kotekan::Config; +using kotekan::Stage; +using nlohmann::json; + +REGISTER_KOTEKAN_STAGE(VisRawReader); + +VisRawReader::VisRawReader(Config& config, const std::string& unique_name, + bufferContainer& buffer_container) : + RawReader(config, unique_name, buffer_container) { + + // Extract data specific indices + _inputs = metadata_json["index_map"]["input"].get>(); + _prods = metadata_json["index_map"]["prod"].get>(); + _ev = metadata_json["index_map"]["ev"].get>(); + if (metadata_json.at("index_map").find("stack") != metadata_json.at("index_map").end()) { + _stack = metadata_json.at("index_map").at("stack").get>(); + _rstack = metadata_json.at("reverse_map").at("stack").get>(); + _num_stack = metadata_json.at("structure").at("num_stack").get(); + } + + // Check metadata is the correct size + if (sizeof(VisMetadata) != metadata_size) { + std::string msg = fmt::format(fmt("Metadata in file {:s} is larger ({:d} bytes) than " + "VisMetadata ({:d} bytes)."), + filename, metadata_size, sizeof(VisMetadata)); + throw std::runtime_error(msg); + } + + // Register a state for the time axis if using comet, or register the replacement dataset ID if + // using + if (update_dataset_id) { + + datasetManager& dm = datasetManager::instance(); + + if (!use_comet) { + // Add data specific states + if (!_stack.empty()) + states.push_back(dm.create_state(_num_stack, std::move(_rstack)).first); + states.push_back(dm.create_state(_inputs).first); + states.push_back(dm.create_state(_ev).first); + states.push_back(dm.create_state(_prods).first); + + // register it as root dataset + static_out_dset_id = dm.add_dataset(states); + + WARN("Updating the dataset IDs without comet is not recommended " + "as it will not preserve dataset ID changes."); + } + } +} + +VisRawReader::~VisRawReader() {} + +void VisRawReader::create_empty_frame(frameID frame_id) { + + // Create frame and set structural metadata + size_t num_vis = _stack.size() > 0 ? _stack.size() : _prods.size(); + + auto frame = + VisFrameView::create_frame_view(out_buf, frame_id, _inputs.size(), num_vis, _ev.size()); + + frame.zero_frame(); + + DEBUG("VisRawReader: Reading empty frame: {:d}", frame_id); +} diff --git a/lib/stages/VisRawReader.hpp b/lib/stages/VisRawReader.hpp new file mode 100644 index 000000000..31ced36b5 --- /dev/null +++ b/lib/stages/VisRawReader.hpp @@ -0,0 +1,77 @@ +/***************************************** +@file +@brief Read visFileRaw data. +- VisRawReader : public RawReader +*****************************************/ +#ifndef _VIS_RAW_READER_HPP +#define _VIS_RAW_READER_HPP + +#include "Config.hpp" // for Config +#include "RawReader.hpp" // for RawReader +#include "bufferContainer.hpp" // for bufferContainer +#include "visBuffer.hpp" +#include "visUtil.hpp" // for frameID, input_ctype, prod_ctype, stack_ctype, rstack_ctype + +#include // for uint32_t +#include // for string +#include // for vector + +/** + * @class VisRawReader + * @brief Stage to read raw visibility data. + * + * This class inherits from the RawReader base class and reads raw visibility data + * @author Richard Shaw, Tristan Pinsonneault-Marotte, Rick Nitsche + */ +class VisRawReader : public RawReader { + +public: + /// default constructor + VisRawReader(kotekan::Config& config, const std::string& unique_name, + kotekan::bufferContainer& buffer_container); + + ~VisRawReader(); + + /** + * @brief Get the products in the file. + **/ + const std::vector& prods() { + return _prods; + } + + /** + * @brief Get the stack in the file. + **/ + const std::vector& stack() { + return _stack; + } + + /** + * @brief Get the inputs in the file. + **/ + const std::vector& inputs() { + return _inputs; + } + + /** + * @brief Get the ev axis in the file. + **/ + const std::vector& ev() { + return _ev; + } + +protected: + // Create an empty frame + void create_empty_frame(frameID frame_id) override; + +private: + // The metadata + std::vector _prods; + std::vector _inputs; + std::vector _stack; + std::vector _rstack; + std::vector _ev; + uint32_t _num_stack; +}; + +#endif diff --git a/lib/stages/visRawReader.cpp b/lib/stages/visRawReader.cpp deleted file mode 100644 index b07134ab3..000000000 --- a/lib/stages/visRawReader.cpp +++ /dev/null @@ -1,580 +0,0 @@ -#include "visRawReader.hpp" - -#include "Config.hpp" // for Config -#include "Hash.hpp" // for Hash, operator<, operator== -#include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate -#include "Telescope.hpp" -#include "buffer.h" // for allocate_new_metadata_object, mark_frame_full, wait_for_e... -#include "bufferContainer.hpp" // for bufferContainer -#include "datasetManager.hpp" // for state_id_t, dset_id_t, datasetManager, DS_UNIQUE_NAME -#include "datasetState.hpp" // for freqState, timeState, eigenvalueState, inputState, metada... -#include "errors.h" // for exit_kotekan, CLEAN_EXIT, ReturnCode -#include "kotekanLogging.hpp" // for INFO, DEBUG, FATAL_ERROR, ERROR, WARN -#include "metadata.h" // for metadataContainer -#include "version.h" // for get_git_commit_hash -#include "visBuffer.hpp" // for VisFrameView, VisMetadata -#include "visUtil.hpp" // for time_ctype, frameID, freq_ctype, prod_ctype, rstack_ctype - -#include "fmt.hpp" // for format, fmt -#include "gsl-lite.hpp" // for span<>::iterator, span -#include "json.hpp" // for basic_json<>::object_t, json, basic_json, basic_json<>::v... - -#include // for fill, min, max -#include // for atomic_bool -#include // for uint32_t, uint8_t -#include // for strerror, memcpy -#include // for __forced_unwind -#include // for errno -#include // for exception -#include // for open, O_RDONLY -#include // for ifstream, ios_base::failure, ios_base, basic_ios, basic_i... -#include // for _Bind_helper<>::type, bind, function -#include // for async, future -#include // for allocator_traits<>::value_type -#include // for match_results<>::_Base_type -#include // for runtime_error, invalid_argument, out_of_range -#include // for madvise, mmap, munmap, MADV_DONTNEED, MADV_WILLNEED, MAP_... -#include // for stat -#include // for system_error -#include // for nanosleep, timespec -#include // for get, make_tuple, tuple -#include // for __decay_and_strip<>::__type -#include // for close, off_t - -using kotekan::bufferContainer; -using kotekan::Config; -using kotekan::Stage; -using nlohmann::json; - -REGISTER_KOTEKAN_STAGE(visRawReader); -REGISTER_KOTEKAN_STAGE(ensureOrdered); - -visRawReader::visRawReader(Config& config, const std::string& unique_name, - bufferContainer& buffer_container) : - Stage(config, unique_name, buffer_container, std::bind(&visRawReader::main_thread, this)) { - - filename = config.get(unique_name, "infile"); - readahead_blocks = config.get(unique_name, "readahead_blocks"); - max_read_rate = config.get_default(unique_name, "max_read_rate", 0.0); - sleep_time = config.get_default(unique_name, "sleep_time", -1); - update_dataset_id = config.get_default(unique_name, "update_dataset_id", true); - use_comet = config.get_default(DS_UNIQUE_NAME, "use_dataset_broker", true); - local_dm = config.get_default(unique_name, "use_local_dataset_man", false); - if (local_dm && use_comet) - FATAL_ERROR("Cannot use local dataset manager and dataset broker together.") - - chunked = config.exists(unique_name, "chunk_size"); - if (chunked) { - chunk_size = config.get>(unique_name, "chunk_size"); - if (chunk_size.size() != 3) - throw std::invalid_argument("Chunk size needs exactly three " - "elements (has " - + std::to_string(chunk_size.size()) + ")."); - chunk_t = chunk_size[2]; - chunk_f = chunk_size[0]; - if (chunk_size[0] < 1 || chunk_size[1] < 1 || chunk_size[2] < 1) - throw std::invalid_argument("visRawReader: config: Chunk size " - "needs to be greater or equal to (1,1,1) (is (" - + std::to_string(chunk_size[0]) + "," - + std::to_string(chunk_size[1]) + "," - + std::to_string(chunk_size[2]) + "))."); - } - - // Get the list of buffers that this stage should connect to - out_buf = get_buffer("out_buf"); - register_producer(out_buf, unique_name.c_str()); - - // Read the metadata - std::string md_filename = (filename + ".meta"); - INFO("Reading metadata file: {:s}", md_filename); - struct stat st; - if (stat(md_filename.c_str(), &st) == -1) - throw std::ios_base::failure( - fmt::format(fmt("visRawReader: Error reading from metadata file: {:s}"), md_filename)); - size_t filesize = st.st_size; - std::vector packed_json(filesize); - - std::ifstream metadata_file(md_filename, std::ios::binary); - if (metadata_file) // only read if no error - metadata_file.read((char*)&packed_json[0], filesize); - if (!metadata_file) // check if open and read successful - throw std::ios_base::failure("visRawReader: Error reading from " - "metadata file: " - + md_filename); - json _t = json::from_msgpack(packed_json); - metadata_file.close(); - - // Extract the attributes and index maps - _metadata = _t["attributes"]; - _times = _t["index_map"]["time"].get>(); - auto freqs = _t["index_map"]["freq"].get>(); - _inputs = _t["index_map"]["input"].get>(); - _prods = _t["index_map"]["prod"].get>(); - _ev = _t["index_map"]["ev"].get>(); - if (_t.at("index_map").find("stack") != _t.at("index_map").end()) { - _stack = _t.at("index_map").at("stack").get>(); - _rstack = _t.at("reverse_map").at("stack").get>(); - _num_stack = _t.at("structure").at("num_stack").get(); - } - - // Match frequencies to IDs in the Telescope... - auto& tel = Telescope::instance(); - std::map inv_freq_map; - - // ... first construct a map of central frequencies to IDs known by the - // telescope object - for (uint32_t id = 0; id < tel.num_freq(); id++) { - inv_freq_map[tel.to_freq(id)] = id; - } - - // ... then use this to match the central frequencies given in the file - for (auto f : freqs) { - - auto it = inv_freq_map.find(f.centre); - - if (it == inv_freq_map.end()) { - FATAL_ERROR("Could not match a frequency ID to channel in file at {} MHz. " - "Check you are specifying the correct telescope.", - f.centre); - return; - } - - DEBUG("restored freq_id for f_centre={:.2f} : {:d}", f.centre, it->second); - _freqs.push_back({it->second, f}); - } - - // check git version tag - // TODO: enforce that they match if build type == "release"? - if (_metadata.at("git_version_tag").get() != std::string(get_git_commit_hash())) - INFO("Git version tags don't match: dataset in file {:s} has tag {:s}, while the local git " - "version tag is {:s}", - filename, _metadata.at("git_version_tag").get(), get_git_commit_hash()); - - // Extract the structure - file_frame_size = _t["structure"]["frame_size"].get(); - metadata_size = _t["structure"]["metadata_size"].get(); - data_size = _t["structure"]["data_size"].get(); - nfreq = _t["structure"]["nfreq"].get(); - ntime = _t["structure"]["ntime"].get(); - - if (chunked) { - // Special case if dimensions less than chunk size - chunk_f = std::min(chunk_f, nfreq); - chunk_t = std::min(chunk_t, ntime); - - // Number of elements in a chunked row - row_size = chunk_t * nfreq; - } - - // Check metadata is the correct size - if (sizeof(VisMetadata) != metadata_size) { - std::string msg = fmt::format(fmt("Metadata in file {:s} is larger ({:d} bytes) than " - "VisMetadata ({:d} bytes)."), - filename, metadata_size, sizeof(VisMetadata)); - throw std::runtime_error(msg); - } - - // Check that buffer is large enough - if ((unsigned int)(out_buf->frame_size) < data_size || out_buf->frame_size < 0) { - std::string msg = - fmt::format(fmt("Data in file {:s} is larger ({:d} bytes) than buffer size " - "({:d} bytes)."), - filename, data_size, out_buf->frame_size); - throw std::runtime_error(msg); - } - - // Register a state for the time axis if using comet, or register the replacement dataset ID if - // using - if (update_dataset_id) { - - datasetManager& dm = datasetManager::instance(); - tstate_id = dm.create_state(_times).first; - - if (!use_comet) { - // Add the states: metadata, time, prod, freq, input, - // eigenvalue and stack. - if (!_stack.empty()) - states.push_back(dm.create_state(_num_stack, std::move(_rstack)).first); - states.push_back(dm.create_state(_inputs).first); - states.push_back(dm.create_state(_ev).first); - states.push_back(dm.create_state(_freqs).first); - states.push_back(dm.create_state(_prods).first); - states.push_back(tstate_id); - states.push_back(dm.create_state(_metadata.at("weight_type"), - _metadata.at("instrument_name"), - _metadata.at("git_version_tag")) - .first); - // register it as root dataset - static_out_dset_id = dm.add_dataset(states); - - WARN("Updating the dataset IDs without comet is not recommended " - "as it will not preserve dataset ID changes."); - } - } - - // Open up the data file and mmap it - INFO("Opening data file: {:s}.data", filename); - if ((fd = open((filename + ".data").c_str(), O_RDONLY)) == -1) { - throw std::runtime_error( - fmt::format(fmt("Failed to open file {:s}.data: {:s}."), filename, strerror(errno))); - } - mapped_file = - (uint8_t*)mmap(nullptr, ntime * nfreq * file_frame_size, PROT_READ, MAP_SHARED, fd, 0); - if (mapped_file == MAP_FAILED) - throw std::runtime_error(fmt::format(fmt("Failed to map file {:s}.data to memory: {:s}."), - filename, strerror(errno))); -} - -visRawReader::~visRawReader() { - if (munmap(mapped_file, ntime * nfreq * file_frame_size) == -1) { - // Make sure kotekan is exiting... - FATAL_ERROR("Failed to unmap file {:s}.data: {:s}.", filename, strerror(errno)); - } - - close(fd); -} - -dset_id_t visRawReader::get_dataset_state(dset_id_t ds_id) { - - // See if we've already processed this ds_id... - auto got_it = ds_in_file.find(ds_id); - - // ... if we have, just return the corresponding output - if (got_it != ds_in_file.end()) - return got_it->second; - - dset_id_t new_id; - - if (!update_dataset_id || ds_id == dset_id_t::null) { - new_id = ds_id; - } else if (local_dm) { - INFO("Registering new dataset with local DM based on {}.", ds_id); - datasetManager& dm = datasetManager::instance(); - new_id = dm.add_dataset(states, ds_id); - } else if (use_comet) { - INFO("Registering new dataset with broker based on {}.", ds_id); - datasetManager& dm = datasetManager::instance(); - new_id = dm.add_dataset(tstate_id, ds_id); - } else { - new_id = static_out_dset_id; - } - - ds_in_file[ds_id] = new_id; - return new_id; -} - -void visRawReader::read_ahead(int ind) { - - off_t offset = position_map(ind) * file_frame_size; - - if (madvise(mapped_file + offset, file_frame_size, MADV_WILLNEED) == -1) - DEBUG("madvise failed: {:s}", strerror(errno)); -} - -int visRawReader::position_map(int ind) { - if (chunked) { - // chunked row index - int ri = ind / row_size; - // Special case at edges of time*freq array - int t_width = std::min(ntime - ri * chunk_t, chunk_t); - // chunked column index - int ci = (ind % row_size) / (t_width * chunk_f); - // edges case - int f_width = std::min(nfreq - ci * chunk_f, chunk_f); - // time and frequency indices - // frequency is fastest varying - int fi = ci * chunk_f + ((ind % row_size) % (t_width * chunk_f)) % f_width; - int ti = ri * chunk_t + ((ind % row_size) % (t_width * chunk_f)) / f_width; - - return ti * nfreq + fi; - } - return ind; -} - -void visRawReader::main_thread() { - - double start_time, end_time; - frameID frame_id(out_buf); - uint8_t* frame; - - size_t ind = 0, read_ind = 0, file_ind; - - size_t nframe = nfreq * ntime; - - // Calculate the minimum time we should take to read the data to satisfy the - // rate limiting - double min_read_time = - (max_read_rate > 0 ? file_frame_size / (max_read_rate * 1024 * 1024) : 0.0); - DEBUG("Minimum read time per frame {}s", min_read_time); - - readahead_blocks = std::min(nframe, readahead_blocks); - // Initial readahead for frames - for (read_ind = 0; read_ind < readahead_blocks; read_ind++) { - read_ahead(read_ind); - } - - while (!stop_thread && ind < nframe) { - - // Get the start time of the loop for rate limiting - start_time = current_time(); - - // Wait for an empty frame in the output buffer - if ((frame = wait_for_empty_frame(out_buf, unique_name.c_str(), frame_id)) == nullptr) { - break; - } - - // Issue the read ahead request - if (read_ind < (ntime * nfreq)) { - read_ahead(read_ind); - } - - // Get the index into the file - file_ind = position_map(ind); - - // Allocate the metadata space - allocate_new_metadata_object(out_buf, frame_id); - - // Check first byte indicating empty frame - if (*(mapped_file + file_ind * file_frame_size) != 0) { - // Copy the metadata from the file - std::memcpy(out_buf->metadata[frame_id]->metadata, - mapped_file + file_ind * file_frame_size + 1, metadata_size); - - // Copy the data from the file - std::memcpy(frame, mapped_file + file_ind * file_frame_size + metadata_size + 1, - data_size); - - } else { - // Create frame and set structural metadata - size_t num_vis = _stack.size() > 0 ? _stack.size() : _prods.size(); - - auto frame = VisFrameView::create_frame_view(out_buf, frame_id, _inputs.size(), num_vis, - _ev.size()); - - // Fill data with zeros - std::fill(frame.vis.begin(), frame.vis.end(), 0.0); - std::fill(frame.weight.begin(), frame.weight.end(), 0.0); - std::fill(frame.eval.begin(), frame.eval.end(), 0.0); - std::fill(frame.evec.begin(), frame.evec.end(), 0.0); - std::fill(frame.gain.begin(), frame.gain.end(), 0.0); - std::fill(frame.flags.begin(), frame.flags.end(), 0.0); - frame.erms = 0; - - // Set non-structural metadata - frame.freq_id = 0; - frame.dataset_id = dset_id_t::null; - frame.time = std::make_tuple(0, timespec{0, 0}); - - // mark frame as empty by ensuring this is 0 - frame.fpga_seq_length = 0; - frame.fpga_seq_total = 0; - - DEBUG("visRawReader: Reading empty frame: {:d}", frame_id); - } - - // Set the dataset ID to the updated value - dset_id_t& ds_id = ((VisMetadata*)(out_buf->metadata[frame_id]->metadata))->dataset_id; - ds_id = get_dataset_state(ds_id); - - // Try and clear out the cached data from the memory map as we don't need it again - if (madvise(mapped_file + file_ind * file_frame_size, file_frame_size, MADV_DONTNEED) == -1) - WARN("madvise failed: {:s}", strerror(errno)); -#ifdef __linux__ - // Try and clear out the cached data from the page cache as we don't need it again - // NOTE: unless we do this in addition to the above madvise the kernel will try and keep as - // much of the file in the page cache as possible and it will fill all the available memory - if (posix_fadvise(fd, file_ind * file_frame_size, file_frame_size, POSIX_FADV_DONTNEED) - == -1) - WARN("fadvise failed: {:s}", strerror(errno)); -#endif - - // Release the frame and advance all the counters - mark_frame_full(out_buf, unique_name.c_str(), frame_id++); - read_ind++; - ind++; - - // Get the end time for the loop and sleep for long enough to satisfy - // the max rate - end_time = current_time(); - double sleep_time_this_frame = min_read_time - (end_time - start_time); - DEBUG("Sleep time {}", sleep_time_this_frame); - if (sleep_time > 0) { - auto ts = double_to_ts(sleep_time_this_frame); - nanosleep(&ts, nullptr); - } - } - - if (sleep_time > 0) { - INFO("Read all data. Sleeping and then exiting kotekan..."); - timespec ts = double_to_ts(sleep_time); - nanosleep(&ts, nullptr); - exit_kotekan(ReturnCode::CLEAN_EXIT); - } else { - INFO("Read all data. Exiting stage, but keeping kotekan alive."); - } -} - -ensureOrdered::ensureOrdered(Config& config, const std::string& unique_name, - bufferContainer& buffer_container) : - Stage(config, unique_name, buffer_container, std::bind(&ensureOrdered::main_thread, this)) { - - max_waiting = config.get_default(unique_name, "max_waiting", 100); - - chunked = config.exists(unique_name, "chunk_size"); - if (chunked) { - chunk_size = config.get>(unique_name, "chunk_size"); - if (chunk_size.size() != 3) { - FATAL_ERROR("Chunk size needs exactly three elements (got {:d}).", chunk_size.size()); - return; - } - chunk_t = chunk_size[2]; - chunk_f = chunk_size[0]; - if (chunk_size[0] < 1 || chunk_size[1] < 1 || chunk_size[2] < 1) { - FATAL_ERROR("Chunk dimensions need to be >= 1 (got ({:d}, {:d}, {:d}).", chunk_size[0], - chunk_size[1], chunk_size[2]); - return; - } - } - - // Get the list of buffers that this stage should connect to - out_buf = get_buffer("out_buf"); - register_producer(out_buf, unique_name.c_str()); - in_buf = get_buffer("in_buf"); - register_consumer(in_buf, unique_name.c_str()); -} - -bool ensureOrdered::get_dataset_state(dset_id_t ds_id) { - - datasetManager& dm = datasetManager::instance(); - - // Get the states synchronously. - auto tstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); - auto fstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); - - const timeState* tstate = tstate_fut.get(); - const freqState* fstate = fstate_fut.get(); - - if (tstate == nullptr || fstate == nullptr) { - ERROR("One of time or freq dataset states is null for dataset {}.", ds_id); - return false; - } - - auto times = tstate->get_times(); - auto freq_pairs = fstate->get_freqs(); - - // construct map of times to axis index - for (size_t i = 0; i < times.size(); i++) { - time_map.insert({times.at(i), i}); - } - // construct map of freq_ind to axis index - for (size_t i = 0; i < freq_pairs.size(); i++) { - freq_map.insert({freq_pairs.at(i).first, i}); - } - - return true; -} - -void ensureOrdered::main_thread() { - - // The index to the current buffer frame - frameID frame_id(in_buf); - frameID output_frame_id(out_buf); - // The index of the current frame relative to the first frame - size_t output_ind = 0; - - // Frequency and time indices - size_t fi, ti; - time_ctype t; - - // The dataset ID we read from the frame - dset_id_t ds_id; - - // Get axes from dataset state - uint32_t first_ind = 0; - while (true) { - // Wait for a frame in the input buffer in order to get the dataset ID - if ((wait_for_full_frame(in_buf, unique_name.c_str(), first_ind)) == nullptr) { - return; - } - auto frame = VisFrameView(in_buf, first_ind); - if (frame.fpga_seq_length == 0) { - INFO("Got empty frame ({:d}).", first_ind); - first_ind++; - } else { - ds_id = frame.dataset_id; - break; - } - } - - auto future_ds_state = std::async(&ensureOrdered::get_dataset_state, this, ds_id); - if (!future_ds_state.get()) { - FATAL_ERROR("Couldn't find ancestor of dataset {}. " - "Make sure there is a stage upstream in the config, that adds the dataset " - "states.\nExiting...", - ds_id); - return; - } - - // main loop: - while (!stop_thread) { - // Wait for a full frame in the input buffer - if ((wait_for_full_frame(in_buf, unique_name.c_str(), frame_id)) == nullptr) { - break; - } - auto frame = VisFrameView(in_buf, frame_id); - - // Figure out the ordered index of this frame - t = {std::get<0>(frame.time), ts_to_double(std::get<1>(frame.time))}; - ti = time_map.at(t); - fi = freq_map.at(frame.freq_id); - size_t ordered_ind = get_frame_ind(ti, fi); - - // Check if this is the index we are ready to send - if (ordered_ind == output_ind) { - // copy frame into output buffer - if (wait_for_empty_frame(out_buf, unique_name.c_str(), output_frame_id) == nullptr) { - return; - } - auto output_frame = - VisFrameView::copy_frame(in_buf, frame_id, out_buf, output_frame_id); - mark_frame_full(out_buf, unique_name.c_str(), output_frame_id++); - - // release input frame - mark_frame_empty(in_buf, unique_name.c_str(), frame_id++); - - // increment output index - output_ind++; - } else if (waiting.size() <= max_waiting) { - INFO("Frame {:d} arrived out of order. Expected {:d}. Adding it to waiting buffer.", - ordered_ind, output_ind); - // Add to waiting frames and move to next (without marking empty!) - waiting.insert({ordered_ind, (int)frame_id}); - frame_id++; - } else { - FATAL_ERROR("Number of frames arriving out of order exceeded maximum buffer size."); - return; - } - - // Check if any of the waiting frames are ready - auto ready = waiting.find(output_ind); - while (ready != waiting.end()) { - // remove this index from waiting map - uint32_t waiting_id = ready->second; - waiting.erase(ready); - INFO("Frame {:d} is ready to be sent. Releasing buffer.", output_ind); - // copy frame into output buffer - auto past_frame = VisFrameView(in_buf, waiting_id); - if (wait_for_empty_frame(out_buf, unique_name.c_str(), output_frame_id) == nullptr) { - return; - } - auto output_frame = - VisFrameView::copy_frame(in_buf, waiting_id, out_buf, output_frame_id); - mark_frame_full(out_buf, unique_name.c_str(), output_frame_id++); - - mark_frame_empty(in_buf, unique_name.c_str(), waiting_id); - output_ind++; - - ready = waiting.find(output_ind); - } - } -} diff --git a/lib/stages/visRawReader.hpp b/lib/stages/visRawReader.hpp deleted file mode 100644 index f5c075916..000000000 --- a/lib/stages/visRawReader.hpp +++ /dev/null @@ -1,287 +0,0 @@ -/***************************************** -@file -@brief Read visFileRaw data. -- visRawReader : public kotekan::Stage -*****************************************/ -#ifndef _VIS_RAW_READER_HPP -#define _VIS_RAW_READER_HPP - -#include "Config.hpp" -#include "Stage.hpp" // for Stage -#include "buffer.h" -#include "bufferContainer.hpp" -#include "datasetManager.hpp" // for dset_id_t -#include "visUtil.hpp" // for freq_ctype (ptr only), input_ctype, prod_ctype, rstack_ctype - -#include "json.hpp" // for json - -#include // for map -#include // for size_t -#include // for uint32_t, uint8_t -#include // for string -#include // for pair -#include // for vector - -/** - * @class visRawReader - * @brief Read and stream a raw visibility file. - * - * This will divide the file up into time-frequency chunks of set size and - * stream out the frames with time as the *fastest* index. The dataset ID - * will be restored from the dataset broker if `use_comet` is set. Otherwise - * a new dataset will be created and the original ID stored in the frames - * will be lost. - * - * @par Buffers - * @buffer out_buf The data read from the raw file. - * @buffer_format VisBuffer structured - * @buffer_metadata VisMetadata - * - * @conf readahead_blocks Int. Number of blocks to advise OS to read ahead - * of current read. - * @conf chunk_size Array of [int, int, int]. Read chunk size (freq, - * prod, time). If not specified will read file - * contiguously. - * @conf infile String. Path to the (data-meta-pair of) files to - * read (e.g. "/path/to/0000_000", without .data or - * .meta). - * @conf max_read_rate Float. Maximum read rate for the process in MB/s. - * If the value is zero (default), then no rate - * limiting is applied. - * @conf sleep_time Float. After the data is read pause this long in - * seconds before sending shutdown. If < 0, never - * send a shutdown signal. Default is -1. - * @conf update_dataset_id Bool. Update the dataset ID with information about the - file, for example which time samples does it contain. - * @conf use_dataset_broker Bool. Restore dataset ID from dataset broker (i.e. comet). - * Should be disabled only for testing. Default is true. - * @conf use_local_dataset_man Bool. Instead of using comet, register metadata on top - * of dataset ID in the file. Should only be used for - * testing or if original dataset IDs can be lost. - * Default is False. - * - * @author Richard Shaw, Tristan Pinsonneault-Marotte, Rick Nitsche - */ -class visRawReader : public kotekan::Stage { - -public: - /// default constructor - visRawReader(kotekan::Config& config, const std::string& unique_name, - kotekan::bufferContainer& buffer_container); - - ~visRawReader(); - - /// Main loop over buffer frames - void main_thread() override; - - /** - * @brief Get the times in the file. - **/ - const std::vector& times() { - return _times; - } - - /** - * @brief Get the frequencies in the file. - **/ - const std::vector>& freqs() { - return _freqs; - } - - /** - * @brief Get the products in the file. - **/ - const std::vector& prods() { - return _prods; - } - - /** - * @brief Get the stack in the file. - **/ - const std::vector& stack() { - return _stack; - } - - /** - * @brief Get the inputs in the file. - **/ - const std::vector& inputs() { - return _inputs; - } - - /** - * @brief Get the ev axis in the file. - **/ - const std::vector& ev() { - return _ev; - } - - /** - * @brief Get the metadata saved into the file. - **/ - const nlohmann::json& metadata() { - return _metadata; - } - -private: - /** - * @brief Get the new dataset ID. - * - * If not using change the ID this just returns its input ID. If using the - * broker, this will simply append a timeState to the current state. - * Otherwise, we just use a static dataset_id constructed from the file - * metadata. - * - * @param ds_id The ID of the read frame. - * @returns The replacement ID - */ - dset_id_t get_dataset_state(dset_id_t ds_id); - - /** - * @brief Read the next frame. - * - * The exact frame read is dependent on whether the reads are time ordered - * or not. - * - * @param ind The frame index to read. - **/ - void read_ahead(int ind); - - /** - * @brief Map the index into a frame position in the file. - * - * The exact frame read dependent on whether the reads are time ordered - * or not. - * - * @param ind The frame index. - * @returns The frame index into the file. - **/ - int position_map(int ind); - - Buffer* out_buf; - - // The metadata - nlohmann::json _metadata; - std::vector _times; - std::vector> _freqs; - std::vector _prods; - std::vector _inputs; - std::vector _stack; - std::vector _rstack; - std::vector _ev; - uint32_t _num_stack; - - // whether to read in chunks - bool chunked; - - // whether to update the dataset ID with info about the file - bool update_dataset_id; - - // whether to use comet to track dataset IDs - bool use_comet; - - // whether to use the local dataset manager - bool local_dm; - - // Read chunk size (freq, prod, time) - std::vector chunk_size; - // time chunk size - size_t chunk_t; - // freq chunk size - size_t chunk_f; - - // Number of elements in a chunked row - size_t row_size; - - // the input file - std::string filename; - int fd; - uint8_t* mapped_file; - - size_t file_frame_size, metadata_size, data_size, nfreq, ntime; - - // Number of blocks to read ahead while reading from disk - size_t readahead_blocks; - - // Dataset states constructed from metadata - std::vector states; - - // Dataset ID to assign to output frames if not using comet - dset_id_t static_out_dset_id; - - // the dataset state for the time axis - state_id_t tstate_id; - - // Map input to output dataset IDs for quick access - std::map ds_in_file; - - // The read rate - double max_read_rate; - - // Sleep time after reading - double sleep_time; -}; - -/** - * @class ensureOrdered - * @brief Check frames are coming through in order and reorder them otherwise. - * Not used presently. - */ -class ensureOrdered : public kotekan::Stage { - -public: - ensureOrdered(kotekan::Config& config, const std::string& unique_name, - kotekan::bufferContainer& buffer_container); - - ~ensureOrdered() = default; - - /// Main loop over buffer frames - void main_thread() override; - -private: - Buffer* in_buf; - Buffer* out_buf; - - // Map of buffer frames waiting for their turn - std::map waiting; - size_t max_waiting; - - // time and frequency axes - std::map time_map; - std::map freq_map; - size_t ntime; - size_t nfreq; - - // HDF5 chunk size - std::vector chunk_size; - // size of time dimension of chunk - size_t chunk_t; - // size of frequency dimension of chunk - size_t chunk_f; - bool chunked; - - bool get_dataset_state(dset_id_t ds_id); - - // map from time and freq index to frame index - // visRawReader reads chunks with frequency as fastest varying index - // within a chunk, frequency is also the fastest varying index. - inline size_t get_frame_ind(size_t ti, size_t fi) { - size_t ind = 0; - // chunk row and column - size_t row = ti / chunk_t; - size_t col = fi / chunk_f; - // special dimension at array edges - size_t this_chunk_t = chunk_t ? row * chunk_t + chunk_t < ntime : ntime - row * chunk_t; - size_t this_chunk_f = chunk_f ? col * chunk_f + chunk_f < nfreq : nfreq - col * chunk_f; - // number of frames in previous rows - ind += nfreq * chunk_t * row; - // number of frames in chunks in this row - ind += (this_chunk_t * chunk_f) * col; - // within a chunk, frequency is fastest varying - ind += (ti % chunk_t) * this_chunk_f + (fi % chunk_f); - - return ind; - }; -}; - -#endif diff --git a/lib/utils/FrameView.hpp b/lib/utils/FrameView.hpp index 4b50eff2c..dab389166 100644 --- a/lib/utils/FrameView.hpp +++ b/lib/utils/FrameView.hpp @@ -76,6 +76,7 @@ class FrameView { static void copy_frame(Buffer* buf_src, int frame_id_src, Buffer* buf_dest, int frame_id_dest); virtual size_t data_size() const = 0; + virtual void zero_frame() = 0; protected: // References to the buffer and metadata we are viewing diff --git a/lib/utils/HFBFrameView.cpp b/lib/utils/HFBFrameView.cpp index afb20037f..918db7d2d 100644 --- a/lib/utils/HFBFrameView.cpp +++ b/lib/utils/HFBFrameView.cpp @@ -16,6 +16,7 @@ #include // for match_results<>::_Base_type #include // for set #include // for runtime_error +#include // for memset #include // for tuple, make_tuple #include // for vector @@ -172,3 +173,18 @@ HFBFrameView HFBFrameView::create_frame_view(Buffer* buf, const uint32_t index, size_t HFBFrameView::data_size() const { return buffer_layout.first; } + +void HFBFrameView::zero_frame() { + + // Fill data with zeros + std::memset(_frame, 0, data_size()); + + // Set non-structural metadata + freq_id = 0; + dataset_id = dset_id_t::null; + time = timespec{0, 0}; + + // mark frame as empty by ensuring this is 0 + fpga_seq_length = 0; + fpga_seq_total = 0; +} diff --git a/lib/utils/HFBFrameView.hpp b/lib/utils/HFBFrameView.hpp index 30d31a300..55e183f40 100644 --- a/lib/utils/HFBFrameView.hpp +++ b/lib/utils/HFBFrameView.hpp @@ -112,6 +112,8 @@ class HFBFrameView : public FrameView { size_t data_size() const override; + void zero_frame() override; + /** * @brief Return a summary of the hyper fine beam buffer contents. * diff --git a/lib/utils/visBuffer.cpp b/lib/utils/visBuffer.cpp index bc3670aca..6966ce80b 100644 --- a/lib/utils/visBuffer.cpp +++ b/lib/utils/visBuffer.cpp @@ -17,6 +17,7 @@ #include // for match_results<>::_Base_type #include // for set #include // for runtime_error +#include // for memset #include // for TIMEVAL_TO_TIMESPEC #include // for __decay_and_strip<>::__type #include // for vector @@ -254,3 +255,19 @@ VisFrameView VisFrameView::create_frame_view(Buffer* buf, const uint32_t index, size_t VisFrameView::data_size() const { return buffer_layout.first; } + +void VisFrameView::zero_frame() { + + // Fill data with zeros + std::memset(_frame, 0, data_size()); + erms = 0; + + // Set non-structural metadata + freq_id = 0; + dataset_id = dset_id_t::null; + time = std::make_tuple(0, timespec{0, 0}); + + // mark frame as empty by ensuring this is 0 + fpga_seq_length = 0; + fpga_seq_total = 0; +} diff --git a/lib/utils/visBuffer.hpp b/lib/utils/visBuffer.hpp index 6ea5c142f..dd65149b3 100644 --- a/lib/utils/visBuffer.hpp +++ b/lib/utils/visBuffer.hpp @@ -159,6 +159,8 @@ class VisFrameView : public FrameView { size_t data_size() const override; + void zero_frame() override; + /** * @brief Return a summary of the visibility buffer contents. * diff --git a/python/kotekan/runner.py b/python/kotekan/runner.py index 74f470df9..97e9bfd37 100644 --- a/python/kotekan/runner.py +++ b/python/kotekan/runner.py @@ -835,7 +835,7 @@ def __init__(self, infile, chunk_size): } stage_config = { - "kotekan_stage": "visRawReader", + "kotekan_stage": "VisRawReader", "infile": infile, "out_buf": self.name, "chunk_size": chunk_size, From f69052a32e7865118fc662eeecd1715933f8793b Mon Sep 17 00:00:00 2001 From: james-s-willis <36906682+james-s-willis@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:06:07 -0500 Subject: [PATCH 02/15] Templatise FreqSubset and refactor HFBAccumulate (#908) * Templatise FreqSubset using VisFrameView and HFBFrameView. Removed old HFB/VisFreqSubset stages. * Moved variables from internalState to HFBAccumulate class. * Use span from HFBFrameView instead of a raw pointer when integrating frames. --- lib/stages/CMakeLists.txt | 2 - lib/stages/FreqSubset.cpp | 48 ++++++++++++----- lib/stages/FreqSubset.hpp | 47 +++++----------- lib/stages/HFBAccumulate.cpp | 102 +++++++++++++++++------------------ lib/stages/HFBAccumulate.hpp | 61 +++++++-------------- lib/stages/HFBFreqSubset.cpp | 45 ---------------- lib/stages/HFBFreqSubset.hpp | 43 --------------- lib/stages/VisFreqSubset.cpp | 45 ---------------- lib/stages/VisFreqSubset.hpp | 43 --------------- 9 files changed, 115 insertions(+), 321 deletions(-) delete mode 100644 lib/stages/HFBFreqSubset.cpp delete mode 100644 lib/stages/HFBFreqSubset.hpp delete mode 100644 lib/stages/VisFreqSubset.cpp delete mode 100644 lib/stages/VisFreqSubset.hpp diff --git a/lib/stages/CMakeLists.txt b/lib/stages/CMakeLists.txt index ff117d2b2..5c7c045cc 100644 --- a/lib/stages/CMakeLists.txt +++ b/lib/stages/CMakeLists.txt @@ -27,8 +27,6 @@ add_library( simpleAutocorr.cpp freqSplit.cpp FreqSubset.cpp - VisFreqSubset.cpp - HFBFreqSubset.cpp prodSubset.cpp countCheck.cpp visAccumulate.cpp diff --git a/lib/stages/FreqSubset.cpp b/lib/stages/FreqSubset.cpp index 6e3a9d74b..695fd7691 100644 --- a/lib/stages/FreqSubset.cpp +++ b/lib/stages/FreqSubset.cpp @@ -1,16 +1,18 @@ #include "FreqSubset.hpp" #include "Config.hpp" // for Config +#include "HFBFrameView.hpp" // for HFBFrameView #include "Hash.hpp" // for operator< +#include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate #include "buffer.h" // for mark_frame_empty, mark_frame_full, register_consumer, reg... #include "bufferContainer.hpp" // for bufferContainer #include "datasetManager.hpp" // for dset_id_t, state_id_t, datasetManager #include "datasetState.hpp" // for freqState #include "kotekanLogging.hpp" // for FATAL_ERROR -#include "visUtil.hpp" // for freq_ctype, frameID, modulo +#include "visBuffer.hpp" // for VisFrameView +#include "visUtil.hpp" // for frameID, freq_ctype, modulo #include // for find, max -#include // for atomic_bool #include // for uint32_t #include // for __forced_unwind #include // for exception @@ -25,9 +27,15 @@ using kotekan::bufferContainer; using kotekan::Config; using kotekan::Stage; +using VisFreqSubset = FreqSubset; +using HFBFreqSubset = FreqSubset; -FreqSubset::FreqSubset(Config& config, const std::string& unique_name, - bufferContainer& buffer_container) : +REGISTER_KOTEKAN_STAGE(VisFreqSubset); +REGISTER_KOTEKAN_STAGE(HFBFreqSubset); + +template +FreqSubset::FreqSubset(Config& config, const std::string& unique_name, + bufferContainer& buffer_container) : Stage(config, unique_name, buffer_container, std::bind(&FreqSubset::main_thread, this)) { // Get list of frequencies to subset from config @@ -42,7 +50,8 @@ FreqSubset::FreqSubset(Config& config, const std::string& unique_name, register_producer(out_buf, unique_name.c_str()); } -void FreqSubset::change_dataset_state(dset_id_t input_dset_id) { +template +void FreqSubset::change_dataset_state(dset_id_t input_dset_id) { auto& dm = datasetManager::instance(); auto fprint = dm.fingerprint(input_dset_id, {"frequencies"}); @@ -81,11 +90,14 @@ void FreqSubset::change_dataset_state(dset_id_t input_dset_id) { dset_id_map[input_dset_id] = dm.add_dataset(states_map.at(fprint), input_dset_id); } -void FreqSubset::main_thread() { +template +void FreqSubset::main_thread() { frameID input_frame_id(in_buf); frameID output_frame_id(out_buf); + std::future change_dset_fut; + while (!stop_thread) { // Wait for the input buffer to be filled with data @@ -93,16 +105,18 @@ void FreqSubset::main_thread() { break; } - // Get dataset ID and freq ID from input frame - std::pair frame_data = get_frame_data(input_frame_id); - dset_id_t input_dataset_id = frame_data.first; - uint32_t freq = frame_data.second; + // Create view to input frame + auto input_frame = T(in_buf, input_frame_id); // check if the input dataset has changed - if (dset_id_map.count(input_dataset_id) == 0) { - change_dset_fut = std::async(&FreqSubset::change_dataset_state, this, input_dataset_id); + if (dset_id_map.count(input_frame.dataset_id) == 0) { + change_dset_fut = + std::async(&FreqSubset::change_dataset_state, this, input_frame.dataset_id); } + // frequency index of this frame + uint32_t freq = input_frame.freq_id; + // If this frame is part of subset // TODO: Apparently std::set can be used to speed up this search if (std::find(_subset_list.begin(), _subset_list.end(), freq) != _subset_list.end()) { @@ -112,8 +126,14 @@ void FreqSubset::main_thread() { break; } - // Copy dataset ID to output frame - copy_dataset_id(input_dataset_id, input_frame_id, output_frame_id); + // Copy frame and create view + auto output_frame = T::copy_frame(in_buf, input_frame_id, out_buf, output_frame_id); + + // Wait for the dataset ID for the outgoing frame + if (change_dset_fut.valid()) + change_dset_fut.wait(); + + output_frame.dataset_id = dset_id_map.at(input_frame.dataset_id); // Mark the output buffer and move on mark_frame_full(out_buf, unique_name.c_str(), output_frame_id++); diff --git a/lib/stages/FreqSubset.hpp b/lib/stages/FreqSubset.hpp index fdfe1e462..8e5e497dd 100644 --- a/lib/stages/FreqSubset.hpp +++ b/lib/stages/FreqSubset.hpp @@ -11,40 +11,33 @@ #include "buffer.h" // for Buffer #include "bufferContainer.hpp" // for bufferContainer #include "datasetManager.hpp" // for dset_id_t, state_id_t, fingerprint_t -#include "visUtil.hpp" // for frameID -#include // for future #include // for map #include // for uint32_t #include // for string -#include // for pair #include // for vector /** * @class FreqSubset - * @brief Base class to output a buffer stream with a subset of the input frequencies. + * @brief Outputs a Buffer stream with a subset of the input frequencies. * - * All classes which inherit from this should provide the following API: - * - * copy_dataset_id(dset_id_t dataset_id, frameID input_frame_id, frameID output_frame_id); - * get_frame_data(frameID input_frame_id); - * - * This task takes data coming out of a buffer stream and selects a subset of + * This task takes data coming out of a Buffer stream and selects a subset of * frequencies to be passed on to the output buffer. * * @par Buffers * @buffer in_buf The original buffer with all frequencies - * @buffer_format VisBuffer/HFBBuffer structured - * @buffer_metadata VisMetadata/HFBMetadata + * @buffer_format Vis/HFBBuffer structured + * @buffer_metadata Vis/HFBMetadata * @buffer out_buf The buffer containing the subset of frequencies - * @buffer_format VisBuffer/HFBBuffer structured - * @buffer_metadata VisMetadata/HFBMetadata + * @buffer_format Vis/HFBBuffer structured + * @buffer_metadata Vis/HFBMetadata * * @conf subset_list Vector of Int. The list of frequencies that go * in the subset. * * @author Mateus Fandino */ +template class FreqSubset : public kotekan::Stage { public: @@ -55,32 +48,20 @@ class FreqSubset : public kotekan::Stage { /// Main loop for the stage void main_thread() override; -protected: - /// Output buffer with subset of frequencies - Buffer* out_buf; - /// Input buffer with all frequencies - Buffer* in_buf; - - std::future change_dset_fut; - - // Map for determining the dataset ID to use - std::map dset_id_map; - private: - // Copy dataset ID from input to output frame - virtual void copy_dataset_id(dset_id_t dataset_id, frameID input_frame_id, - frameID output_frame_id) = 0; - - // Get dataset ID and frequency ID from frame - virtual std::pair get_frame_data(frameID input_frame_id) = 0; - /// adds state and dataset and gets a new output dataset ID from manager void change_dataset_state(dset_id_t input_dset_id); // List of frequencies for the subset std::vector _subset_list; - // Map for determining the dataset ID to use + /// Output buffer with subset of frequencies + Buffer* out_buf; + /// Input buffer with all frequencies + Buffer* in_buf; + + // Maps for determining the dataset ID to use + std::map dset_id_map; std::map states_map; }; #endif diff --git a/lib/stages/HFBAccumulate.cpp b/lib/stages/HFBAccumulate.cpp index e62f4c78d..b376dcef7 100644 --- a/lib/stages/HFBAccumulate.cpp +++ b/lib/stages/HFBAccumulate.cpp @@ -38,33 +38,31 @@ REGISTER_KOTEKAN_STAGE(HFBAccumulate); HFBAccumulate::HFBAccumulate(Config& config_, const std::string& unique_name, bufferContainer& buffer_container) : - Stage(config_, unique_name, buffer_container, std::bind(&HFBAccumulate::main_thread, this)) { + Stage(config_, unique_name, buffer_container, std::bind(&HFBAccumulate::main_thread, this)), + in_buf(get_buffer("hfb_input_buf")), + cls_buf(get_buffer("compressed_lost_samples_buf")), + out_buf(get_buffer("hfb_output_buf")), + _num_frames_to_integrate( + config.get_default(unique_name, "num_frames_to_integrate", 80)), + _num_frb_total_beams(config.get(unique_name, "num_frb_total_beams")), + _factor_upchan(config.get(unique_name, "factor_upchan")), + _samples_per_data_set(config.get(unique_name, "samples_per_data_set")), + _good_samples_threshold(config.get(unique_name, "good_samples_threshold")) { - auto& tel = Telescope::instance(); - - // Apply config. - _num_frb_total_beams = config.get(unique_name, "num_frb_total_beams"); - _num_frames_to_integrate = - config.get_default(unique_name, "num_frames_to_integrate", 80); - _factor_upchan = config.get(unique_name, "factor_upchan"); - _samples_per_data_set = config.get(unique_name, "samples_per_data_set"); - _good_samples_threshold = config.get(unique_name, "good_samples_threshold"); - - in_buf = get_buffer("hfb_input_buf"); register_consumer(in_buf, unique_name.c_str()); - - cls_buf = get_buffer("compressed_lost_samples_buf"); register_consumer(cls_buf, unique_name.c_str()); - - out_buf = get_buffer("hfb_output_buf"); register_producer(out_buf, unique_name.c_str()); + hfb1.resize(_num_frb_total_beams * _factor_upchan, 0.0); + hfb2.resize(_num_frb_total_beams * _factor_upchan, 0.0); + // weight calculation is hardcoded, so is the weight type name const std::string weight_type = "inverse_var"; const std::string git_tag = get_git_commit_hash(); const std::string instrument_name = config.get_default(unique_name, "instrument_name", "chime"); + auto& tel = Telescope::instance(); std::vector freq_ids; // Get the frequency IDs that are on this stream, check the config or just @@ -93,23 +91,21 @@ HFBAccumulate::HFBAccumulate(Config& config_, const std::string& unique_name, HFBAccumulate::~HFBAccumulate() {} -void HFBAccumulate::init_first_frame(float* input_data, float* sum_data, - const uint32_t in_frame_id) { +void HFBAccumulate::init_first_frame(float* input_data, const uint32_t in_frame_id) { int64_t fpga_seq_num_start = fpga_seq_num_end - (_num_frames_to_integrate - 1) * _samples_per_data_set; - memcpy(sum_data, input_data, _num_frb_total_beams * _factor_upchan * sizeof(float)); + memcpy(out_hfb.data(), input_data, _num_frb_total_beams * _factor_upchan * sizeof(float)); total_lost_timesamples += get_fpga_seq_num(in_buf, in_frame_id) - fpga_seq_num_start; // Get the first FPGA sequence no. to check for missing frames fpga_seq_num = get_fpga_seq_num(in_buf, in_frame_id); frame++; - DEBUG("\nInit frame. fpga_seq_start: {:d}. sum_data[0]: {:f}", - get_fpga_seq_num(in_buf, in_frame_id), sum_data[0]); + DEBUG("\nInit frame. fpga_seq_start: {:d}. out_hfb[0]: {:f}", + get_fpga_seq_num(in_buf, in_frame_id), out_hfb[0]); } -void HFBAccumulate::integrate_frame(float* input_data, float* sum_data, - const uint32_t in_frame_id) { +void HFBAccumulate::integrate_frame(float* input_data, const uint32_t in_frame_id) { frame++; fpga_seq_num += _samples_per_data_set; total_lost_timesamples += get_fpga_seq_num(in_buf, in_frame_id) - fpga_seq_num; @@ -117,19 +113,19 @@ void HFBAccumulate::integrate_frame(float* input_data, float* sum_data, // Integrates data from the input buffer to the output buffer. for (uint32_t i = 0; i < _num_frb_total_beams * _factor_upchan; i++) { - sum_data[i] += input_data[i]; + out_hfb[i] += input_data[i]; } - DEBUG2("\nIntegrate frame {:d}, total_lost_timesamples: {:d}, sum_data[0]: {:f}\n", frame, - total_lost_timesamples, sum_data[0]); + DEBUG2("\nIntegrate frame {:d}, total_lost_timesamples: {:d}, out_hfb[0]: {:f}\n", frame, + total_lost_timesamples, out_hfb[0]); } -void HFBAccumulate::normalise_frame(float* sum_data, const uint32_t in_frame_id) { +void HFBAccumulate::normalise_frame(const uint32_t in_frame_id) { const float normalise_frac = (float)1.f / (total_timesamples - total_lost_timesamples); for (uint32_t i = 0; i < _num_frb_total_beams * _factor_upchan; i++) { - sum_data[i] *= normalise_frac; + out_hfb[i] *= normalise_frac; } DEBUG("Integration completed with {:d} lost samples (~{}%). normalise_frac: {}", @@ -149,7 +145,6 @@ void HFBAccumulate::main_thread() { // Temporary arrays for storing intermediates std::vector hfb_even(_num_frb_total_beams * _factor_upchan); int32_t samples_even = 0; - internalState dset = internalState(_num_frb_total_beams, _factor_upchan); auto& tel = Telescope::instance(); @@ -213,7 +208,7 @@ void HFBAccumulate::main_thread() { auto out_frame = HFBFrameView(out_buf, out_frame_id); - float* sum_data = (float*)out_frame.hfb.data(); + out_hfb = out_frame.hfb; float* input_data = (float*)in_buf->frames[in_frame_id]; // Find where the end of the integration is @@ -246,7 +241,7 @@ void HFBAccumulate::main_thread() { // multiplications // Perform primary accumulation (assume that the weight is one) for (size_t i = 0; i < _num_frb_total_beams * _factor_upchan; i++) { - dset.hfb1[i] += input[i]; + hfb1[i] += input[i]; } // We are calculating the weights by differencing even and odd samples. @@ -264,15 +259,15 @@ void HFBAccumulate::main_thread() { else { for (size_t i = 0; i < _num_frb_total_beams * _factor_upchan; i++) { float d = input[i] - hfb_even[i]; - dset.hfb2[i] += d * d; + hfb2[i] += d * d; } - DEBUG2("hfb2[{}]: {}, input[0]: {}, hfb_even[0]: {}", 0, dset.hfb2[0], input[0], + DEBUG2("hfb2[{}]: {}, input[0]: {}, hfb_even[0]: {}", 0, hfb2[0], input[0], hfb_even[0]); // Accumulate the squared samples difference which we need for // debiasing the variance estimate float samples_diff = samples_in_frame - samples_even; - dset.weight_diff_sum += samples_diff * samples_diff; + weight_diff_sum += samples_diff * samples_diff; } // When all frames have been integrated output the result @@ -287,7 +282,7 @@ void HFBAccumulate::main_thread() { (float)(total_timesamples - total_lost_timesamples) / total_timesamples; // Normalise data - normalise_frame(sum_data, in_frame_id); + normalise_frame(in_frame_id); // Only output integration if there are enough good samples if (good_samples_frac >= _good_samples_threshold) { @@ -307,10 +302,10 @@ void HFBAccumulate::main_thread() { float sample_weight_total = total_timesamples - total_lost_timesamples; // Debias the weights estimate, by subtracting out the bias estimation - float w_debias = dset.weight_diff_sum / pow(sample_weight_total, 2); + float w_debias = weight_diff_sum / pow(sample_weight_total, 2); for (size_t i = 0; i < _num_frb_total_beams * _factor_upchan; i++) { - float d = dset.hfb1[i]; - dset.hfb2[i] -= w_debias * (d * d); + float d = hfb1[i]; + hfb2[i] -= w_debias * (d * d); } // Determine the weighting factors (if weight is zero we should just @@ -319,7 +314,7 @@ void HFBAccumulate::main_thread() { // Unpack and invert the weights for (uint32_t i = 0; i < _num_frb_total_beams * _factor_upchan; i++) { - float t = dset.hfb2[i]; + float t = hfb2[i]; out_frame.weight[i] = w * w / t; } @@ -337,7 +332,12 @@ void HFBAccumulate::main_thread() { if (wait_for_empty_frame(out_buf, unique_name.c_str(), out_frame_id) == nullptr) return; - sum_data = (float*)out_buf->frames[out_frame_id]; + // Create new metadata for new output frame + allocate_new_metadata_object(out_buf, out_frame_id); + HFBFrameView::set_metadata(out_buf, out_frame_id, _num_frb_total_beams, + _factor_upchan); + + out_hfb = HFBFrameView(out_buf, out_frame_id).hfb; } else { DEBUG("Integration discarded. Too many lost samples."); } @@ -348,17 +348,17 @@ void HFBAccumulate::main_thread() { // Already started next integration if (fpga_seq_num > fpga_seq_num_end_old) { - init_first_frame(input_data, sum_data, in_frame_id); - reset_state(dset); + init_first_frame(input_data, in_frame_id); + reset_state(); } } else { // If we are on the first frame copy it directly into the // output buffer frame so that we don't need to zero the frame if (frame == 0) { - init_first_frame(input_data, sum_data, in_frame_id); - reset_state(dset); + init_first_frame(input_data, in_frame_id); + reset_state(); } else - integrate_frame(input_data, sum_data, in_frame_id); + integrate_frame(input_data, in_frame_id); } fpga_seq_num_end_old = fpga_seq_num_end; @@ -370,17 +370,13 @@ void HFBAccumulate::main_thread() { } // end stop thread } -bool HFBAccumulate::reset_state(HFBAccumulate::internalState& state) { +bool HFBAccumulate::reset_state() { // Reset the internal counters - state.weight_diff_sum = 0; + weight_diff_sum = 0; // Zero out accumulation arrays - std::fill(state.hfb1.begin(), state.hfb1.end(), 0.0); - std::fill(state.hfb2.begin(), state.hfb2.end(), 0.0); + std::fill(hfb1.begin(), hfb1.end(), 0.0); + std::fill(hfb2.begin(), hfb2.end(), 0.0); return true; } - -HFBAccumulate::internalState::internalState(size_t num_beams, size_t num_sub_freqs) : - hfb1(num_beams * num_sub_freqs), - hfb2(num_beams * num_sub_freqs) {} diff --git a/lib/stages/HFBAccumulate.hpp b/lib/stages/HFBAccumulate.hpp index 523b25a9e..e592425df 100644 --- a/lib/stages/HFBAccumulate.hpp +++ b/lib/stages/HFBAccumulate.hpp @@ -13,7 +13,8 @@ #include "bufferContainer.hpp" // for bufferContainer #include "datasetManager.hpp" // for datasetManager, state_id_t, dset_id_t -#include // for size_t +#include "gsl-lite.hpp" // for span + #include // for uint32_t, int32_t, int64_t #include // for string #include // for vector @@ -69,55 +70,21 @@ class HFBAccumulate : public kotekan::Stage { private: /// Copy the first frame of the integration - void init_first_frame(float* input_data, float* sum_data, const uint32_t in_frame_id); + void init_first_frame(float* input_data, const uint32_t in_frame_id); /// Add a frame to the integration - void integrate_frame(float* input_data, float* sum_data, const uint32_t in_frame_id); + void integrate_frame(float* input_data, const uint32_t in_frame_id); /// Normalise frame after integration has been completed - void normalise_frame(float* sum_data, const uint32_t in_frame_id); - - // NOTE: Annoyingly this can't be forward declared, and defined fully externally - // as the std::deque needs the complete type - // TODO: Place all of internalState's members directly on main class - /** - * @class internalState - * @brief Hold the internal state of a gated accumulation. - **/ - struct internalState { - - /** - * @brief Initialise the required fields. - * - * Everything else will be set by the reset_state call during - * initialisation. - * - * @param num_beams No. of FRB beams. - * @param num_sub_freqs No. of sub-frequencies. - **/ - internalState(size_t num_beams, size_t num_sub_freqs); - - /// The sum of the squared weight difference. This is needed for - /// de-biasing the weight calculation - float weight_diff_sum; - - /// Accumulation vectors - std::vector hfb1; - std::vector hfb2; - - friend HFBAccumulate; - }; - - /** - * @brief Reset the state when we restart an integration. - * - * @param state State to reset. - * @returns True if this accumulation was enabled. - **/ - bool reset_state(internalState& state); + void normalise_frame(const uint32_t in_frame_id); + /// Reset the state when we restart an integration. + bool reset_state(); Buffer* in_buf; Buffer* cls_buf; Buffer* out_buf; + /// View of the output frame data. + gsl::span out_hfb; + /// Config variables uint32_t _num_frames_to_integrate; uint32_t _num_frb_total_beams; @@ -132,6 +99,14 @@ class HFBAccumulate : public kotekan::Stage { int64_t fpga_seq_num; int64_t fpga_seq_num_end; + /// The sum of the squared weight difference. This is needed for + /// de-biasing the weight calculation + float weight_diff_sum; + + /// Accumulation vectors + std::vector hfb1; + std::vector hfb2; + // dataset ID for the base states dset_id_t base_dataset_id; diff --git a/lib/stages/HFBFreqSubset.cpp b/lib/stages/HFBFreqSubset.cpp deleted file mode 100644 index 17a7dc299..000000000 --- a/lib/stages/HFBFreqSubset.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "HFBFreqSubset.hpp" - -#include "Config.hpp" // for Config -#include "HFBFrameView.hpp" // for HFBFrameView -#include "Hash.hpp" // for operator< -#include "Stage.hpp" // for Stage -#include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate -#include "bufferContainer.hpp" // for bufferContainer -#include "datasetManager.hpp" // for dset_id_t -#include "visUtil.hpp" // for frameID - -#include // for uint32_t -#include // for future -#include // for map -#include // for pair - -using kotekan::bufferContainer; -using kotekan::Config; -using kotekan::Stage; - -REGISTER_KOTEKAN_STAGE(HFBFreqSubset); - - -HFBFreqSubset::HFBFreqSubset(Config& config, const std::string& unique_name, - bufferContainer& buffer_container) : - FreqSubset(config, unique_name, buffer_container) {} - -void HFBFreqSubset::copy_dataset_id(dset_id_t dataset_id, frameID input_frame_id, - frameID output_frame_id) { - - // Copy frame and create view - auto output_frame = HFBFrameView::copy_frame(in_buf, input_frame_id, out_buf, output_frame_id); - - // Wait for the dataset ID for the outgoing frame - if (change_dset_fut.valid()) - change_dset_fut.wait(); - - output_frame.dataset_id = dset_id_map.at(dataset_id); -} - -std::pair HFBFreqSubset::get_frame_data(frameID input_frame_id) { - - auto frame = HFBFrameView(in_buf, input_frame_id); - return {frame.dataset_id, frame.freq_id}; -} diff --git a/lib/stages/HFBFreqSubset.hpp b/lib/stages/HFBFreqSubset.hpp deleted file mode 100644 index dbdaa114d..000000000 --- a/lib/stages/HFBFreqSubset.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/***************************************** -@file -@brief Stage for subsetting absorber data by frequency. -- HFBFreqSubset : FreqSubset -*****************************************/ -#ifndef HFB_FREQ_SUBSET_HPP -#define HFB_FREQ_SUBSET_HPP - -#include "Config.hpp" // for Config -#include "FreqSubset.hpp" // for FreqSubset -#include "bufferContainer.hpp" // for bufferContainer -#include "datasetManager.hpp" // for dset_id_t -#include "visUtil.hpp" // for frameID - -#include // for uint32_t -#include // for string -#include // for pair - -/** - * @class HFBFreqSubset - * @brief Outputs a HFBBuffer stream with a subset of the input frequencies. - * - * This class inherits from the FreqSubset base class and outputs a subset of frequencies - * @author James Willis - */ -class HFBFreqSubset : public FreqSubset { - -public: - /// Default constructor - HFBFreqSubset(kotekan::Config& config, const std::string& unique_name, - kotekan::bufferContainer& buffer_container); - -protected: - // Copy dataset ID from input to output frame - void copy_dataset_id(dset_id_t dataset_id, frameID input_frame_id, - frameID output_frame_id) override; - - // Get dataset ID and frequency ID from frame - std::pair get_frame_data(frameID input_frame_id) override; -}; - - -#endif diff --git a/lib/stages/VisFreqSubset.cpp b/lib/stages/VisFreqSubset.cpp deleted file mode 100644 index 989c0ab63..000000000 --- a/lib/stages/VisFreqSubset.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "VisFreqSubset.hpp" - -#include "Config.hpp" // for Config -#include "Hash.hpp" // for operator< -#include "Stage.hpp" // for Stage -#include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate -#include "bufferContainer.hpp" // for bufferContainer -#include "datasetManager.hpp" // for dset_id_t -#include "visBuffer.hpp" // for VisFrameView -#include "visUtil.hpp" // for frameID - -#include // for uint32_t -#include // for future -#include // for map -#include // for pair - -using kotekan::bufferContainer; -using kotekan::Config; -using kotekan::Stage; - -REGISTER_KOTEKAN_STAGE(VisFreqSubset); - - -VisFreqSubset::VisFreqSubset(Config& config, const std::string& unique_name, - bufferContainer& buffer_container) : - FreqSubset(config, unique_name, buffer_container) {} - -void VisFreqSubset::copy_dataset_id(dset_id_t dataset_id, frameID input_frame_id, - frameID output_frame_id) { - - // Copy frame and create view - auto output_frame = VisFrameView::copy_frame(in_buf, input_frame_id, out_buf, output_frame_id); - - // Wait for the dataset ID for the outgoing frame - if (change_dset_fut.valid()) - change_dset_fut.wait(); - - output_frame.dataset_id = dset_id_map.at(dataset_id); -} - -std::pair VisFreqSubset::get_frame_data(frameID input_frame_id) { - - auto frame = VisFrameView(in_buf, input_frame_id); - return {frame.dataset_id, frame.freq_id}; -} diff --git a/lib/stages/VisFreqSubset.hpp b/lib/stages/VisFreqSubset.hpp deleted file mode 100644 index 212d0fdbb..000000000 --- a/lib/stages/VisFreqSubset.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/***************************************** -@file -@brief Stage for subsetting visibility data by frequency. -- VisFreqSubset : public FreqSubset -*****************************************/ -#ifndef VIS_FREQ_SUBSET_HPP -#define VIS_FREQ_SUBSET_HPP - -#include "Config.hpp" // for Config -#include "FreqSubset.hpp" // for FreqSubset -#include "bufferContainer.hpp" // for bufferContainer -#include "datasetManager.hpp" // for dset_id_t -#include "visUtil.hpp" // for frameID - -#include // for uint32_t -#include // for string -#include // for pair - -/** - * @class VisFreqSubset - * @brief Outputs a VisBuffer stream with a subset of the input frequencies. - * - * This class inherits from the FreqSubset base class and outputs a subset of frequencies - * @author Mateus Fandino - */ -class VisFreqSubset : public FreqSubset { - -public: - /// Default constructor - VisFreqSubset(kotekan::Config& config, const std::string& unique_name, - kotekan::bufferContainer& buffer_container); - -protected: - // Copy dataset ID from input to output frame - void copy_dataset_id(dset_id_t dataset_id, frameID input_frame_id, - frameID output_frame_id) override; - - // Get dataset ID and frequency ID from frame - std::pair get_frame_data(frameID input_frame_id) override; -}; - - -#endif From c4dfd19731911cf2957bf52db6940ed69592e505 Mon Sep 17 00:00:00 2001 From: james-s-willis <36906682+james-s-willis@users.noreply.github.com> Date: Tue, 9 Feb 2021 13:15:47 -0500 Subject: [PATCH 03/15] HDF5 writer for HFB data (#901) * Abstracted the visRawReader stage into RawReader. * Renamed visRawReader -> VisRawReader. * Created HFBRawReader to inherit from RawReader base class. * Created a HFB equivalent of vis H5 files and file archives. * Abstracted visTranspose stage to Transpose. * Renamed visTranspose -> VisTranspose. * Created HFBTranspose stage to transpose raw HFB files. * Moved HighFive create_datatype templates to a common header file, H5Support.hpp. * Store the Nyquist zone in the ICETelescope. --- config/tests/chime_science_run_recv_hfb.yaml | 10 +- config/tests/test_HFBRawReader.yaml | 57 +++ lib/stages/CMakeLists.txt | 2 +- lib/stages/HFBTranspose.cpp | 261 ++++++++++ lib/stages/HFBTranspose.hpp | 82 +++ lib/stages/RawReader.hpp | 8 +- lib/stages/Transpose.cpp | 231 +++++++++ .../{visTranspose.hpp => Transpose.hpp} | 146 +++--- lib/stages/VisTranspose.cpp | 311 ++++++++++++ lib/stages/VisTranspose.hpp | 91 ++++ lib/stages/applyGains.cpp | 2 +- lib/stages/basebandReadout.cpp | 2 +- lib/stages/visTranspose.cpp | 479 ------------------ lib/testing/FakeHFB.cpp | 12 +- lib/testing/fakeGpu.cpp | 4 + lib/testing/fakeGpu.hpp | 1 + lib/utils/CMakeLists.txt | 2 +- lib/utils/FileArchive.hpp | 25 + lib/utils/H5Support.hpp | 99 ++++ lib/utils/HFBFileArchive.cpp | 290 +++++++++++ lib/utils/HFBFileArchive.hpp | 133 +++++ lib/utils/ICETelescope.cpp | 5 + lib/utils/ICETelescope.hpp | 2 + lib/utils/Telescope.hpp | 7 + lib/utils/visFileArchive.cpp | 81 +-- lib/utils/visFileArchive.hpp | 25 +- lib/utils/visFileH5.cpp | 55 +- lib/utils/visFileH5.hpp | 30 -- tests/boost/test_transpose.cpp | 4 +- tests/test_transpose.py | 4 +- 30 files changed, 1699 insertions(+), 762 deletions(-) create mode 100644 config/tests/test_HFBRawReader.yaml create mode 100644 lib/stages/HFBTranspose.cpp create mode 100644 lib/stages/HFBTranspose.hpp create mode 100644 lib/stages/Transpose.cpp rename lib/stages/{visTranspose.hpp => Transpose.hpp} (53%) create mode 100644 lib/stages/VisTranspose.cpp create mode 100644 lib/stages/VisTranspose.hpp delete mode 100644 lib/stages/visTranspose.cpp create mode 100644 lib/utils/FileArchive.hpp create mode 100644 lib/utils/H5Support.hpp create mode 100644 lib/utils/HFBFileArchive.cpp create mode 100644 lib/utils/HFBFileArchive.hpp diff --git a/config/tests/chime_science_run_recv_hfb.yaml b/config/tests/chime_science_run_recv_hfb.yaml index b43a647bb..a3e3e30e0 100644 --- a/config/tests/chime_science_run_recv_hfb.yaml +++ b/config/tests/chime_science_run_recv_hfb.yaml @@ -21,7 +21,7 @@ init_dm: True #num_frames: 5 #samples_per_data_set: 49152 samples_per_data_set: 1024 -freq_ids: [3, 777, 554] +freq_ids: [3, 50, 777, 554] cadence: 0.01 # Constants @@ -51,12 +51,12 @@ gen_data: write_hfb: file_length: 512 file_type: hfbraw - root_path: /mnt/ramdisk + root_path: . kotekan_stage: HFBWriter in_buf: hfbbuf_10s instrument_name: chimeHFB -buffer_status: - kotekan_stage: bufferStatus - print_status: false +#buffer_status: +# kotekan_stage: bufferStatus +# print_status: false diff --git a/config/tests/test_HFBRawReader.yaml b/config/tests/test_HFBRawReader.yaml new file mode 100644 index 000000000..015023e07 --- /dev/null +++ b/config/tests/test_HFBRawReader.yaml @@ -0,0 +1,57 @@ +# Read raw HFB data and write it as HDF5 data + +# Constants +cpu_affinity: [] +log_level: info +num_elements: 2048 +num_ev: 4 +num_local_freq: 1 +num_frb_total_beams: 1024 +factor_upchan: 128 + +dataset_manager: + ds_broker_host: "127.0.0.1" + ds_broker_port: 12050 + use_dataset_broker: False + +read_buffer: + kotekan_buffer: hfb + metadata_pool: hfb_pool + num_frames: '4096' + num_prod: 2048 + +hfb_pool: + kotekan_metadata_pool: HFBMetadata + num_metadata_objects: '655360' + +read_raw: + log_level: debug + infile: 20210125T212808Z_chimeHFB_corr/00000000_0000 + kotekan_stage: HFBRawReader + max_read_rate: 0.0 + out_buf: read_buffer + readahead_blocks: 4 + ignore_versions: True + chunk_size: + - 16 # chunk_freq + - 64 # chunk_stack + - 256 # chunk_time + - 16 # chunk_beam + - 128 # chunk_sub-freq + +write_hdf5: + log_level: debug + comet_timeout: 180. + kotekan_stage: HFBTranspose + in_buf: read_buffer + outfile: ./test + chunk_size: + - 16 # chunk_freq + - 64 # chunk_stack + - 256 # chunk_time + - 16 # chunk_beam + - 128 # chunk_sub-freq + +#hfb_debug: +# kotekan_stage: visDebug +# in_buf: read_buffer diff --git a/lib/stages/CMakeLists.txt b/lib/stages/CMakeLists.txt index 5c7c045cc..be154bf7c 100644 --- a/lib/stages/CMakeLists.txt +++ b/lib/stages/CMakeLists.txt @@ -93,7 +93,7 @@ if(${USE_HDF5}) endif() if(${USE_HDF5} AND ${USE_OMP}) - target_sources(kotekan_stages PRIVATE visTranspose.cpp) + target_sources(kotekan_stages PRIVATE Transpose.cpp VisTranspose.cpp HFBTranspose.cpp) endif() if(${USE_OMP}) diff --git a/lib/stages/HFBTranspose.cpp b/lib/stages/HFBTranspose.cpp new file mode 100644 index 000000000..7d796b352 --- /dev/null +++ b/lib/stages/HFBTranspose.cpp @@ -0,0 +1,261 @@ +#include "HFBTranspose.hpp" + +#include "Config.hpp" // for Config +#include "H5Support.hpp" // for dset_id_str, DSET_ID_LEN +#include "HFBFileArchive.hpp" // for HFBFileArchive +#include "HFBFrameView.hpp" // for HFBFrameView +#include "Hash.hpp" // for Hash, operator!= +#include "Stage.hpp" // for Stage +#include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate +#include "Telescope.hpp" // for Telescope +#include "bufferContainer.hpp" // for bufferContainer +#include "datasetManager.hpp" // for dset_id_t, datasetManager +#include "datasetState.hpp" // for metadataState, stackState, acqDatasetIdState, eigenvalu... +#include "errors.h" // for exit_kotekan, CLEAN_EXIT, ReturnCode +#include "kotekanLogging.hpp" // for DEBUG, FATAL_ERROR, logLevel, INFO +#include "prometheusMetrics.hpp" // for Metrics, Gauge + +#include "fmt.hpp" // for format +#include "gsl-lite.hpp" // for span +#include "json.hpp" // for basic_json<>::object_t, json, basic_json, + +#include // for max, fill, min +#include // for uint32_t +#include // for __forced_unwind +#include // for exception +#include // for async, future +#include // for out_of_range, invalid_argument +#include // for uint32_t, uint64_t +#include // for system_error + + +using kotekan::bufferContainer; +using kotekan::Config; +using kotekan::Stage; +using kotekan::prometheus::Metrics; + +REGISTER_KOTEKAN_STAGE(HFBTranspose); + +HFBTranspose::HFBTranspose(Config& config, const std::string& unique_name, + bufferContainer& buffer_container) : + Transpose(config, unique_name, buffer_container) { + + // get chunk dimensions for write from config file + chunk = config.get>(unique_name, "chunk_size"); + if (chunk.size() != 5) + throw std::invalid_argument("Chunk size needs exactly five elements " + "(has " + + std::to_string(chunk.size()) + ")."); + if (chunk[0] < 1 || chunk[1] < 1 || chunk[2] < 1 || chunk[3] < 1 || chunk[4] < 1) + throw std::invalid_argument("HFBTranspose: Config: Chunk size needs " + "to be equal to or greater than one."); + chunk_t = chunk[2]; + chunk_f = chunk[0]; + + metadata["archive_version"] = "4.3.0"; + + DEBUG("chunk_t: {}, chunk_f: {}, chunk_b: {}, chunk_sf: {}", chunk_t, chunk_f, chunk[3], + chunk[4]); +} + +bool HFBTranspose::get_dataset_state(dset_id_t ds_id) { + + datasetManager& dm = datasetManager::instance(); + + // Get the states synchronously. + auto tstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); + auto fstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); + auto bstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); + auto sfstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); + auto mstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); + + const metadataState* mstate; + const timeState* tstate; + const freqState* fstate; + const beamState* bstate; + const subfreqState* sfstate; + bool timed_out = mstate_fut.wait_for(timeout) == std::future_status::timeout; + if (!timed_out) + mstate = mstate_fut.get(); + timed_out = timed_out || (tstate_fut.wait_for(timeout) == std::future_status::timeout); + if (!timed_out) + tstate = tstate_fut.get(); + timed_out = timed_out || (fstate_fut.wait_for(timeout) == std::future_status::timeout); + if (!timed_out) + fstate = fstate_fut.get(); + timed_out = timed_out || (bstate_fut.wait_for(timeout) == std::future_status::timeout); + if (!timed_out) + bstate = bstate_fut.get(); + timed_out = timed_out || (sfstate_fut.wait_for(timeout) == std::future_status::timeout); + if (!timed_out) + sfstate = sfstate_fut.get(); + if (timed_out) { + ERROR("Communication with dataset broker timed out for dataset id {}.", ds_id); + dm.stop(); + exit_kotekan(ReturnCode::DATASET_MANAGER_FAILURE); + return false; + } + + + if (mstate == nullptr || tstate == nullptr || bstate == nullptr || fstate == nullptr + || sfstate == nullptr) { + FATAL_ERROR("One of the dataset states is null."); + return false; + } + + // TODO split instrument_name up into the real instrument name, + // registered by HFBAccumulate (?) and a data type, registered where + // data is written to file the first time + metadata["instrument_name"] = mstate->get_instrument_name(); + metadata["weight_type"] = mstate->get_weight_type(); + + std::string git_commit_hash_dataset = mstate->get_git_version_tag(); + + // TODO: enforce this if build type == release? + if (git_commit_hash_dataset != metadata["git_version_tag"].get()) + INFO("Git version tags don't match: dataset {} has tag {:s}," + "while the local git version tag is {:s}", + ds_id, git_commit_hash_dataset, metadata["git_version_tag"].get()); + + times = tstate->get_times(); + beams = bstate->get_beams(); + + // unzip the vector of pairs in freqState + for (auto& [id, freq] : fstate->get_freqs()) { + (void)id; + freqs.push_back(freq); + } + + num_subfreq = sfstate->get_subfreqs().size(); + num_time = times.size(); + num_freq = freqs.size(); + num_beams = beams.size(); + + // Populate the sub-frequency index map + // Each sub-frequency is the difference from the centre of the sub-band + auto& tel = Telescope::instance(); + + sub_freqs.resize(num_subfreq); + + // Work out if the frequencies are decreasing or increasing + const double sign = (tel.nyquist_zone() % 2 ? 1 : -1); + const double freq_width = tel.freq_width(0); + double freq_diff = -sign * 0.5 * freq_width; + const double freq_inc = freq_width / num_subfreq; + for (auto& sf : sub_freqs) { + sf = freq_diff; + freq_diff += sign * freq_inc; + } + + // the dimension of the visibilities is different for stacked data + eff_data_dim = num_beams * num_subfreq; + + // Ensure chunk_size not too large + chunk_t = std::min(chunk_t, num_time); + write_t = chunk_t; + chunk_f = std::min(chunk_f, num_freq); + write_f = chunk_f; + + DEBUG("Dataset {} has {:d} times, {:d} frequencies, {:d} beams and {:d} sub-frequencies.\n " + "Data dimension: {:d} bytes, time chunk: {:d}, freq chunk: {:d}", + ds_id, num_time, num_freq, num_beams, num_subfreq, eff_data_dim, chunk_t, chunk_f); + + // Allocate memory for collecting frames + hfb.resize(chunk_t * chunk_f * eff_data_dim, 0.); + hfb_weight.resize(chunk_t * chunk_f * eff_data_dim, 0.); + // init frac_lost to 1.0 to match empty frames + frac_lost.resize(chunk_t * chunk_f, 1.); + dset_id.resize(chunk_t * chunk_f); + + // Initialise dataset ID array with null IDs + std::string null_ds_id = fmt::format("{}", dset_id_t::null); + for (auto& ds : dset_id) { + std::copy(null_ds_id.c_str(), null_ds_id.c_str() + DSET_ID_LEN, ds.hash); + } + + return true; +} + +std::tuple HFBTranspose::get_frame_data() { + + auto frame = HFBFrameView(in_buf, frame_id); + return std::make_tuple(frame.calculate_frame_size(num_beams, num_subfreq), frame.fpga_seq_total, + frame.dataset_id); +} + +void HFBTranspose::write_chunk(size_t t_ind, size_t f_ind) { + DEBUG("Writing at freq {:d} and time {:d}", f_ind, t_ind); + DEBUG("Writing block of {:d} freqs and {:d} times. data: {}...{}...{}", write_f, write_t, + hfb[0], hfb[write_t], hfb[write_t * 2]); + + file->write_block("hfb", f_ind, t_ind, write_f, write_t, hfb.data()); + file->write_block("hfb_weight", f_ind, t_ind, write_f, write_t, hfb_weight.data()); + file->write_block("flags/frac_lost", f_ind, t_ind, write_f, write_t, frac_lost.data()); + file->write_block("flags/dataset_id", f_ind, t_ind, write_f, write_t, dset_id.data()); +} + +// increment between chunks +// chunks come in (time, freq) order +// WARNING: This order must be consistent with how HFBRawReader +// implements chunked reads. The mechanism for avoiding +// overwriting flags also relies on this ordering. +void HFBTranspose::increment_chunk(size_t& t_ind, size_t& f_ind, bool& t_edge, bool& f_edge) { + // Figure out where the next chunk starts + f_ind = f_edge ? 0 : (f_ind + chunk_f) % num_freq; + if (f_ind == 0) { + // set incomplete chunk flag + f_edge = (num_freq < chunk_f); + t_ind += chunk_t; + if (num_time - t_ind < chunk_t) { + // Reached an incomplete chunk + t_edge = true; + } + } else if (num_freq - f_ind < chunk_f) { + // Reached an incomplete chunk + f_edge = true; + } + // Determine size of next chunk + write_f = f_edge ? num_freq - f_ind : chunk_f; + write_t = t_edge ? num_time - t_ind : chunk_t; +} + +void HFBTranspose::create_hdf5_file() { + // Create HDF5 file + file = std::unique_ptr( + new HFBFileArchive(filename, metadata, times, freqs, beams, sub_freqs, chunk, + kotekan::logLevel(_member_log_level))); +} + +void HFBTranspose::copy_frame_data(uint32_t freq_index, uint32_t time_index) { + + auto frame = HFBFrameView(in_buf, frame_id); + + std::vector data_cpy(frame.hfb.data(), frame.hfb.data() + eff_data_dim); + std::vector weight_cpy(frame.weight.data(), frame.weight.data() + eff_data_dim); + +// Transpose beam and sub-frequency order +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (uint32_t n = 0; n < eff_data_dim; n++) { + uint32_t i = n / num_beams; + uint32_t j = n % num_beams; + frame.hfb[n] = data_cpy[j * num_subfreq + i]; + frame.weight[n] = weight_cpy[j * num_subfreq + i]; + } + + // Collect frames until a chunk is filled + // Time-transpose as frames come in + // Fastest varying is time (needs to be consistent with reader!) + uint32_t offset = freq_index * write_t; + strided_copy(frame.hfb.data(), hfb.data(), offset * eff_data_dim + time_index, write_t, + eff_data_dim); + strided_copy(frame.weight.data(), hfb_weight.data(), offset * eff_data_dim + time_index, + write_t, eff_data_dim); + frac_lost[offset + time_index] = + frame.fpga_seq_length == 0 ? 1. : 1. - float(frame.fpga_seq_total) / frame.fpga_seq_length; +} + +void HFBTranspose::copy_flags(uint32_t time_index) { + (void)time_index; +} diff --git a/lib/stages/HFBTranspose.hpp b/lib/stages/HFBTranspose.hpp new file mode 100644 index 000000000..3f3724b2c --- /dev/null +++ b/lib/stages/HFBTranspose.hpp @@ -0,0 +1,82 @@ +/***************************************** +@file +@brief Transpose HFBFileRaw data and write into HDF5 files. +- HFBTranspose : public Transpose +*****************************************/ +#ifndef HFB_TRANSPOSE_HPP +#define HFB_TRANSPOSE_HPP + +#include "Config.hpp" // for Config +#include "HFBFileArchive.hpp" // for HFBFileArchive +#include "Transpose.hpp" // for Transpose +#include "bufferContainer.hpp" // for bufferContainer +#include "datasetManager.hpp" // for dset_id_t +#include "visUtil.hpp" // for cfloat, time_ctype, freq_ctype, input_ctype, prod_ctype + +#include // for shared_ptr +#include // for size_t +#include // for uint32_t +#include // for string +#include // for tuple +#include // for vector + + +/** + * @class HFBTranspose + * @brief Stage to transpose raw HFB data + * + * This class inherits from the Transpose base class and transposes raw HFB data. + * @author James Willis + */ +class HFBTranspose : public Transpose { +public: + /// Constructor; loads parameters from config + HFBTranspose(kotekan::Config& config, const std::string& unique_name, + kotekan::bufferContainer& buffer_container); + ~HFBTranspose() = default; + +protected: + /// Request dataset states from the datasetManager and prepare all metadata + /// that is not already set in the constructor. + bool get_dataset_state(dset_id_t ds_id) override; + + // Get frame size, fpga_seq_total and dataset_id from HFBFrameView + std::tuple get_frame_data() override; + + // Create HFBFileArchive + void create_hdf5_file() override; + + // Copy data into local vectors using HFBFrameView + void copy_frame_data(uint32_t freq_index, uint32_t time_index) override; + + // Copy flags into local vectors using HFBFrameView + void copy_flags(uint32_t time_index) override; + + // Write datasets to file + void write_chunk(size_t t_ind, size_t f_ind) override; + + // Increment between chunks + void increment_chunk(size_t& t_ind, size_t& f_ind, bool& t_edge, bool& f_edge) override; + +private: + // Datasets to be stored until ready to write + std::vector time; + std::vector hfb; + std::vector hfb_weight; + std::vector frac_lost; + + /// The list of frequencies and inputs that get written into the index maps + /// of the HDF5 files + std::vector times; + std::vector freqs; + std::vector beams; + std::vector sub_freqs; + + /// Number of products to write + size_t num_beams; + size_t num_subfreq; + + std::shared_ptr file; +}; + +#endif diff --git a/lib/stages/RawReader.hpp b/lib/stages/RawReader.hpp index f7fec0007..040d89625 100644 --- a/lib/stages/RawReader.hpp +++ b/lib/stages/RawReader.hpp @@ -255,8 +255,8 @@ RawReader::RawReader(Config& config, const std::string& unique_name, chunked = config.exists(unique_name, "chunk_size"); if (chunked) { chunk_size = config.get>(unique_name, "chunk_size"); - if (chunk_size.size() != 3) - throw std::invalid_argument("Chunk size needs exactly three " + if (chunk_size.size() < 3) + throw std::invalid_argument("Chunk size needs at least three " "elements (has " + std::to_string(chunk_size.size()) + ")."); chunk_t = chunk_size[2]; @@ -269,6 +269,8 @@ RawReader::RawReader(Config& config, const std::string& unique_name, + std::to_string(chunk_size[2]) + "))."); } + DEBUG("Chunked: {}, chunk_t: {}, chunk_f: {}", chunked, chunk_t, chunk_f); + // Get the list of buffers that this stage should connect to out_buf = get_buffer("out_buf"); register_producer(out_buf, unique_name.c_str()); @@ -486,7 +488,7 @@ void RawReader::main_thread() { // the max rate end_time = current_time(); double sleep_time_this_frame = min_read_time - (end_time - start_time); - DEBUG("Sleep time {}", sleep_time_this_frame); + DEBUG2("Sleep time {}", sleep_time_this_frame); if (sleep_time > 0) { auto ts = double_to_ts(sleep_time_this_frame); nanosleep(&ts, nullptr); diff --git a/lib/stages/Transpose.cpp b/lib/stages/Transpose.cpp new file mode 100644 index 000000000..be9181d76 --- /dev/null +++ b/lib/stages/Transpose.cpp @@ -0,0 +1,231 @@ +#include "Transpose.hpp" + +#include "Config.hpp" // for Config +#include "Hash.hpp" // for Hash, operator!= +#include "SystemInterface.hpp" // for get_hostname, get_username +#include "buffer.h" // for wait_for_full_frame, mark_frame_empty, register_consumer +#include "bufferContainer.hpp" // for bufferContainer +#include "dataset.hpp" // for dataset +#include "datasetManager.hpp" // for dset_id_t, datasetManager +#include "datasetState.hpp" // for metadataState, stackState, acqDatasetIdState, eigenvalu... +#include "errors.h" // for exit_kotekan, CLEAN_EXIT, ReturnCode +#include "kotekanLogging.hpp" // for DEBUG, FATAL_ERROR, logLevel, INFO +#include "prometheusMetrics.hpp" // for Metrics, Gauge +#include "version.h" // for get_git_commit_hash + +#include "fmt.hpp" // for format + +#include // for max, fill, min +#include // for atomic_bool +#include // for __forced_unwind +#include // for exception +#include // for _Bind_helper<>::type, bind, function +#include // for async, future +#include // for map +#include // for allocator_traits<>::value_type +#include // for match_results<>::_Base_type +#include // for out_of_range, invalid_argument +#include // for uint32_t, uint64_t +#include // for system_error + + +using kotekan::bufferContainer; +using kotekan::Config; +using kotekan::Stage; +using kotekan::prometheus::Metrics; + +Transpose::Transpose(Config& config, const std::string& unique_name, + bufferContainer& buffer_container) : + Stage(config, unique_name, buffer_container, std::bind(&Transpose::main_thread, this)), + in_buf(get_buffer("in_buf")), + frame_id(in_buf) { + + register_consumer(in_buf, unique_name.c_str()); + + // Get file path to write to + filename = config.get(unique_name, "outfile"); + + // Get a timeout for communication with broker + timeout = + std::chrono::duration(config.get_default(unique_name, "comet_timeout", 60.)); + + // Collect some metadata. The rest is requested from the datasetManager, + // once we received the first frame. + metadata["notes"] = ""; + metadata["git_version_tag"] = get_git_commit_hash(); + metadata["system_user"] = get_username(); + metadata["collection_server"] = get_hostname(); +} + +void Transpose::main_thread() { + + uint32_t frames_so_far = 0; + // frequency and time indices within chunk + uint32_t fi = 0, ti = 0; + // offset for copying into buffer + uint32_t offset = 0; + + // Flags to indicate incomplete chunks + bool t_edge = false, f_edge = false; + size_t f_ind = 0, t_ind = 0; + + // The dataset ID we read from the frame + dset_id_t ds_id; + // The dataset ID of the state without the time axis + dset_id_t base_ds_id; + // String formatted dataset ID to be written to the file + std::string ds_id_str; + + size_t frame_size = 0; + + // wait for a non-empty frame to get dataset ID from + uint32_t num_empty_skip = 0; + while (true) { + // Wait for a frame in the input buffer in order to get the dataset ID + if ((wait_for_full_frame(in_buf, unique_name.c_str(), frame_id)) == nullptr) { + return; + } + + // Get the frame size to publish metrics later + uint64_t fpga_seq_total; + dset_id_t frame_ds_id; + std::tie(frame_size, fpga_seq_total, frame_ds_id) = get_frame_data(); + + // If the frame is empty, release the buffer and continue + if (fpga_seq_total == 0 && frame_ds_id == dset_id_t::null) { + DEBUG("Got empty frame ({:d}).", frame_id); + mark_frame_empty(in_buf, unique_name.c_str(), frame_id++); + num_empty_skip++; + } else { + ds_id = frame_ds_id; + break; + } + } + + if (num_empty_skip > 0) { + INFO("Found {:d} empty frames at the start of the file.", num_empty_skip); + } + + if (!get_dataset_state(ds_id)) { + FATAL_ERROR("Couldn't find ancestor of dataset {}. " + "Make sure there is a stage upstream in the config, that adds the dataset " + "states.\nExiting...", + ds_id); + return; + } + + // Get the original dataset ID (before adding time axis) + base_ds_id = base_dset(ds_id); + + found_flags = std::vector(write_t, false); + + // Create HDF5 file + create_hdf5_file(); + + auto& transposed_bytes_metric = + Metrics::instance().add_counter("kotekan_transpose_data_transposed_bytes", unique_name); + + while (!stop_thread) { + if (num_empty_skip > 0) { + // Write out empty frames that were skipped at start + // All arrays are initialised to zero, so we just need to move through them + num_empty_skip--; + } else { + // Wait for a full frame in the input buffer + if ((wait_for_full_frame(in_buf, unique_name.c_str(), frame_id)) == nullptr) { + break; + } + + // Collect frames until a chunk is filled + // Time-transpose as frames come in + // Fastest varying is time (needs to be consistent with reader!) + copy_frame_data(fi, ti); + + auto [frame_size, fpga_seq_total, frame_ds_id] = get_frame_data(); + (void)frame_size; + + // Parse the dataset ID + if (fpga_seq_total == 0 && frame_ds_id == dset_id_t::null) { + DEBUG2("Got an empty frame."); + // Empty frames have a null dataset ID + ds_id_str = fmt::format("{}", dset_id_t::null); + } else if (frame_ds_id != ds_id) { + // TODO assuming that dataset ID changes here never change dataset dimensions + DEBUG("Dataset ID has changed from {} to {}.", ds_id, frame_ds_id); + // Update the dataset ID we are writing out + ds_id = frame_ds_id; + // Store original dataset ID (before adding time axis) + base_ds_id = base_dset(ds_id); + ds_id_str = fmt::format("{}", base_ds_id); + } else { + // Dataset ID hasn't changed + ds_id_str = fmt::format("{}", base_ds_id); + } + if (ds_id_str.length() != DSET_ID_LEN - 1) { + FATAL_ERROR("Formatted dataset ID string does not have expected length."); + return; + } + std::copy(ds_id_str.c_str(), ds_id_str.c_str() + DSET_ID_LEN, + dset_id[offset + ti].hash); + + // Only copy flags if we haven't already + copy_flags(ti); + + // move to next frame + mark_frame_empty(in_buf, unique_name.c_str(), frame_id++); + } + + // Increment within read chunk + // within a chunk, frequency is the fastest varying index + fi = (fi + 1) % write_f; + if (fi == 0) + ti++; + if (ti == write_t) { + // chunk is complete + write_chunk(t_ind, f_ind); + // increment between chunks + increment_chunk(t_ind, f_ind, t_edge, f_edge); + fi = 0; + ti = 0; + + // export prometheus metric + transposed_bytes_metric.inc(frame_size); + } + + frames_so_far++; + // Exit when all frames have been written + if (frames_so_far == num_time * num_freq) { + INFO("Done. Exiting."); + exit_kotekan(ReturnCode::CLEAN_EXIT); + return; + } + } +} + +dset_id_t Transpose::base_dset(dset_id_t ds_id) { + + datasetManager& dm = datasetManager::instance(); + + try { + return dm.datasets().at(ds_id).base_dset(); + } catch (std::out_of_range& e) { + DEBUG("Fetching metadata state..."); + // fetch a metadata state just to ensure we have a copy of that dataset + auto mstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); + auto ready = mstate_fut.wait_for(timeout); + if (ready == std::future_status::timeout) { + ERROR("Communication with dataset broker timed out for datatset id {}.", ds_id); + dm.stop(); + exit_kotekan(ReturnCode::DATASET_MANAGER_FAILURE); + return ds_id; + } + const metadataState* mstate = mstate_fut.get(); + (void)mstate; + try { + return dm.datasets().at(ds_id).base_dset(); + } catch (std::out_of_range& e) { + FATAL_ERROR("Failed to get base dataset of dataset with ID {}. {}", ds_id, e.what()); + return ds_id; + } + } +} diff --git a/lib/stages/visTranspose.hpp b/lib/stages/Transpose.hpp similarity index 53% rename from lib/stages/visTranspose.hpp rename to lib/stages/Transpose.hpp index fac87edb8..a9bc4c42f 100644 --- a/lib/stages/visTranspose.hpp +++ b/lib/stages/Transpose.hpp @@ -1,36 +1,51 @@ -#ifndef VISTRANSPOSE -#define VISTRANSPOSE +/***************************************** +@file +@brief Base class for transposing raw files into HDF5 files. +- Transpose : public kotekan::Stage +*****************************************/ +#ifndef TRANSPOSE_HPP +#define TRANSPOSE_HPP + #include "Config.hpp" // for Config +#include "H5Support.hpp" // for AtomicType<>::AtomicType, dset_id_str #include "Stage.hpp" // for Stage #include "buffer.h" // for Buffer #include "bufferContainer.hpp" // for bufferContainer #include "datasetManager.hpp" // for dset_id_t -#include "visFileArchive.hpp" // for visFileArchive -#include "visFileH5.hpp" -#include "visUtil.hpp" // for cfloat, time_ctype, freq_ctype, input_ctype, prod_ctype +#include "visUtil.hpp" // for cfloat, time_ctype, freq_ctype, input_ctype, prod_ctype #include "json.hpp" // for json #include -#include // for shared_ptr #include // for size_t #include // for uint32_t #include // for string +#include // for tuple #include // for vector /** - * @class visTranspose - * @brief Transposes the data to make time the fastest varying value, compresses - * it and writes it to a file. + * @class Transpose + * @brief Generic base class that transposes the data to make time the fastest varying value, + * compresses it and writes it to a file. + * + * All classes which inherit from this should provide the following API: + * + * get_dataset_state(dset_id_t ds_id); + * get_frame_data(); + * create_hdf5_file(); + * copy_frame_data(uint32_t freq_index, uint32_t time_index); + * copy_flags(uint32_t time_index); + * write_chunk(size_t t_ind, size_t f_ind); + * increment_chunk(size_t &t_ind, size_t &f_ind, bool &t_edge, bool &f_edge); * - * The data (vis, weight, eval and evac) is received as one-dimensional arrays + * The data is received as one-dimensional arrays * that represent flattened-out time-X-frequency matrices. These are transposed * and flattened out again to be written to a file. In other words, * the transposition makes time the fastest-varying for the data values, * where it was frequency before. - * This stage expects the data to be ordered like visRawReader does. + * This stage expects the data to be ordered like RawReader does. * Other stages might not guarentee this same order. * * @warning Don't run this anywhere but on the transpose (gossec) node. @@ -39,8 +54,8 @@ * * @par Buffers * @buffer in_buf The input stream. - * @buffer_format VisBuffer. - * @buffer_metadata VisMetadata + * @buffer_format Buffer. + * @buffer_metadata Metadata * * @conf chunk_size Array of [int, int, int]. Chunk size of the data * (freq, prod, time). @@ -50,96 +65,85 @@ * dataset broker. * * @par Metrics - * @metric kotekan_vistranspose_data_transposed_bytes + * @metric kotekan_transpose_data_transposed_bytes * The total amount of data processed in bytes. * * @author Tristan Pinsonneault-Marotte, Rick Nitsche */ -class visTranspose : public kotekan::Stage { +class Transpose : public kotekan::Stage { public: /// Constructor; loads parameters from config - visTranspose(kotekan::Config& config, const std::string& unique_name, - kotekan::bufferContainer& buffer_container); - ~visTranspose() = default; + Transpose(kotekan::Config& config, const std::string& unique_name, + kotekan::bufferContainer& buffer_container); + ~Transpose() = default; /// Main loop over buffer frames void main_thread() override; -private: - /// Request dataset states from the datasetManager and prepare all metadata - /// that is not already set in the constructor. - bool get_dataset_state(dset_id_t ds_id); - - /// Extract the base dataset ID - dset_id_t base_dset(dset_id_t ds_id); +protected: + // Config values + std::string filename; + std::chrono::duration timeout; // Buffers Buffer* in_buf; + // Input buffer frame ID + frameID frame_id; + + nlohmann::json metadata; + // HDF5 chunk size std::vector chunk; - // size of time dimension of chunk + + // Size of time dimension of chunk size_t chunk_t; - // size of frequency dimension of chunk - size_t chunk_f; - // Config values - std::string filename; - std::chrono::duration timeout; + // Size of frequency dimension of chunk + size_t chunk_f; - // Datasets to be stored until ready to write - std::vector time; - std::vector vis; - std::vector vis_weight; - std::vector eval; - std::vector evec; - std::vector erms; - std::vector gain; - std::vector frac_lost; - std::vector frac_rfi; - std::vector dset_id; - std::vector input_flags; - std::vector reverse_stack; + // Keep track of the non-zero flags found so far + std::vector found_flags; // Keep track of the size to write out // size of frequency and time dimension of chunk when written to file size_t write_f, write_t; - // flags to indicate incomplete chunks - bool t_edge = false; - bool f_edge = false; - void increment_chunk(); - - // keep track of the non-zero flags found so far - std::vector found_flags; - /// The list of frequencies and inputs that get written into the index maps - /// of the HDF5 files - std::vector times; - std::vector freqs; - std::vector inputs; - std::vector prods; - std::vector ev; - std::vector stack; - nlohmann::json metadata; + // Effective dimension of data + size_t eff_data_dim; /// Number of products to write - size_t num_prod; - size_t num_input; size_t num_time; size_t num_freq; - size_t num_ev; - size_t eff_prod_dim; - // write datasets to file - void write(); + // Datasets to be stored until ready to write + std::vector dset_id; - std::shared_ptr file; +private: + /// Request dataset states from the datasetManager and prepare all metadata + /// that is not already set in the constructor. + virtual bool get_dataset_state(dset_id_t ds_id) = 0; + + // Get frame size, fpga_seq_total and dataset_id from FrameView + virtual std::tuple get_frame_data() = 0; + + // Create FileArchive + virtual void create_hdf5_file() = 0; + + // Copy data into local vectors + virtual void copy_frame_data(uint32_t freq_index, uint32_t time_index) = 0; - // Buffer for writing - std::vector write_buf; + // Copy flags into local vectors + virtual void copy_flags(uint32_t time_index) = 0; - size_t f_ind = 0; - size_t t_ind = 0; + // Write datasets to file + virtual void write_chunk(size_t t_ind, size_t f_ind) = 0; + + // Increment between chunks + virtual void increment_chunk(size_t& t_ind, size_t& f_ind, bool& t_edge, bool& f_edge) = 0; + + /// Extract the base dataset ID + dset_id_t base_dset(dset_id_t ds_id); }; template diff --git a/lib/stages/VisTranspose.cpp b/lib/stages/VisTranspose.cpp new file mode 100644 index 000000000..9467810a7 --- /dev/null +++ b/lib/stages/VisTranspose.cpp @@ -0,0 +1,311 @@ +#include "VisTranspose.hpp" + +#include "Config.hpp" // for Config +#include "H5Support.hpp" // for dset_id_str, DSET_ID_LEN +#include "Hash.hpp" // for Hash, operator!= +#include "Stage.hpp" // for Stage +#include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate +#include "bufferContainer.hpp" // for bufferContainer +#include "datasetManager.hpp" // for dset_id_t, datasetManager +#include "datasetState.hpp" // for metadataState, stackState, acqDatasetIdState, eigenvalu... +#include "errors.h" // for exit_kotekan, CLEAN_EXIT, ReturnCode +#include "kotekanLogging.hpp" // for DEBUG, FATAL_ERROR, logLevel, INFO +#include "prometheusMetrics.hpp" // for Metrics, Gauge +#include "visBuffer.hpp" // for VisFrameView +#include "visFileArchive.hpp" // for visFileArchive + +#include "fmt.hpp" // for format +#include "gsl-lite.hpp" // for span +#include "json.hpp" // for basic_json<>::object_t, json, basic_json, + +#include // for max, fill, min +#include // for complex +#include // for uint32_t +#include // for __forced_unwind +#include // for exception +#include // for async, future +#include // for out_of_range, invalid_argument +#include // for uint32_t, uint64_t +#include // for uint +#include // for system_error + + +using kotekan::bufferContainer; +using kotekan::Config; +using kotekan::Stage; +using kotekan::prometheus::Metrics; + +REGISTER_KOTEKAN_STAGE(VisTranspose); + +VisTranspose::VisTranspose(Config& config, const std::string& unique_name, + bufferContainer& buffer_container) : + Transpose(config, unique_name, buffer_container) { + + // get chunk dimensions for write from config file + chunk = config.get>(unique_name, "chunk_size"); + if (chunk.size() != 3) + throw std::invalid_argument("Chunk size needs exactly three elements " + "(has " + + std::to_string(chunk.size()) + ")."); + if (chunk[0] < 1 || chunk[1] < 1 || chunk[2] < 1) + throw std::invalid_argument("VisTranspose: Config: Chunk size needs " + "to be equal to or greater than one."); + chunk_t = chunk[2]; + chunk_f = chunk[0]; + + metadata["archive_version"] = "3.1.0"; + + DEBUG("chunk_t: {}, chunk_f: {}", chunk_t, chunk_f); +} + +bool VisTranspose::get_dataset_state(dset_id_t ds_id) { + + datasetManager& dm = datasetManager::instance(); + + // Get the states synchronously. + auto tstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); + auto pstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); + auto fstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); + auto istate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); + auto evstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); + auto mstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); + auto sstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); + + const stackState* sstate; + const metadataState* mstate; + const eigenvalueState* evstate; + const timeState* tstate; + const prodState* pstate; + const freqState* fstate; + const inputState* istate; + bool timed_out = sstate_fut.wait_for(timeout) == std::future_status::timeout; + if (!timed_out) + sstate = sstate_fut.get(); + timed_out = timed_out || (mstate_fut.wait_for(timeout) == std::future_status::timeout); + if (!timed_out) + mstate = mstate_fut.get(); + timed_out = timed_out || (evstate_fut.wait_for(timeout) == std::future_status::timeout); + if (!timed_out) + evstate = evstate_fut.get(); + timed_out = timed_out || (tstate_fut.wait_for(timeout) == std::future_status::timeout); + if (!timed_out) + tstate = tstate_fut.get(); + timed_out = timed_out || (pstate_fut.wait_for(timeout) == std::future_status::timeout); + if (!timed_out) + pstate = pstate_fut.get(); + timed_out = timed_out || (fstate_fut.wait_for(timeout) == std::future_status::timeout); + if (!timed_out) + fstate = fstate_fut.get(); + timed_out = timed_out || (istate_fut.wait_for(timeout) == std::future_status::timeout); + if (!timed_out) + istate = istate_fut.get(); + if (timed_out) { + ERROR("Communication with dataset broker timed out for datatset id {}.", ds_id); + dm.stop(); + exit_kotekan(ReturnCode::DATASET_MANAGER_FAILURE); + return false; + } + + + if (mstate == nullptr || tstate == nullptr || pstate == nullptr || fstate == nullptr + || istate == nullptr || evstate == nullptr) { + FATAL_ERROR("One of the dataset states is null."); + return false; + } + + // TODO split instrument_name up into the real instrument name, + // registered by visAccumulate (?) and a data type, registered where + // data is written to file the first time + metadata["instrument_name"] = mstate->get_instrument_name(); + metadata["weight_type"] = mstate->get_weight_type(); + + std::string git_commit_hash_dataset = mstate->get_git_version_tag(); + + // TODO: enforce this if build type == release? + if (git_commit_hash_dataset != metadata["git_version_tag"].get()) + INFO("Git version tags don't match: dataset {} has tag {:s}," + "while the local git version tag is {:s}", + ds_id, git_commit_hash_dataset, metadata["git_version_tag"].get()); + + times = tstate->get_times(); + inputs = istate->get_inputs(); + prods = pstate->get_prods(); + ev = evstate->get_ev(); + + // unzip the vector of pairs in freqState + for (auto& [id, freq] : fstate->get_freqs()) { + (void)id; + freqs.push_back(freq); + } + + // Check if this is baseline-stacked data + if (sstate) { + stack = sstate->get_stack_map(); + // TODO: verify this is where it gets stored + reverse_stack = sstate->get_rstack_map(); + } + + num_time = times.size(); + num_freq = freqs.size(); + num_input = inputs.size(); + num_prod = prods.size(); + num_ev = ev.size(); + + // the dimension of the visibilities is different for stacked data + eff_data_dim = (stack.size() > 0) ? stack.size() : num_prod; + + DEBUG("Dataset {} has {:d} times, {:d} frequencies, {:d} products", ds_id, num_time, num_freq, + eff_data_dim); + + // Ensure chunk_size not too large + chunk_t = std::min(chunk_t, num_time); + write_t = chunk_t; + chunk_f = std::min(chunk_f, num_freq); + write_f = chunk_f; + + // Allocate memory for collecting frames + vis.resize(chunk_t * chunk_f * eff_data_dim, 0.); + vis_weight.resize(chunk_t * chunk_f * eff_data_dim, 0.); + eval.resize(chunk_t * chunk_f * num_ev, 0.); + evec.resize(chunk_t * chunk_f * num_ev * num_input, 0.); + erms.resize(chunk_t * chunk_f, 0.); + gain.resize(chunk_t * chunk_f * num_input, 0.); + // init frac_lost to 1.0 to match empty frames + frac_lost.resize(chunk_t * chunk_f, 1.); + frac_rfi.resize(chunk_t * chunk_f, 0.); + input_flags.resize(chunk_t * num_input, 0.); + dset_id.resize(chunk_t * chunk_f); + + // Initialise dataset ID array with null IDs + std::string null_ds_id = fmt::format("{}", dset_id_t::null); + for (auto& ds : dset_id) { + std::copy(null_ds_id.c_str(), null_ds_id.c_str() + DSET_ID_LEN, ds.hash); + } + + return true; +} + +std::tuple VisTranspose::get_frame_data() { + + auto frame = VisFrameView(in_buf, frame_id); + return std::make_tuple(frame.calculate_frame_size(num_input, num_prod, num_ev), + frame.fpga_seq_total, frame.dataset_id); +} + +void VisTranspose::write_chunk(size_t t_ind, size_t f_ind) { + DEBUG("Writing at freq {:d} and time {:d}", f_ind, t_ind); + DEBUG("Writing block of {:d} freqs and {:d} times", write_f, write_t); + + file->write_block("vis", f_ind, t_ind, write_f, write_t, vis.data()); + + file->write_block("vis_weight", f_ind, t_ind, write_f, write_t, vis_weight.data()); + + if (num_ev > 0) { + file->write_block("eval", f_ind, t_ind, write_f, write_t, eval.data()); + file->write_block("evec", f_ind, t_ind, write_f, write_t, evec.data()); + file->write_block("erms", f_ind, t_ind, write_f, write_t, erms.data()); + } + + file->write_block("gain", f_ind, t_ind, write_f, write_t, gain.data()); + + file->write_block("flags/frac_lost", f_ind, t_ind, write_f, write_t, frac_lost.data()); + + file->write_block("flags/frac_rfi", f_ind, t_ind, write_f, write_t, frac_rfi.data()); + + file->write_block("flags/inputs", f_ind, t_ind, write_f, write_t, input_flags.data()); + + file->write_block("flags/dataset_id", f_ind, t_ind, write_f, write_t, dset_id.data()); +} + +// increment between chunks +// chunks come in (time, freq) order +// WARNING: This order must be consistent with how VisRawReader +// implements chunked reads. The mechanism for avoiding +// overwriting flags also relies on this ordering. +void VisTranspose::increment_chunk(size_t& t_ind, size_t& f_ind, bool& t_edge, bool& f_edge) { + // Figure out where the next chunk starts + f_ind = f_edge ? 0 : (f_ind + chunk_f) % num_freq; + if (f_ind == 0) { + // set incomplete chunk flag + f_edge = (num_freq < chunk_f); + t_ind += chunk_t; + // clear flags buffer for next time chunk + std::fill(input_flags.begin(), input_flags.end(), 0.); + std::fill(found_flags.begin(), found_flags.end(), false); + if (num_time - t_ind < chunk_t) { + // Reached an incomplete chunk + t_edge = true; + } + } else if (num_freq - f_ind < chunk_f) { + // Reached an incomplete chunk + f_edge = true; + } + // Determine size of next chunk + write_f = f_edge ? num_freq - f_ind : chunk_f; + write_t = t_edge ? num_time - t_ind : chunk_t; +} + +void VisTranspose::create_hdf5_file() { + // Create HDF5 file + if (stack.size() > 0) { + file = std::unique_ptr( + new visFileArchive(filename, metadata, times, freqs, inputs, prods, stack, + reverse_stack, num_ev, chunk, kotekan::logLevel(_member_log_level))); + } else { + file = std::unique_ptr( + new visFileArchive(filename, metadata, times, freqs, inputs, prods, num_ev, chunk, + kotekan::logLevel(_member_log_level))); + } +} + +void VisTranspose::copy_frame_data(uint32_t freq_index, uint32_t time_index) { + + auto frame = VisFrameView(in_buf, frame_id); + + // Collect frames until a chunk is filled + // Time-transpose as frames come in + // Fastest varying is time (needs to be consistent with reader!) + uint32_t offset = freq_index * write_t; + strided_copy(frame.vis.data(), vis.data(), offset * eff_data_dim + time_index, write_t, + eff_data_dim); + strided_copy(frame.weight.data(), vis_weight.data(), offset * eff_data_dim + time_index, + write_t, eff_data_dim); + + strided_copy(frame.eval.data(), eval.data(), freq_index * num_ev * write_t + time_index, + write_t, num_ev); + + strided_copy(frame.evec.data(), evec.data(), + freq_index * num_ev * num_input * write_t + time_index, write_t, + num_ev * num_input); + + erms[offset + time_index] = frame.erms; + frac_lost[offset + time_index] = + frame.fpga_seq_length == 0 ? 1. : 1. - float(frame.fpga_seq_total) / frame.fpga_seq_length; + frac_rfi[offset + time_index] = + frame.fpga_seq_length == 0 ? 0. : float(frame.rfi_total) / frame.fpga_seq_length; + strided_copy(frame.gain.data(), gain.data(), offset * num_input + time_index, write_t, + num_input); +} + +void VisTranspose::copy_flags(uint32_t time_index) { + + auto frame = VisFrameView(in_buf, frame_id); + + // Only copy flags if we haven't already + if (!found_flags[time_index]) { + // Only update flags if they are non-zero + bool nz_flags = false; + for (uint i = 0; i < num_input; i++) { + if (frame.flags[i] != 0.) { + nz_flags = true; + break; + } + } + if (nz_flags) { + // Copy flags into the buffer. These will not be overwritten until + // the chunks increment in time + strided_copy(frame.flags.data(), input_flags.data(), time_index, write_t, num_input); + found_flags[time_index] = true; + } + } +} diff --git a/lib/stages/VisTranspose.hpp b/lib/stages/VisTranspose.hpp new file mode 100644 index 000000000..3741d0a2d --- /dev/null +++ b/lib/stages/VisTranspose.hpp @@ -0,0 +1,91 @@ +/***************************************** +@file +@brief Transpose VisFileRaw data and write into HDF5 files. +- VisTranspose : public Transpose +*****************************************/ +#ifndef VIS_TRANSPOSE_HPP +#define VIS_TRANSPOSE_HPP + +#include "Config.hpp" // for Config +#include "Transpose.hpp" +#include "bufferContainer.hpp" // for bufferContainer +#include "datasetManager.hpp" // for dset_id_t +#include "visFileArchive.hpp" // for visFileArchive +#include "visUtil.hpp" // for cfloat, time_ctype, freq_ctype, input_ctype, prod_ctype + +#include // for shared_ptr +#include // for size_t +#include // for uint32_t +#include // for string +#include // for tuple +#include // for vector + + +/** + * @class VisTranspose + * @brief Stage to transpose raw visibility data + * + * This class inherits from the Transpose base class and transposes raw visibility data. + * @author Tristan Pinsonneault-Marotte, Rick Nitsche + */ +class VisTranspose : public Transpose { +public: + /// Constructor; loads parameters from config + VisTranspose(kotekan::Config& config, const std::string& unique_name, + kotekan::bufferContainer& buffer_container); + ~VisTranspose() = default; + +protected: + /// Request dataset states from the datasetManager and prepare all metadata + /// that is not already set in the constructor. + bool get_dataset_state(dset_id_t ds_id) override; + + // Get frame size, fpga_seq_total and dataset_id from VisFrameView + std::tuple get_frame_data() override; + + // Create VisFileArchive + void create_hdf5_file() override; + + // Copy data into local vectors using VisFrameView + void copy_frame_data(uint32_t freq_index, uint32_t time_index) override; + + // Copy flags into local vectors using VisFrameView + void copy_flags(uint32_t time_index) override; + + // Write datasets to file + void write_chunk(size_t t_ind, size_t f_ind) override; + + // Increment between chunks + void increment_chunk(size_t& t_ind, size_t& f_ind, bool& t_edge, bool& f_edge) override; + +private: + // Datasets to be stored until ready to write + std::vector vis; + std::vector vis_weight; + std::vector eval; + std::vector evec; + std::vector erms; + std::vector gain; + std::vector frac_lost; + std::vector frac_rfi; + std::vector input_flags; + std::vector reverse_stack; + + /// The list of frequencies and inputs that get written into the index maps + /// of the HDF5 files + std::vector times; + std::vector freqs; + std::vector inputs; + std::vector prods; + std::vector ev; + std::vector stack; + + /// Number of products to write + size_t num_prod; + size_t num_input; + size_t num_ev; + + std::shared_ptr file; +}; + +#endif diff --git a/lib/stages/applyGains.cpp b/lib/stages/applyGains.cpp index cba8260db..99c42f1b3 100644 --- a/lib/stages/applyGains.cpp +++ b/lib/stages/applyGains.cpp @@ -1,6 +1,7 @@ #include "applyGains.hpp" #include "Config.hpp" // for Config +#include "H5Support.hpp" // IWYU pragma: keep #include "Hash.hpp" // for operator< #include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate #include "buffer.h" // for mark_frame_empty, wait_for_full_frame, allocate_new... @@ -13,7 +14,6 @@ #include "prometheusMetrics.hpp" // for Metrics, Counter, Gauge #include "restClient.hpp" // for restClient::restReply, restClient #include "visBuffer.hpp" // for VisFrameView, VisField, VisField::vis, VisField::we... -#include "visFileH5.hpp" // IWYU pragma: keep #include "visUtil.hpp" // for cfloat, modulo, double_to_ts, ts_to_double, frameID #include "fmt.hpp" // for format, fmt diff --git a/lib/stages/basebandReadout.cpp b/lib/stages/basebandReadout.cpp index 1d94201ae..1450efc3e 100644 --- a/lib/stages/basebandReadout.cpp +++ b/lib/stages/basebandReadout.cpp @@ -1,6 +1,7 @@ #include "basebandReadout.hpp" #include "Config.hpp" // for Config +#include "H5Support.hpp" // for create_datatype #include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate #include "Telescope.hpp" // for Telescope #include "basebandApiManager.hpp" // for basebandApiManager @@ -14,7 +15,6 @@ #include "prometheusMetrics.hpp" // for Counter, Gauge, MetricFamily, Metrics #include "version.h" // for get_git_commit_hash #include "visFile.hpp" // for create_lockfile -#include "visFileH5.hpp" // for create_datatype #include "visUtil.hpp" // for input_ctype, ts_to_double, parse_reorder_default #include "fmt.hpp" // for format, fmt diff --git a/lib/stages/visTranspose.cpp b/lib/stages/visTranspose.cpp deleted file mode 100644 index 7b30d27c4..000000000 --- a/lib/stages/visTranspose.cpp +++ /dev/null @@ -1,479 +0,0 @@ -#include "visTranspose.hpp" - -#include "Config.hpp" // for Config -#include "Hash.hpp" // for Hash, operator!= -#include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate -#include "buffer.h" // for wait_for_full_frame, mark_frame_empty, register_consumer -#include "bufferContainer.hpp" // for bufferContainer -#include "dataset.hpp" // for dataset -#include "datasetManager.hpp" // for dset_id_t, datasetManager -#include "datasetState.hpp" // for metadataState, stackState, acqDatasetIdState, eigenvalu... -#include "errors.h" // for exit_kotekan, CLEAN_EXIT, ReturnCode -#include "kotekanLogging.hpp" // for DEBUG, FATAL_ERROR, logLevel, INFO -#include "prometheusMetrics.hpp" // for Metrics, Gauge -#include "version.h" // for get_git_commit_hash -#include "visBuffer.hpp" // for VisFrameView -#include "visFileArchive.hpp" // for visFileArchive - -#include "fmt.hpp" // for format -#include "gsl-lite.hpp" // for span - -#include // for max, fill, min -#include // for atomic_bool -#include // for complex -#include // for uint32_t -#include // for __forced_unwind -#include // for exception -#include // for _Bind_helper<>::type, bind, function -#include // for async, future -#include // for make_move_iterator, move_iterator, operator!= -#include // for map -#include // for match_results<>::_Base_type -#include // for out_of_range, invalid_argument -#include // for uint32_t, uint64_t -#include // for uint -#include // for system_error -#include // for gethostname, getlogin_r -#include // for move, pair - - -using kotekan::bufferContainer; -using kotekan::Config; -using kotekan::Stage; -using kotekan::prometheus::Metrics; - -REGISTER_KOTEKAN_STAGE(visTranspose); - -visTranspose::visTranspose(Config& config, const std::string& unique_name, - bufferContainer& buffer_container) : - Stage(config, unique_name, buffer_container, std::bind(&visTranspose::main_thread, this)) { - - // Fetch the buffers, register - in_buf = get_buffer("in_buf"); - register_consumer(in_buf, unique_name.c_str()); - - // get chunk dimensions for write from config file - chunk = config.get>(unique_name, "chunk_size"); - if (chunk.size() != 3) - throw std::invalid_argument("Chunk size needs exactly three elements " - "(has " - + std::to_string(chunk.size()) + ")."); - if (chunk[0] < 1 || chunk[1] < 1 || chunk[2] < 1) - throw std::invalid_argument("visTranspose: Config: Chunk size needs " - "to be equal to or greater than one."); - chunk_t = chunk[2]; - chunk_f = chunk[0]; - - // Get file path to write to - filename = config.get(unique_name, "outfile"); - - // Get a timeout for communication with broker - timeout = - std::chrono::duration(config.get_default(unique_name, "comet_timeout", 60.)); - - // Collect some metadata. The rest is requested from the datasetManager, - // once we received the first frame. - metadata["archive_version"] = "3.1.0"; - metadata["notes"] = ""; - metadata["git_version_tag"] = get_git_commit_hash(); - char temp[256]; - std::string username = (getlogin_r(temp, 256) == 0) ? temp : "unknown"; - metadata["system_user"] = username; - gethostname(temp, 256); - std::string hostname = temp; - metadata["collection_server"] = hostname; -} - -bool visTranspose::get_dataset_state(dset_id_t ds_id) { - - datasetManager& dm = datasetManager::instance(); - - // Get the states synchronously. - auto tstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); - auto pstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); - auto fstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); - auto istate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); - auto evstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); - auto mstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); - auto sstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); - - const stackState* sstate; - const metadataState* mstate; - const eigenvalueState* evstate; - const timeState* tstate; - const prodState* pstate; - const freqState* fstate; - const inputState* istate; - bool timed_out = sstate_fut.wait_for(timeout) == std::future_status::timeout; - if (!timed_out) - sstate = sstate_fut.get(); - timed_out = timed_out || (mstate_fut.wait_for(timeout) == std::future_status::timeout); - if (!timed_out) - mstate = mstate_fut.get(); - timed_out = timed_out || (evstate_fut.wait_for(timeout) == std::future_status::timeout); - if (!timed_out) - evstate = evstate_fut.get(); - timed_out = timed_out || (tstate_fut.wait_for(timeout) == std::future_status::timeout); - if (!timed_out) - tstate = tstate_fut.get(); - timed_out = timed_out || (pstate_fut.wait_for(timeout) == std::future_status::timeout); - if (!timed_out) - pstate = pstate_fut.get(); - timed_out = timed_out || (fstate_fut.wait_for(timeout) == std::future_status::timeout); - if (!timed_out) - fstate = fstate_fut.get(); - timed_out = timed_out || (istate_fut.wait_for(timeout) == std::future_status::timeout); - if (!timed_out) - istate = istate_fut.get(); - if (timed_out) { - ERROR("Communication with dataset broker timed out for datatset id {}.", ds_id); - dm.stop(); - exit_kotekan(ReturnCode::DATASET_MANAGER_FAILURE); - return false; - } - - - if (mstate == nullptr || tstate == nullptr || pstate == nullptr || fstate == nullptr - || istate == nullptr || evstate == nullptr) { - FATAL_ERROR("One of the dataset states is null."); - return false; - } - - // TODO split instrument_name up into the real instrument name, - // registered by visAccumulate (?) and a data type, registered where - // data is written to file the first time - metadata["instrument_name"] = mstate->get_instrument_name(); - metadata["weight_type"] = mstate->get_weight_type(); - - std::string git_commit_hash_dataset = mstate->get_git_version_tag(); - - // TODO: enforce this if build type == release? - if (git_commit_hash_dataset != metadata["git_version_tag"].get()) - INFO("Git version tags don't match: dataset {} has tag {:s}," - "while the local git version tag is {:s}", - ds_id, git_commit_hash_dataset, metadata["git_version_tag"].get()); - - times = tstate->get_times(); - inputs = istate->get_inputs(); - prods = pstate->get_prods(); - ev = evstate->get_ev(); - - // unzip the vector of pairs in freqState - auto freq_pairs = fstate->get_freqs(); - for (auto it = std::make_move_iterator(freq_pairs.begin()), - end = std::make_move_iterator(freq_pairs.end()); - it != end; ++it) { - freqs.push_back(std::move(it->second)); - } - - // Check if this is baseline-stacked data - if (sstate) { - stack = sstate->get_stack_map(); - // TODO: verify this is where it gets stored - reverse_stack = sstate->get_rstack_map(); - } - - num_time = times.size(); - num_freq = freqs.size(); - num_input = inputs.size(); - num_prod = prods.size(); - num_ev = ev.size(); - - // the dimension of the visibilities is different for stacked data - eff_prod_dim = (stack.size() > 0) ? stack.size() : num_prod; - - DEBUG("Dataset {} has {:d} times, {:d} frequencies, {:d} products", ds_id, num_time, num_freq, - eff_prod_dim); - - // Ensure chunk_size not too large - chunk_t = std::min(chunk_t, num_time); - write_t = chunk_t; - chunk_f = std::min(chunk_f, num_freq); - write_f = chunk_f; - - // Allocate memory for collecting frames - vis.resize(chunk_t * chunk_f * eff_prod_dim, 0.); - vis_weight.resize(chunk_t * chunk_f * eff_prod_dim, 0.); - eval.resize(chunk_t * chunk_f * num_ev, 0.); - evec.resize(chunk_t * chunk_f * num_ev * num_input, 0.); - erms.resize(chunk_t * chunk_f, 0.); - gain.resize(chunk_t * chunk_f * num_input, 0.); - // init frac_lost to 1.0 to match empty frames - frac_lost.resize(chunk_t * chunk_f, 1.); - frac_rfi.resize(chunk_t * chunk_f, 0.); - input_flags.resize(chunk_t * num_input, 0.); - dset_id.resize(chunk_t * chunk_f); - - // Initialise dataset ID array with null IDs - std::string null_ds_id = fmt::format("{}", dset_id_t::null); - for (auto& ds : dset_id) { - std::copy(null_ds_id.c_str(), null_ds_id.c_str() + DSET_ID_LEN, ds.hash); - } - - return true; -} - -void visTranspose::main_thread() { - - frameID frame_id(in_buf); - uint32_t frames_so_far = 0; - // frequency and time indices within chunk - uint32_t fi = 0; - uint32_t ti = 0; - // offset for copying into buffer - uint32_t offset = 0; - - // The dataset ID we read from the frame - dset_id_t ds_id; - // The dataset ID of the state without the time axis - dset_id_t base_ds_id; - // String formatted dataset ID to be written to the file - std::string ds_id_str; - - uint64_t frame_size = 0; - - // wait for a non-empty frame to get dataset ID from - uint32_t num_empty_skip = 0; - while (true) { - // Wait for a frame in the input buffer in order to get the dataset ID - if ((wait_for_full_frame(in_buf, unique_name.c_str(), frame_id)) == nullptr) { - return; - } - auto frame = VisFrameView(in_buf, frame_id); - - // Get the frame size to publish metrics later - if (frame_size == 0) - frame_size = frame.calculate_buffer_layout(num_input, num_prod, num_ev).first; - - // If the frame is empty, release the buffer and continue - if (frame.fpga_seq_total == 0 && frame.dataset_id == dset_id_t::null) { - DEBUG("Got empty frame ({:d}).", frame_id); - mark_frame_empty(in_buf, unique_name.c_str(), frame_id++); - num_empty_skip++; - } else { - ds_id = frame.dataset_id; - break; - } - } - - if (num_empty_skip > 0) { - INFO("Found {:d} empty frames at the start of the file.", num_empty_skip); - } - - if (!get_dataset_state(ds_id)) { - FATAL_ERROR("Couldn't find ancestor of dataset {}. " - "Make sure there is a stage upstream in the config, that adds the dataset " - "states.\nExiting...", - ds_id); - return; - } - - // Get the original dataset ID (before adding time axis) - base_ds_id = base_dset(ds_id); - - // Once the async get_dataset_state() is done, we have all the metadata to - // create a file. - - found_flags = std::vector(write_t, false); - - // Create HDF5 file - if (stack.size() > 0) { - file = std::unique_ptr( - new visFileArchive(filename, metadata, times, freqs, inputs, prods, stack, - reverse_stack, num_ev, chunk, kotekan::logLevel(_member_log_level))); - } else { - file = std::unique_ptr( - new visFileArchive(filename, metadata, times, freqs, inputs, prods, num_ev, chunk, - kotekan::logLevel(_member_log_level))); - } - - // TODO: it seems like this should be a Counter? - auto& transposed_bytes_metric = - Metrics::instance().add_gauge("kotekan_vistranspose_data_transposed_bytes", unique_name); - - while (!stop_thread) { - if (num_empty_skip > 0) { - // Write out empty frames that were skipped at start - // All arrays are initialised to zero, so we just need to move through them - num_empty_skip--; - } else { - // Wait for a full frame in the input buffer - if ((wait_for_full_frame(in_buf, unique_name.c_str(), frame_id)) == nullptr) { - break; - } - auto frame = VisFrameView(in_buf, frame_id); - - // Collect frames until a chunk is filled - // Time-transpose as frames come in - // Fastest varying is time (needs to be consistent with reader!) - offset = fi * write_t; - strided_copy(frame.vis.data(), vis.data(), offset * eff_prod_dim + ti, write_t, - eff_prod_dim); - strided_copy(frame.weight.data(), vis_weight.data(), offset * eff_prod_dim + ti, - write_t, eff_prod_dim); - strided_copy(frame.eval.data(), eval.data(), fi * num_ev * write_t + ti, write_t, - num_ev); - strided_copy(frame.evec.data(), evec.data(), fi * num_ev * num_input * write_t + ti, - write_t, num_ev * num_input); - erms[offset + ti] = frame.erms; - frac_lost[offset + ti] = frame.fpga_seq_length == 0 - ? 1. - : 1. - float(frame.fpga_seq_total) / frame.fpga_seq_length; - frac_rfi[offset + ti] = - frame.fpga_seq_length == 0 ? 0. : float(frame.rfi_total) / frame.fpga_seq_length; - strided_copy(frame.gain.data(), gain.data(), offset * num_input + ti, write_t, - num_input); - - // Parse the dataset ID - if (frame.fpga_seq_total == 0 && frame.dataset_id == dset_id_t::null) { - DEBUG("Got an empty frame."); - // Empty frames have a null dataset ID - ds_id_str = fmt::format("{}", dset_id_t::null); - } else if (frame.dataset_id != ds_id) { - // TODO assuming that dataset ID changes here never change dataset dimensions - DEBUG("Dataset ID has changed from {} to {}.", ds_id, frame.dataset_id); - // Update the dataset ID we are writing out - ds_id = frame.dataset_id; - // Store original dataset ID (before adding time axis) - base_ds_id = base_dset(ds_id); - ds_id_str = fmt::format("{}", base_ds_id); - } else { - // Dataset ID hasn't changed - ds_id_str = fmt::format("{}", base_ds_id); - } - if (ds_id_str.length() != DSET_ID_LEN - 1) { - FATAL_ERROR("Formatted dataset ID string does not have expected length."); - return; - } - std::copy(ds_id_str.c_str(), ds_id_str.c_str() + DSET_ID_LEN, - dset_id[offset + ti].hash); - - // Only copy flags if we haven't already - if (!found_flags[ti]) { - // Only update flags if they are non-zero - bool nz_flags = false; - for (uint i = 0; i < num_input; i++) { - if (frame.flags[i] != 0.) { - nz_flags = true; - break; - } - } - if (nz_flags) { - // Copy flags into the buffer. These will not be overwritten until - // the chunks increment in time - strided_copy(frame.flags.data(), input_flags.data(), ti, write_t, num_input); - found_flags[ti] = true; - } - } - - // move to next frame - mark_frame_empty(in_buf, unique_name.c_str(), frame_id++); - } - - // Increment within read chunk - // within a chunk, frequency is the fastest varying index - fi = (fi + 1) % write_f; - if (fi == 0) - ti++; - if (ti == write_t) { - // chunk is complete - write(); - // increment between chunks - increment_chunk(); - fi = 0; - ti = 0; - - // export prometheus metric - transposed_bytes_metric.set(frame_size * frames_so_far); - } - - frames_so_far++; - // Exit when all frames have been written - if (frames_so_far == num_time * num_freq) { - INFO("Done. Exiting."); - exit_kotekan(ReturnCode::CLEAN_EXIT); - return; - } - } -} - -void visTranspose::write() { - DEBUG("Writing at freq {:d} and time {:d}", f_ind, t_ind); - DEBUG("Writing block of {:d} freqs and {:d} times", write_f, write_t); - - file->write_block("vis", f_ind, t_ind, write_f, write_t, vis.data()); - - file->write_block("vis_weight", f_ind, t_ind, write_f, write_t, vis_weight.data()); - - if (num_ev > 0) { - file->write_block("eval", f_ind, t_ind, write_f, write_t, eval.data()); - file->write_block("evec", f_ind, t_ind, write_f, write_t, evec.data()); - file->write_block("erms", f_ind, t_ind, write_f, write_t, erms.data()); - } - - file->write_block("gain", f_ind, t_ind, write_f, write_t, gain.data()); - - file->write_block("flags/frac_lost", f_ind, t_ind, write_f, write_t, frac_lost.data()); - - file->write_block("flags/frac_rfi", f_ind, t_ind, write_f, write_t, frac_rfi.data()); - - file->write_block("flags/inputs", f_ind, t_ind, write_f, write_t, input_flags.data()); - - file->write_block("flags/dataset_id", f_ind, t_ind, write_f, write_t, dset_id.data()); -} - -// increment between chunks -// chunks come in (time, freq) order -// WARNING: This order must be consistent with how visRawReader -// implements chunked reads. The mechanism for avoiding -// overwriting flags also relies on this ordering. -void visTranspose::increment_chunk() { - // Figure out where the next chunk starts - f_ind = f_edge ? 0 : (f_ind + chunk_f) % num_freq; - if (f_ind == 0) { - // set incomplete chunk flag - f_edge = (num_freq < chunk_f); - t_ind += chunk_t; - // clear flags buffer for next time chunk - std::fill(input_flags.begin(), input_flags.end(), 0.); - std::fill(found_flags.begin(), found_flags.end(), false); - if (num_time - t_ind < chunk_t) { - // Reached an incomplete chunk - t_edge = true; - } - } else if (num_freq - f_ind < chunk_f) { - // Reached an incomplete chunk - f_edge = true; - } - // Determine size of next chunk - write_f = f_edge ? num_freq - f_ind : chunk_f; - write_t = t_edge ? num_time - t_ind : chunk_t; -} - -dset_id_t visTranspose::base_dset(dset_id_t ds_id) { - - datasetManager& dm = datasetManager::instance(); - - try { - return dm.datasets().at(ds_id).base_dset(); - } catch (std::out_of_range& e) { - DEBUG("Fetching metadata state..."); - // fetch a metadata state just to ensure we have a copy of that dataset - auto mstate_fut = std::async(&datasetManager::dataset_state, &dm, ds_id); - auto ready = mstate_fut.wait_for(timeout); - if (ready == std::future_status::timeout) { - ERROR("Communication with dataset broker timed out for datatset id {}.", ds_id); - dm.stop(); - exit_kotekan(ReturnCode::DATASET_MANAGER_FAILURE); - return ds_id; - } - const metadataState* mstate = mstate_fut.get(); - (void)mstate; - try { - return dm.datasets().at(ds_id).base_dset(); - } catch (std::out_of_range& e) { - FATAL_ERROR("Failed to get base dataset of dataset with ID {}. {}", ds_id, e.what()); - return ds_id; - } - } -} diff --git a/lib/testing/FakeHFB.cpp b/lib/testing/FakeHFB.cpp index 19a3f1105..53fc44eed 100644 --- a/lib/testing/FakeHFB.cpp +++ b/lib/testing/FakeHFB.cpp @@ -156,16 +156,10 @@ void FakeHFB::main_thread() { output_frame.fpga_seq_length = delta_seq; output_frame.fpga_seq_total = delta_seq; - // Fill out the frame with the selected pattern - // pattern->fill(output_frame); - for (uint32_t i = 0; i < output_frame.num_beams * output_frame.num_subfreq; i++) { - float data = (i % 2 == 0 ? output_frame.freq_id : output_frame.fpga_seq_start); - - output_frame.hfb[i] = data; - // Set weights to zero for now - output_frame.weight[i] = 0.0; + output_frame.hfb[i] = output_frame.freq_id; + output_frame.weight[i] = 1.0; } @@ -244,9 +238,9 @@ void ReplaceHFB::main_thread() { float data = (i % 2 == 0 ? output_frame.freq_id : output_frame.fpga_seq_start); output_frame.hfb[i] = data; + output_frame.weight[i] = 1.f; } - // Mark the output buffer and move on mark_frame_full(out_buf, unique_name.c_str(), output_frame_id); diff --git a/lib/testing/fakeGpu.cpp b/lib/testing/fakeGpu.cpp index 4bd806da7..7e85ea845 100644 --- a/lib/testing/fakeGpu.cpp +++ b/lib/testing/fakeGpu.cpp @@ -200,6 +200,10 @@ uint32_t FakeTelescope::num_freq() const { return 1024; } +uint8_t FakeTelescope::nyquist_zone() const { + return 2; +} + timespec FakeTelescope::to_time(uint64_t /*seq*/) const { return {0, 0}; } diff --git a/lib/testing/fakeGpu.hpp b/lib/testing/fakeGpu.hpp index 1f5bbf553..d7c59e1e9 100644 --- a/lib/testing/fakeGpu.hpp +++ b/lib/testing/fakeGpu.hpp @@ -107,6 +107,7 @@ class FakeTelescope : public Telescope { double freq_width(freq_id_t freq_id) const override; uint32_t num_freq_per_stream() const override; uint32_t num_freq() const override; + uint8_t nyquist_zone() const override; // Dummy time map implementations timespec to_time(uint64_t seq) const override; diff --git a/lib/utils/CMakeLists.txt b/lib/utils/CMakeLists.txt index e44b82f92..24fcce1b8 100644 --- a/lib/utils/CMakeLists.txt +++ b/lib/utils/CMakeLists.txt @@ -36,7 +36,7 @@ target_include_directories(kotekan_utils PUBLIC .) # HDF5 stuff if(${USE_HDF5}) - target_sources(kotekan_utils PRIVATE visFileH5.cpp visFileArchive.cpp) + target_sources(kotekan_utils PRIVATE visFileH5.cpp visFileArchive.cpp HFBFileArchive.cpp) target_include_directories(kotekan_utils SYSTEM INTERFACE ${HDF5_INCLUDE_DIRS} ${HIGHFIVE_PATH}/include) target_link_libraries(kotekan_utils PRIVATE ${HDF5_HL_LIBRARIES} ${HDF5_LIBRARIES}) diff --git a/lib/utils/FileArchive.hpp b/lib/utils/FileArchive.hpp new file mode 100644 index 000000000..d249f14e2 --- /dev/null +++ b/lib/utils/FileArchive.hpp @@ -0,0 +1,25 @@ +#ifndef FILE_ARCHIVE_HPP +#define FILE_ARCHIVE_HPP + +#include "kotekanLogging.hpp" // for logLevel, kotekanLogging + +#include // for H5Pcreate, H5Pset_chunk, H5Pset_filter, H5P_DATAS... + +/** @brief A Bitshuffle header file. + * + * Header file to store common Bitshuffle constants. + * + * @author James Willis + **/ +class FileArchive : public kotekan::kotekanLogging { + +protected: + // Bitshuffle parameters + H5Z_filter_t H5Z_BITSHUFFLE = 32008; + unsigned int BSHUF_H5_COMPRESS_LZ4 = 2; + unsigned int BSHUF_BLOCK = 0; // let bitshuffle choose + + const std::vector BSHUF_CD = {BSHUF_BLOCK, BSHUF_H5_COMPRESS_LZ4}; +}; + +#endif diff --git a/lib/utils/H5Support.hpp b/lib/utils/H5Support.hpp new file mode 100644 index 000000000..0e11be5b1 --- /dev/null +++ b/lib/utils/H5Support.hpp @@ -0,0 +1,99 @@ +#ifndef H5_SUPPORT_HPP +#define H5_SUPPORT_HPP + +#include "visUtil.hpp" // for freq_ctype, prod_ctype, time_ctype, input_ctype + +#include // for DataType, AtomicType, DataType::DataType + +using namespace HighFive; + +const size_t DSET_ID_LEN = 33; // Length of the string used to represent dataset IDs +struct dset_id_str { + char hash[DSET_ID_LEN]; +}; + +namespace HighFive { +// \cond NO_DOC +// Fixed length string to store dataset ID +template<> +inline AtomicType::AtomicType() { + _hid = H5Tcopy(H5T_C_S1); + H5Tset_size(_hid, DSET_ID_LEN); +} +// \endcond +}; // namespace HighFive + +// These templated functions are needed in order to tell HighFive how the +// various structs are converted into HDF5 datatypes +// \cond NO_DOC +template<> +inline DataType HighFive::create_datatype() { + CompoundType f; + f.addMember("centre", H5T_IEEE_F64LE); + f.addMember("width", H5T_IEEE_F64LE); + f.autoCreate(); + return std::move(f); +} + +template<> +inline DataType HighFive::create_datatype() { + CompoundType t; + t.addMember("fpga_count", H5T_STD_U64LE); + t.addMember("ctime", H5T_IEEE_F64LE); + t.autoCreate(); + return std::move(t); +} + +template<> +inline DataType HighFive::create_datatype() { + + CompoundType i; + hid_t s32 = H5Tcopy(H5T_C_S1); + H5Tset_size(s32, 32); + // AtomicType s32; + i.addMember("chan_id", H5T_STD_U16LE, 0); + i.addMember("correlator_input", s32, 2); + i.manualCreate(34); + + return std::move(i); +} + +template<> +inline DataType HighFive::create_datatype() { + + CompoundType p; + p.addMember("input_a", H5T_STD_U16LE); + p.addMember("input_b", H5T_STD_U16LE); + p.autoCreate(); + return std::move(p); +} + +template<> +inline DataType HighFive::create_datatype() { + CompoundType c; + c.addMember("r", H5T_IEEE_F32LE); + c.addMember("i", H5T_IEEE_F32LE); + c.autoCreate(); + return std::move(c); +} + +template<> +inline DataType HighFive::create_datatype() { + CompoundType c; + c.addMember("stack", H5T_STD_U32LE); + c.addMember("conjugate", H5T_STD_U8LE); + c.autoCreate(); + return std::move(c); +} + +template<> +inline DataType HighFive::create_datatype() { + CompoundType c; + c.addMember("prod", H5T_STD_U32LE); + c.addMember("conjugate", H5T_STD_U8LE); + c.autoCreate(); + return std::move(c); +} +// \endcond + +#endif diff --git a/lib/utils/HFBFileArchive.cpp b/lib/utils/HFBFileArchive.cpp new file mode 100644 index 000000000..6dfba7771 --- /dev/null +++ b/lib/utils/HFBFileArchive.cpp @@ -0,0 +1,290 @@ +#include "HFBFileArchive.hpp" + +#include "H5Support.hpp" // for AtomicType<>::AtomicType, dset_id_str +#include "visFile.hpp" // for create_lockfile + +#include "fmt.hpp" // for format, fmt + +#include // for copy, min +#include // for remove +#include // for Attribute, Attribute::write +#include // for DataSet, AnnotateTraits::createAttribute, DataSet... +#include // for DataSpace::From, DataSpace, DataSpace::getDimensions +#include // for CompoundType, create_datatype, CompoundType::addM... +#include // for DataSpaceException, HDF5ErrMapper +#include // for File, NodeTraits::createGroup, File::flush, NodeT... +#include // for Group +#include // for hid_t +#include // for H5Pcreate, H5Pset_chunk, H5Pset_filter, H5P_DATAS... +#include // for SliceTraits::select, Selection, SliceTraits::write +#include // for invalid_argument +#include // for make_tuple, get, tuple +#include // for __decay_and_strip<>::__type +#include // for move, pair + +using namespace HighFive; + + +// Create an archive file for uncompressed products +HFBFileArchive::HFBFileArchive(const std::string& name, + const std::map& metadata, + const std::vector& times, + const std::vector& freqs, + const std::vector& beams, + const std::vector& subfreqs, std::vector chunk_size, + const kotekan::logLevel log_level) { + + set_log_level(log_level); + + // Check axes and create file + setup_file(name, metadata, times, freqs, beams, subfreqs, chunk_size); + + // Make datasets + create_axes(times, freqs, beams, subfreqs); + create_datasets(); +} + + +// Create an archive file for baseline-stacked products +HFBFileArchive::HFBFileArchive( + const std::string& name, const std::map& metadata, + const std::vector& times, const std::vector& freqs, + const std::vector& beams, const std::vector& subfreqs, + const std::vector& stack, std::vector& reverse_stack, + std::vector chunk_size, const kotekan::logLevel log_level) { + + set_log_level(log_level); + + // Check axes and create file + setup_file(name, metadata, times, freqs, beams, subfreqs, chunk_size); + // Different bound check for stacked data + if (chunk[1] > (int)stack.size()) { + chunk[1] = stack.size(); + INFO("HFBFileArchive: Chunk stack dimension greater than axes. Will use a smaller chunk.") + } + + // Make datasets, for stacked data + stacked = true; + create_axes(times, freqs, beams, subfreqs, stack); + create_datasets(); + + // Write the reverse map of products to stack + dset("reverse_map/stack").select({0}, {length("prod")}).write(reverse_stack.data()); +} + + +void HFBFileArchive::setup_file(const std::string& name, + const std::map& metadata, + const std::vector& times, + const std::vector& freqs, + const std::vector& beams, + const std::vector& subfreqs, std::vector chunk_size) { + + std::string data_filename = fmt::format(fmt("{:s}.h5"), name); + + lock_filename = create_lockfile(data_filename); + + // Set HDF5 chunk size + chunk = chunk_size; + // Check chunk size + if (chunk[0] < 1 || chunk[1] < 1 || chunk[2] < 1 || chunk[3] < 1 || chunk[4] < 1) + throw std::invalid_argument( + fmt::format(fmt("HFBFileArchive: config: Chunk size needs to " + "be greater or equal to (1,1,1,1,1) (is ({:d},{:d}," + "{:d},{:d},{:d}))."), + chunk[0], chunk[1], chunk[2], chunk[3], chunk[4])); + if (chunk[0] > (int)freqs.size()) { + + INFO("HFBFileArchive: Chunk frequency ({}) dimension greater than axes ({}). Will use a " + "smaller " + "chunk.", + chunk[0], freqs.size()) + chunk[0] = freqs.size(); + } + if (chunk[2] > (int)times.size()) { + + INFO("HFBFileArchive: Chunk time ({}) dimension greater than axes({}). Will use a smaller " + "chunk.", + chunk[2], times.size()) + chunk[2] = times.size(); + } + if (chunk[3] > (int)beams.size()) { + + INFO("HFBFileArchive: Chunk beam ({}) dimension greater than axes ({}). Will use a smaller " + "chunk.", + chunk[3], beams.size()) + chunk[3] = beams.size(); + } + if (chunk[4] > (int)subfreqs.size()) { + + INFO("HFBFileArchive: Chunk sub-frequency ({}) dimension greater than axes ({}). Will use " + "a smaller " + "chunk.", + chunk[4], subfreqs.size()) + chunk[4] = subfreqs.size(); + } + + INFO("Creating new archive file {:s}", name); + + file = std::unique_ptr( + new File(data_filename, File::ReadWrite | File::Create | File::Truncate)); + + // Write out metadata into flle + for (auto item : metadata) { + file->createAttribute(item.first, DataSpace::From(item.second)) + .write(item.second); + } + + // Get weight type flag + weight_type = metadata.at("weight_type"); +} + + +template +void HFBFileArchive::write_block(std::string name, size_t f_ind, size_t t_ind, size_t chunk_f, + size_t chunk_t, const T* data) { + DEBUG2("Writing {}...", name); + if (name == "flags/frac_lost" || name == "flags/frac_rfi" || name == "flags/dataset_id") { + dset(name).select({f_ind, t_ind}, {chunk_f, chunk_t}).write(data); + } else { + size_t subfreq_last_dim = dset(name).getSpace().getDimensions().at(1); + size_t beam_last_dim = dset(name).getSpace().getDimensions().at(2); + // DEBUG("writing {:d} freq, {:d} times, {:d} beams, {:d} sub-freq at ({:d}, 0, 0, {:d}). + // Data[0]: {}", chunk_f, chunk_t, beam_last_dim, subfreq_last_dim, f_ind, t_ind, data[0]); + dset(name) + .select({f_ind, 0, 0, t_ind}, {chunk_f, subfreq_last_dim, beam_last_dim, chunk_t}) + .write(data); + } +} + +// Instantiate for types that will get used to satisfy linker +template void HFBFileArchive::write_block>(std::string name, size_t f_ind, + size_t t_ind, size_t chunk_f, + size_t chunk_t, + std::complex const*); +template void HFBFileArchive::write_block(std::string name, size_t f_ind, size_t t_ind, + size_t chunk_f, size_t chunk_t, float const*); +template void HFBFileArchive::write_block(std::string name, size_t f_ind, size_t t_ind, + size_t chunk_f, size_t chunk_t, int const*); +template void HFBFileArchive::write_block(std::string name, size_t f_ind, size_t t_ind, + size_t chunk_f, size_t chunk_t, + dset_id_str const*); + +HFBFileArchive::~HFBFileArchive() { + file->flush(); + file.reset(nullptr); + std::remove(lock_filename.c_str()); +} + +void HFBFileArchive::create_axes(const std::vector& times, + const std::vector& freqs, + const std::vector& beams, + const std::vector& subfreqs) { + + create_axis("freq", freqs); + create_axis("time", times); + create_axis("beam", beams); + create_axis("subfreq", subfreqs); +} + +void HFBFileArchive::create_axes(const std::vector& times, + const std::vector& freqs, + const std::vector& beams, + const std::vector& subfreqs, + const std::vector& stack) { + + create_axes(times, freqs, beams, subfreqs); + + create_axis("stack", stack); +} + +template +void HFBFileArchive::create_axis(std::string name, const std::vector& axis) { + + Group indexmap = + file->exist("index_map") ? file->getGroup("index_map") : file->createGroup("index_map"); + + DataSet index = indexmap.createDataSet(name, DataSpace(axis.size())); + index.write(axis); +} + +void HFBFileArchive::create_datasets() { + + Group flags = file->createGroup("flags"); + + bool compress = true; + bool no_compress = false; + + // Create transposed dataset shapes + create_dataset("hfb", {"freq", "subfreq", "beam", "time"}, create_datatype(), compress); + create_dataset("flags/hfb_weight", {"freq", "subfreq", "beam", "time"}, + create_datatype(), compress); + create_dataset("flags/frac_lost", {"freq", "time"}, create_datatype(), no_compress); + create_dataset("flags/dataset_id", {"freq", "time"}, create_datatype(), + no_compress); + + // Add weight type flag where gossec expects it + dset("hfb_weight") + .createAttribute("type", DataSpace::From(weight_type)) + .write(weight_type); + + file->flush(); +} + +void HFBFileArchive::create_dataset(const std::string& name, const std::vector& axes, + DataType type, const bool& compress) { + + // Mapping of axis names to sizes (start, chunk) + std::map> size_map; + size_map["freq"] = std::make_tuple(length("freq"), chunk[0]); + size_map["beam"] = std::make_tuple(length("beam"), chunk[3]); + size_map["time"] = std::make_tuple(length("time"), chunk[2]); + size_map["subfreq"] = std::make_tuple(length("subfreq"), chunk[4]); + if (stacked) + size_map["stack"] = std::make_tuple(length("stack"), chunk[1]); + + std::vector cur_dims, max_dims, chunk_dims; + + for (auto axis : axes) { + auto cs = size_map[axis]; + cur_dims.push_back(std::get<0>(cs)); + chunk_dims.push_back(std::get<1>(cs)); + } + + DataSpace space = DataSpace(cur_dims); + + if (compress) { + // Add chunking and bitshuffle filter to plist + // Pulled this out of HighFive createDataSet source + std::vector real_chunk(chunk_dims.size()); + std::copy(chunk_dims.begin(), chunk_dims.end(), real_chunk.begin()); + // Set dataset creation properties to enable chunking + hid_t plist = H5Pcreate(H5P_DATASET_CREATE); + if (H5Pset_chunk(plist, int(chunk_dims.size()), &(real_chunk.at(0))) < 0) { + HDF5ErrMapper::ToException("Failed trying to create chunk."); + } + // Set bitshuffle compression filter + if (H5Pset_filter(plist, H5Z_BITSHUFFLE, H5Z_FLAG_MANDATORY, BSHUF_CD.size(), + BSHUF_CD.data()) + < 0) { + HDF5ErrMapper::ToException( + "Failed trying to set bishuffle filter."); + } + + DataSet dset = file->createDataSet(name, space, type, plist); + dset.createAttribute("axis", DataSpace::From(axes)).write(axes); + } else { + DataSet dset = file->createDataSet(name, space, type, chunk_dims); + dset.createAttribute("axis", DataSpace::From(axes)).write(axes); + } +} + +// Quick functions for fetching datasets and dimensions +DataSet HFBFileArchive::dset(const std::string& name) { + const std::string dset_name = name == "hfb_weight" ? "flags/hfb_weight" : name; + return file->getDataSet(dset_name); +} + +size_t HFBFileArchive::length(const std::string& axis_name) { + return dset(fmt::format(fmt("index_map/{:s}"), axis_name)).getSpace().getDimensions()[0]; +} diff --git a/lib/utils/HFBFileArchive.hpp b/lib/utils/HFBFileArchive.hpp new file mode 100644 index 000000000..e3b69f7d7 --- /dev/null +++ b/lib/utils/HFBFileArchive.hpp @@ -0,0 +1,133 @@ +#ifndef HFB_FILE_ARCHIVE_HPP +#define HFB_FILE_ARCHIVE_HPP + +#include "FileArchive.hpp" +#include "kotekanLogging.hpp" // for logLevel, kotekanLogging +#include "visUtil.hpp" // for freq_ctype, prod_ctype, time_ctype, input_ctype + +#include // for DataSet +#include // for DataType +#include // for File +#include // for map +#include // for allocator, unique_ptr +#include // for size_t +#include // for uint32_t +#include // for string +#include // for vector + + +/** @brief A CHIME correlator archive file. + * + * The class creates and manages writes to a CHIME style correlator archive + * file in the standard HDF5 format. It also manages the lock file. + * + * @author James Willis + **/ +class HFBFileArchive : public FileArchive { + +public: + /** + * @brief Creates a HFBFileArchive object. + * + * @param name Path of the file to write into (without file extension). + * @param metadata Metadata attributes. + * @param times Vector of time indices. + * @param freqs Vector of frequency indices. + * @param beams Vector of beam indices. + * @param subfreqs Vector of sub-frequency differences from centre of sub-band. + * @param chunk_size HDF5 chunk size (frequencies * beams * sub-frequencies * times). + * @param log_level kotekan log level for any logging generated by the HFBFileArchive instance + * + * @return Instance of HFBFileArchive. + **/ + HFBFileArchive(const std::string& name, const std::map& metadata, + const std::vector& times, const std::vector& freqs, + const std::vector& beams, const std::vector& subfreqs, + std::vector chunk_size, const kotekan::logLevel log_level); + /** + * @brief Creates a HFBFileArchive object. + * + * @param name Path of the file to write into (without file extension). + * @param metadata Metadata attributes. + * @param times Vector of time indices. + * @param freqs Vector of frequency indices. + * @param beams Vector of beam indices. + * @param subfreqs Vector of sub-frequency differences from centre of sub-band. + * @param stack Vector of stack indices. + * @param reverse_stack Vector mapping products to stacks. + * @param chunk_size HDF5 chunk size (frequencies * beams * sub-frequencies * times). + * @param log_level kotekan log level for any logging generated by the HFBFileArchive instance + * + * @return Instance of HFBFileArchive. + **/ + HFBFileArchive(const std::string& name, const std::map& metadata, + const std::vector& times, const std::vector& freqs, + const std::vector& beams, const std::vector& subfreqs, + const std::vector& stack, std::vector& reverse_stack, + std::vector chunk_size, const kotekan::logLevel log_level); + + /** + * @brief Destructor. + **/ + virtual ~HFBFileArchive(); + + /** + * @brief Write a block in time/freq. + * + * @param name Path of the file to write into (without file extension). + * @param f_ind Frequency index. + * @param t_ind Time index. + * @param chunk_f Size of chunk in frequency dimension. + * @param chunk_t Size of chunk in time dimension. + * @param data Pointer to the data. + **/ + template + void write_block(std::string name, size_t f_ind, size_t t_ind, size_t chunk_f, size_t chunk_t, + const T* data); + + +private: + // Prepare a file + void setup_file(const std::string& name, const std::map& metadata, + const std::vector& times, const std::vector& freqs, + const std::vector& beams, const std::vector& subfreqs, + std::vector chunk_size); + + // Helper to create datasets + virtual void create_dataset(const std::string& name, const std::vector& axes, + HighFive::DataType type, const bool& compress); + + // Helper function to create an axis + template + void create_axis(std::string name, const std::vector& axis); + + // Create the index maps from the frequencies and the inputs + void create_axes(const std::vector& times, const std::vector& freqs, + const std::vector& beams, const std::vector& subfreqs); + void create_axes(const std::vector& times, const std::vector& freqs, + const std::vector& beams, const std::vector& subfreqs, + const std::vector& stack); + + // Create the main 21cm absorber holding datasets + void create_datasets(); + + // Get datasets + HighFive::DataSet dset(const std::string& name); + size_t length(const std::string& axis_name); + + // Description of weights stored in hfb_weight dataset + std::string weight_type; + + // HDF5 chunk size + std::vector chunk; + + // Pointer to the underlying HighFive file + std::unique_ptr file; + + std::string lock_filename; + + // Whether the products have been compressed via baseline stacking + bool stacked = false; +}; + +#endif diff --git a/lib/utils/ICETelescope.cpp b/lib/utils/ICETelescope.cpp index 060579043..76ffa55fa 100644 --- a/lib/utils/ICETelescope.cpp +++ b/lib/utils/ICETelescope.cpp @@ -55,6 +55,7 @@ void ICETelescope::set_sampling_params(double sample_rate, uint32_t fft_length, freq0_MHz = (zone / 2) * sample_rate; df_MHz = (zone % 2 ? 1 : -1) * sample_rate / fft_length; nfreq = fft_length / 2; + ny_zone = zone; // TODO: revisit this if we think the length might ever not be an integer // number of ns @@ -186,6 +187,10 @@ uint64_t ICETelescope::seq_length_nsec() const { return dt_ns; } +uint8_t ICETelescope::nyquist_zone() const { + return ny_zone; +} + ice_stream_id_t ice_extract_stream_id(const stream_t encoded_stream_id) { ice_stream_id_t stream_id; diff --git a/lib/utils/ICETelescope.hpp b/lib/utils/ICETelescope.hpp index 1d339230e..d69a7c9c4 100644 --- a/lib/utils/ICETelescope.hpp +++ b/lib/utils/ICETelescope.hpp @@ -36,6 +36,7 @@ class ICETelescope : public Telescope { uint32_t num_freq_per_stream() const override; uint32_t num_freq() const override; double freq_width(freq_id_t freq_id) const override; + uint8_t nyquist_zone() const override; // Implementations of the required time mapping functions bool gps_time_enabled() const override; @@ -87,6 +88,7 @@ class ICETelescope : public Telescope { double freq0_MHz; double df_MHz; uint32_t nfreq; + uint8_t ny_zone; /// Should we try to get the GPS time from remote server bool _query_gps; diff --git a/lib/utils/Telescope.hpp b/lib/utils/Telescope.hpp index 2671f4688..773fabb2d 100644 --- a/lib/utils/Telescope.hpp +++ b/lib/utils/Telescope.hpp @@ -154,6 +154,13 @@ class Telescope : public kotekan::kotekanLogging { **/ virtual double freq_width(freq_id_t freq_id) const = 0; + /** + * @brief Get which Nyquist zone we are in. + * + * @return The Nyquist zone. + **/ + virtual uint8_t nyquist_zone() const = 0; + /** * Convert a sequence number into a UNIX epoch time. * diff --git a/lib/utils/visFileArchive.cpp b/lib/utils/visFileArchive.cpp index 7f92c195e..a4f938b7e 100644 --- a/lib/utils/visFileArchive.cpp +++ b/lib/utils/visFileArchive.cpp @@ -1,7 +1,7 @@ #include "visFileArchive.hpp" +#include "H5Support.hpp" // for AtomicType<>::AtomicType, dset_id_str #include "visFile.hpp" // for create_lockfile -#include "visFileH5.hpp" // for create_datatype, AtomicType<>::AtomicType, dset_i... #include "fmt.hpp" // for format, fmt @@ -27,13 +27,6 @@ using namespace HighFive; -// Bitshuffle parameters -H5Z_filter_t H5Z_BITSHUFFLE = 32008; -unsigned int BSHUF_H5_COMPRESS_LZ4 = 2; -unsigned int BSHUF_BLOCK = 0; // let bitshuffle choose -const std::vector BSHUF_CD = {BSHUF_BLOCK, BSHUF_H5_COMPRESS_LZ4}; - - // Create an archive file for uncompressed products visFileArchive::visFileArchive(const std::string& name, const std::map& metadata, @@ -320,75 +313,3 @@ size_t visFileArchive::length(const std::string& axis_name) { return 0; return dset(fmt::format(fmt("index_map/{:s}"), axis_name)).getSpace().getDimensions()[0]; } - - -// TODO: these should be included from visFileH5 -// Add support for all our custom types to HighFive -template<> -inline DataType HighFive::create_datatype() { - CompoundType f; - f.addMember("centre", H5T_IEEE_F64LE); - f.addMember("width", H5T_IEEE_F64LE); - f.autoCreate(); - return std::move(f); -} - -template<> -inline DataType HighFive::create_datatype() { - CompoundType t; - t.addMember("fpga_count", H5T_STD_U64LE); - t.addMember("ctime", H5T_IEEE_F64LE); - t.autoCreate(); - return std::move(t); -} - -template<> -inline DataType HighFive::create_datatype() { - - CompoundType i; - hid_t s32 = H5Tcopy(H5T_C_S1); - H5Tset_size(s32, 32); - // AtomicType s32; - i.addMember("chan_id", H5T_STD_U16LE, 0); - i.addMember("correlator_input", s32, 2); - i.manualCreate(34); - - return std::move(i); -} - -template<> -inline DataType HighFive::create_datatype() { - - CompoundType p; - p.addMember("input_a", H5T_STD_U16LE); - p.addMember("input_b", H5T_STD_U16LE); - p.autoCreate(); - return std::move(p); -} - -template<> -inline DataType HighFive::create_datatype() { - CompoundType c; - c.addMember("r", H5T_IEEE_F32LE); - c.addMember("i", H5T_IEEE_F32LE); - c.autoCreate(); - return std::move(c); -} - -template<> -inline DataType HighFive::create_datatype() { - CompoundType c; - c.addMember("stack", H5T_STD_U32LE); - c.addMember("conjugate", H5T_STD_U8LE); - c.autoCreate(); - return std::move(c); -} - -template<> -inline DataType HighFive::create_datatype() { - CompoundType c; - c.addMember("prod", H5T_STD_U32LE); - c.addMember("conjugate", H5T_STD_U8LE); - c.autoCreate(); - return std::move(c); -} diff --git a/lib/utils/visFileArchive.hpp b/lib/utils/visFileArchive.hpp index a2fd7ea92..8e41ddb14 100644 --- a/lib/utils/visFileArchive.hpp +++ b/lib/utils/visFileArchive.hpp @@ -1,6 +1,7 @@ #ifndef VIS_FILE_ARCHIVE_HPP #define VIS_FILE_ARCHIVE_HPP +#include "FileArchive.hpp" #include "kotekanLogging.hpp" // for logLevel, kotekanLogging #include "visUtil.hpp" // for freq_ctype, prod_ctype, time_ctype, input_ctype @@ -21,7 +22,7 @@ * * @author Richard Shaw **/ -class visFileArchive : public kotekan::kotekanLogging { +class visFileArchive : public FileArchive { public: /** @@ -143,26 +144,4 @@ inline std::string visFileArchive::prod_or_stack() { return stacked ? "stack" : "prod"; } - -// TODO: these should be included from visFileH5 -// These templated functions are needed in order to tell HighFive how the -// various structs are converted into HDF5 datatypes -namespace HighFive { -template<> -DataType create_datatype(); -template<> -DataType create_datatype(); -template<> -DataType create_datatype(); -template<> -DataType create_datatype(); -template<> -DataType create_datatype(); -template<> -DataType create_datatype(); -template<> -DataType create_datatype(); -}; // namespace HighFive - - #endif diff --git a/lib/utils/visFileH5.cpp b/lib/utils/visFileH5.cpp index a403439ab..74f8de715 100644 --- a/lib/utils/visFileH5.cpp +++ b/lib/utils/visFileH5.cpp @@ -1,6 +1,7 @@ #include "visFileH5.hpp" +#include "H5Support.hpp" // for AtomicType<>::AtomicType, dset_id_str #include "Hash.hpp" // for Hash #include "datasetManager.hpp" // for datasetManager, dset_id_t #include "datasetState.hpp" // for eigenvalueState, freqState, inputState, prodState @@ -32,7 +33,6 @@ #include // for fstat, stat #include // for system_error #include // for make_tuple, tuple, get -#include // for remove_reference<>::type #include // for pwrite, TEMP_FAILURE_RETRY #include // for move, pair @@ -522,56 +522,3 @@ void visFileH5Fast::write_sample(uint32_t time_ind, uint32_t freq_ind, size_t visFileH5Fast::num_time() { return ntime; } - - -// Add support for all our custom types to HighFive -template<> -DataType HighFive::create_datatype() { - CompoundType f; - f.addMember("centre", H5T_IEEE_F64LE); - f.addMember("width", H5T_IEEE_F64LE); - f.autoCreate(); - return std::move(f); -} - -template<> -DataType HighFive::create_datatype() { - CompoundType t; - t.addMember("fpga_count", H5T_STD_U64LE); - t.addMember("ctime", H5T_IEEE_F64LE); - t.autoCreate(); - return std::move(t); -} - -template<> -DataType HighFive::create_datatype() { - - CompoundType i; - hid_t s32 = H5Tcopy(H5T_C_S1); - H5Tset_size(s32, 32); - // AtomicType s32; - i.addMember("chan_id", H5T_STD_U16LE, 0); - i.addMember("correlator_input", s32, 2); - i.manualCreate(34); - - return std::move(i); -} - -template<> -DataType HighFive::create_datatype() { - - CompoundType p; - p.addMember("input_a", H5T_STD_U16LE); - p.addMember("input_b", H5T_STD_U16LE); - p.autoCreate(); - return std::move(p); -} - -template<> -DataType HighFive::create_datatype() { - CompoundType c; - c.addMember("r", H5T_IEEE_F32LE); - c.addMember("i", H5T_IEEE_F32LE); - c.autoCreate(); - return std::move(c); -} diff --git a/lib/utils/visFileH5.hpp b/lib/utils/visFileH5.hpp index b117de8ef..c6b39596e 100644 --- a/lib/utils/visFileH5.hpp +++ b/lib/utils/visFileH5.hpp @@ -251,34 +251,4 @@ class visFileH5Fast : public visFileH5 { erms_offset, time_offset; }; - -// These templated functions are needed in order to tell HighFive how the -// various structs are converted into HDF5 datatypes -const size_t DSET_ID_LEN = 33; // Length of the string used to represent dataset IDs -struct dset_id_str { - char hash[DSET_ID_LEN]; -}; -namespace HighFive { -template<> -DataType create_datatype(); -template<> -DataType create_datatype(); -template<> -DataType create_datatype(); -template<> -DataType create_datatype(); -template<> -DataType create_datatype(); - -// \cond NO_DOC -// Fixed length string to store dataset ID -template<> -inline AtomicType::AtomicType() { - _hid = H5Tcopy(H5T_C_S1); - H5Tset_size(_hid, DSET_ID_LEN); -} -// \endcond -}; // namespace HighFive - - #endif diff --git a/tests/boost/test_transpose.cpp b/tests/boost/test_transpose.cpp index 92d0c617a..74f18da14 100644 --- a/tests/boost/test_transpose.cpp +++ b/tests/boost/test_transpose.cpp @@ -1,6 +1,6 @@ -#define BOOST_TEST_MODULE "test_visTranspose" +#define BOOST_TEST_MODULE "test_VisTranspose" -#include "visTranspose.hpp" // for strided_copy +#include "Transpose.hpp" // for strided_copy #include // for BOOST_PP_IIF_1, BOOST_PP_IIF_0, BOOST_PP_BO... #include // for uint32_t, uint8_t diff --git a/tests/test_transpose.py b/tests/test_transpose.py index 8fc5c1ef5..269521ee1 100644 --- a/tests/test_transpose.py +++ b/tests/test_transpose.py @@ -232,7 +232,7 @@ def transpose(tmpdir_factory, cal_broker): raw_buf = runner.ReadRawBuffer(infile, writer_params["chunk_size"]) outfile = tmpdir + "/transposed" transposer = runner.KotekanStageTester( - "visTranspose", + "VisTranspose", { "outfile": outfile, "chunk_size": writer_params["chunk_size"], @@ -361,7 +361,7 @@ def transpose_stack(tmpdir_factory): raw_buf = runner.ReadRawBuffer(infile, stack_params["chunk_size"]) outfile = tmpdir + "/transposed" transposer = runner.KotekanStageTester( - "visTranspose", + "VisTranspose", { "outfile": outfile, "infile": infile, From 43891a6e81e250e98d1b20a2640fe233b249f340 Mon Sep 17 00:00:00 2001 From: james-s-willis <36906682+james-s-willis@users.noreply.github.com> Date: Tue, 9 Feb 2021 15:14:17 -0500 Subject: [PATCH 04/15] Created HFB truncate stage (#915) * Created HFBTruncate stage. * Optimise vectorisation in truncation stage with constant vector. * Renamed visTruncate -> VisTruncate. * Use FATAL_ERROR instead of invalid_argument. --- config/tests/test_HFBRawReader.yaml | 23 ++- lib/stages/CMakeLists.txt | 2 +- lib/stages/HFBTruncate.cpp | 135 ++++++++++++++++++ lib/stages/HFBTruncate.hpp | 61 ++++++++ .../{visTruncate.cpp => VisTruncate.cpp} | 31 ++-- .../{visTruncate.hpp => VisTruncate.hpp} | 8 +- tests/test_truncate.py | 4 +- 7 files changed, 237 insertions(+), 27 deletions(-) create mode 100644 lib/stages/HFBTruncate.cpp create mode 100644 lib/stages/HFBTruncate.hpp rename lib/stages/{visTruncate.cpp => VisTruncate.cpp} (85%) rename lib/stages/{visTruncate.hpp => VisTruncate.hpp} (91%) diff --git a/config/tests/test_HFBRawReader.yaml b/config/tests/test_HFBRawReader.yaml index 015023e07..4ca191cf2 100644 --- a/config/tests/test_HFBRawReader.yaml +++ b/config/tests/test_HFBRawReader.yaml @@ -9,6 +9,12 @@ num_local_freq: 1 num_frb_total_beams: 1024 factor_upchan: 128 +# Maximum increase in noise (variance) from numerical truncation, +# with factor of 3 from uniform distribution of errors. +error_sq_lim: 3e-3 +data_fixed_prec: 1e-4 +weight_fixed_prec: 0.1 + dataset_manager: ds_broker_host: "127.0.0.1" ds_broker_port: 12050 @@ -20,13 +26,18 @@ read_buffer: num_frames: '4096' num_prod: 2048 +trunc_buffer: + kotekan_buffer: hfb + metadata_pool: hfb_pool + num_frames: '4096' + hfb_pool: kotekan_metadata_pool: HFBMetadata num_metadata_objects: '655360' read_raw: log_level: debug - infile: 20210125T212808Z_chimeHFB_corr/00000000_0000 + infile: 20210209T183459Z_chimeHFB_corr/00000000_0000 kotekan_stage: HFBRawReader max_read_rate: 0.0 out_buf: read_buffer @@ -39,11 +50,19 @@ read_raw: - 16 # chunk_beam - 128 # chunk_sub-freq +truncate: + kotekan_stage: HFBTruncate + err_sq_lim: error_sq_lim + data_fixed_precision: data_fixed_prec + weight_fixed_precision: weight_fixed_prec + in_buf: read_buffer + out_buf: trunc_buffer + write_hdf5: log_level: debug comet_timeout: 180. kotekan_stage: HFBTranspose - in_buf: read_buffer + in_buf: trunc_buffer outfile: ./test chunk_size: - 16 # chunk_freq diff --git a/lib/stages/CMakeLists.txt b/lib/stages/CMakeLists.txt index be154bf7c..2737f9cba 100644 --- a/lib/stages/CMakeLists.txt +++ b/lib/stages/CMakeLists.txt @@ -97,7 +97,7 @@ if(${USE_HDF5} AND ${USE_OMP}) endif() if(${USE_OMP}) - target_sources(kotekan_stages PRIVATE visTruncate.cpp) + target_sources(kotekan_stages PRIVATE VisTruncate.cpp HFBTruncate.cpp) endif() if(${USE_LAPACK}) diff --git a/lib/stages/HFBTruncate.cpp b/lib/stages/HFBTruncate.cpp new file mode 100644 index 000000000..7e8120913 --- /dev/null +++ b/lib/stages/HFBTruncate.cpp @@ -0,0 +1,135 @@ +#include "HFBTruncate.hpp" + +#include "Config.hpp" // for Config +#include "HFBFrameView.hpp" // for HFBFrameView +#include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate +#include "buffer.h" // for wait_for_full_frame, allocate_new_metadata_object, mark_fr... +#include "kotekanLogging.hpp" // for DEBUG +#include "truncate.hpp" // for bit_truncate_float +#include "visUtil.hpp" // for cfloat + +#include "gsl-lite.hpp" // for span + +#include // for atomic_bool +#include // for abs, sqrt +#include // for int32_t +#include // for memset, size_t +#include // for exception +#include // for _Bind_helper<>::type, bind, function +#include // for _mm256_broadcast_ss, _mm256_div_ps, _mm256_loadu_ps, _mm25... +#include // for _mm_free, _mm_malloc +#include // for match_results<>::_Base_type +#include // for vector + + +using kotekan::bufferContainer; +using kotekan::Config; +using kotekan::Stage; + +REGISTER_KOTEKAN_STAGE(HFBTruncate); + +HFBTruncate::HFBTruncate(Config& config, const std::string& unique_name, + bufferContainer& buffer_container) : + Stage(config, unique_name, buffer_container, std::bind(&HFBTruncate::main_thread, this)) { + + // Fetch the buffers, register + in_buf = get_buffer("in_buf"); + register_consumer(in_buf, unique_name.c_str()); + out_buf = get_buffer("out_buf"); + register_producer(out_buf, unique_name.c_str()); + + // Get truncation parameters from config + err_sq_lim = config.get(unique_name, "err_sq_lim"); + if (err_sq_lim < 0) + FATAL_ERROR("HFBTruncate: config: err_sq_lim should be positive (is %f).", err_sq_lim); + w_prec = config.get(unique_name, "weight_fixed_precision"); + if (w_prec < 0) + FATAL_ERROR("HFBTruncate: config: weight_fixed_precision should be positive (is %f).", + w_prec); + hfb_prec = config.get(unique_name, "data_fixed_precision"); + if (hfb_prec < 0) + FATAL_ERROR("HFBTruncate: config: data_fixed_precision should be positive (is %f).", + hfb_prec); +} + +void HFBTruncate::main_thread() { + + frameID frame_id(in_buf), output_frame_id(out_buf); + const float err_init = 0.5 * err_sq_lim; + + float err, tr_hfb; + const __m256 err_init_vec = _mm256_set1_ps(err_init); + __m256 err_vec, wgt_vec; + int32_t i_vec; + float* err_all; + + // get the first frame (just to find out about data size) + // (we don't mark it empty, so it's read again in the main loop) + if (wait_for_full_frame(in_buf, unique_name.c_str(), frame_id) == nullptr) + return; + auto frame = HFBFrameView(in_buf, frame_id); + + uint32_t data_size = frame.num_beams * frame.num_subfreq; + + // reserve enough memory for all err to be computed per frame + // 32byte-aligned memory allocation (_m256_store_ps() asks for it) + err_all = (float*)_mm_malloc(sizeof(float) * data_size, 32); + std::memset(err_all, 0, sizeof(float) * data_size); + + while (!stop_thread) { + // Wait for the buffer to be filled with data + if ((wait_for_full_frame(in_buf, unique_name.c_str(), frame_id)) == nullptr) { + break; + } + auto frame = HFBFrameView(in_buf, frame_id); + + // Wait for empty frame + if ((wait_for_empty_frame(out_buf, unique_name.c_str(), output_frame_id)) == nullptr) { + break; + } + + // Copy frame into output buffer + auto output_frame = HFBFrameView::copy_frame(in_buf, frame_id, out_buf, output_frame_id); + + // truncate absorber data and weights (8 at a time) + for (i_vec = 0; i_vec < int32_t(data_size) - 7; i_vec += 8) { + wgt_vec = _mm256_loadu_ps(&output_frame.weight[i_vec]); + err_vec = _mm256_div_ps(err_init_vec, wgt_vec); + err_vec = _mm256_sqrt_ps(err_vec); + _mm256_store_ps(err_all + i_vec, err_vec); + } + // use std::sqrt for the last few (less than 8) + for (i_vec = (data_size < 8) ? 0 : i_vec - 8; i_vec < int32_t(data_size); i_vec++) + err_all[i_vec] = std::sqrt(0.5 / output_frame.weight[i_vec] * err_sq_lim); + +#pragma omp parallel for private(err, tr_hfb) + for (size_t i = 0; i < data_size; i++) { + // Get truncation precision from weights + if (output_frame.weight[i] == 0.) { + zero_weight_found = true; + err = hfb_prec * std::abs(output_frame.hfb[i]); + } else { + err = err_all[i]; + } + // truncate hfb using weights + tr_hfb = bit_truncate_float(output_frame.hfb[i], err); + output_frame.hfb[i] = tr_hfb; + // truncate weights to fixed precision + output_frame.weight[i] = + bit_truncate_float(output_frame.weight[i], w_prec * output_frame.weight[i]); + } + + if (zero_weight_found) { + DEBUG("HFBTruncate: Frame {:d} has at least one weight value " + "being zero.", + frame_id); + zero_weight_found = false; + } + + // mark as full + mark_frame_full(out_buf, unique_name.c_str(), output_frame_id++); + // move to next frame + mark_frame_empty(in_buf, unique_name.c_str(), frame_id++); + } + _mm_free(err_all); +} diff --git a/lib/stages/HFBTruncate.hpp b/lib/stages/HFBTruncate.hpp new file mode 100644 index 000000000..9b14d3139 --- /dev/null +++ b/lib/stages/HFBTruncate.hpp @@ -0,0 +1,61 @@ +#ifndef HFBTRUNCATE +#define HFBTRUNCATE + +#include "Config.hpp" +#include "Stage.hpp" // for Stage +#include "buffer.h" +#include "bufferContainer.hpp" + +#include // for string + +/** + * @class HFBTruncate + * @brief Truncates absorber data and weight values. + * + * Absorber values are truncated to a precision based on their + * weight. + * + * @warning Don't run this anywhere but on the transpose (gossec) node. + * The OpenMP calls could cause issues on systems using kotekan pin + * priority threads (likely the GPU nodes). + * + * @par Buffers + * @buffer in_buf The input stream. + * @buffer_format HFBBuffer. + * @buffer_metadata HFBMetadata + * @buffer out_buf The output stream with truncated values. + * @buffer_format HFBBuffer. + * @buffer_metadata HFBMetadata + * + * @conf err_sq_lim Limit for the error of absorber truncation. + * @conf weight_fixed_precision Fixed precision for weight truncation. + * @conf data_fixed_precision Fixed precision for absorber truncation (if + * weights are zero). + * + * @author James Willis + */ +class HFBTruncate : public kotekan::Stage { +public: + /// Constructor; loads parameters from config + HFBTruncate(kotekan::Config& config, const std::string& unique_name, + kotekan::bufferContainer& buffer_container); + ~HFBTruncate() = default; + + /// Main loop over buffer frames + void main_thread() override; + +private: + // Buffers + Buffer* in_buf; + Buffer* out_buf; + + // Truncation parameters + float err_sq_lim; + float w_prec; + float hfb_prec; + + // Flag for frame with a zero weight + bool zero_weight_found; +}; + +#endif diff --git a/lib/stages/visTruncate.cpp b/lib/stages/VisTruncate.cpp similarity index 85% rename from lib/stages/visTruncate.cpp rename to lib/stages/VisTruncate.cpp index fab51a55a..c538aa125 100644 --- a/lib/stages/visTruncate.cpp +++ b/lib/stages/VisTruncate.cpp @@ -1,4 +1,4 @@ -#include "visTruncate.hpp" +#include "VisTruncate.hpp" #include "Config.hpp" // for Config #include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate @@ -20,7 +20,6 @@ #include // for _mm256_broadcast_ss, _mm256_div_ps, _mm256_loadu_ps, _mm25... #include // for _mm_free, _mm_malloc #include // for match_results<>::_Base_type -#include // for invalid_argument #include // for vector @@ -28,11 +27,11 @@ using kotekan::bufferContainer; using kotekan::Config; using kotekan::Stage; -REGISTER_KOTEKAN_STAGE(visTruncate); +REGISTER_KOTEKAN_STAGE(VisTruncate); -visTruncate::visTruncate(Config& config, const std::string& unique_name, +VisTruncate::VisTruncate(Config& config, const std::string& unique_name, bufferContainer& buffer_container) : - Stage(config, unique_name, buffer_container, std::bind(&visTruncate::main_thread, this)) { + Stage(config, unique_name, buffer_container, std::bind(&VisTruncate::main_thread, this)) { // Fetch the buffers, register in_buf = get_buffer("in_buf"); @@ -43,28 +42,25 @@ visTruncate::visTruncate(Config& config, const std::string& unique_name, // Get truncation parameters from config err_sq_lim = config.get(unique_name, "err_sq_lim"); if (err_sq_lim < 0) - throw std::invalid_argument("visTruncate: config: err_sq_lim should" - " be positive (is " - + std::to_string(err_sq_lim) + ")."); + FATAL_ERROR("VisTruncate: config: err_sq_lim should be positive (is %f).", err_sq_lim); w_prec = config.get(unique_name, "weight_fixed_precision"); if (w_prec < 0) - throw std::invalid_argument("visTruncate: config: " - "weight_fixed_precision should be positive (is " - + std::to_string(w_prec) + ")."); + FATAL_ERROR("VisTruncate: config: weight_fixed_precision should be positive (is %f).", + w_prec); vis_prec = config.get(unique_name, "data_fixed_precision"); if (vis_prec < 0) - throw std::invalid_argument("visTruncate: config: " - "data_fixed_precision should be positive (is " - + std::to_string(vis_prec) + ")."); + FATAL_ERROR("VisTruncate: config: data_fixed_precision should be positive (is %f).", + vis_prec); } -void visTruncate::main_thread() { +void VisTruncate::main_thread() { unsigned int frame_id = 0; unsigned int output_frame_id = 0; const float err_init = 0.5 * err_sq_lim; float err_r, err_i; + const __m256 err_init_vec = _mm256_set1_ps(err_init); cfloat tr_vis, tr_evec; __m256 err_vec, wgt_vec; int32_t i_vec; @@ -98,9 +94,8 @@ void visTruncate::main_thread() { // truncate visibilities and weights (8 at a time) for (i_vec = 0; i_vec < int32_t(frame.num_prod) - 7; i_vec += 8) { - err_vec = _mm256_broadcast_ss(&err_init); wgt_vec = _mm256_loadu_ps(&output_frame.weight[i_vec]); - err_vec = _mm256_div_ps(err_vec, wgt_vec); + err_vec = _mm256_div_ps(err_init_vec, wgt_vec); err_vec = _mm256_sqrt_ps(err_vec); _mm256_store_ps(err_all + i_vec, err_vec); } @@ -150,7 +145,7 @@ void visTruncate::main_thread() { } if (zero_weight_found) { - DEBUG("visTruncate: Frame {:d} has at least one weight value " + DEBUG("VisTruncate: Frame {:d} has at least one weight value " "being zero.", frame_id); zero_weight_found = false; diff --git a/lib/stages/visTruncate.hpp b/lib/stages/VisTruncate.hpp similarity index 91% rename from lib/stages/visTruncate.hpp rename to lib/stages/VisTruncate.hpp index e1a64d28b..ac9160ee2 100644 --- a/lib/stages/visTruncate.hpp +++ b/lib/stages/VisTruncate.hpp @@ -9,7 +9,7 @@ #include // for string /** - * @class visTruncate + * @class VisTruncate * @brief Truncates visibility, eigenvalue and weight values. * * eigenvalues and weights are truncated with a fixed precision that is set in @@ -35,12 +35,12 @@ * * @author Tristan Pinsonneault-Marotte, Rick Nitsche */ -class visTruncate : public kotekan::Stage { +class VisTruncate : public kotekan::Stage { public: /// Constructor; loads parameters from config - visTruncate(kotekan::Config& config, const std::string& unique_name, + VisTruncate(kotekan::Config& config, const std::string& unique_name, kotekan::bufferContainer& buffer_container); - ~visTruncate() = default; + ~VisTruncate() = default; /// Main loop over buffer frames void main_thread() override; diff --git a/tests/test_truncate.py b/tests/test_truncate.py index 258379209..98c8a40aa 100644 --- a/tests/test_truncate.py +++ b/tests/test_truncate.py @@ -51,7 +51,7 @@ def vis_data(tmpdir_factory): out_dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester( - "visTruncate", + "VisTruncate", trunc_params, buffers_in=fakevis_buffer, buffers_out=out_dump_buffer, @@ -90,7 +90,7 @@ def vis_data_zero_weights(tmpdir_factory): out_dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester( - "visTruncate", + "VisTruncate", trunc_params, buffers_in=fakevis_buffer, buffers_out=out_dump_buffer, From 81918288147435cef8ad52db05da0988c999a7dd Mon Sep 17 00:00:00 2001 From: james-s-willis <36906682+james-s-willis@users.noreply.github.com> Date: Fri, 12 Feb 2021 14:09:10 -0500 Subject: [PATCH 05/15] Fix rate limiting when reading raw files. (#928) --- lib/stages/RawReader.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/stages/RawReader.hpp b/lib/stages/RawReader.hpp index 040d89625..126d38804 100644 --- a/lib/stages/RawReader.hpp +++ b/lib/stages/RawReader.hpp @@ -489,7 +489,7 @@ void RawReader::main_thread() { end_time = current_time(); double sleep_time_this_frame = min_read_time - (end_time - start_time); DEBUG2("Sleep time {}", sleep_time_this_frame); - if (sleep_time > 0) { + if (sleep_time_this_frame > 0) { auto ts = double_to_ts(sleep_time_this_frame); nanosleep(&ts, nullptr); } From 9210eacc7417fa2c2b42f13e8ee25529ff52603e Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 17 Feb 2021 17:41:28 -0500 Subject: [PATCH 06/15] Correct comment (#929) --- lib/stages/beamformingPostProcess.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/stages/beamformingPostProcess.cpp b/lib/stages/beamformingPostProcess.cpp index 7986bedb7..bc02f0287 100644 --- a/lib/stages/beamformingPostProcess.cpp +++ b/lib/stages/beamformingPostProcess.cpp @@ -224,7 +224,7 @@ void beamformingPostProcess::main_thread() { for (uint32_t freq = 0; freq < _num_local_freq; ++freq) { unsigned char* in_buf_data = (unsigned char*)in_frame[thread_id]; // The two polarizations. - // Each sample is 4-bit real, 4-bit complex, so byte operations work just + // Each sample is 4-bit real, 4-bit imaginary, so byte operations work just // fine here. out_buf[station_0_index + freq] = in_buf_data[i * 16 + freq * 2]; out_buf[station_1_index + freq] = in_buf_data[i * 16 + freq * 2 + 1]; From 3389522d307d7070728ba97cd7b920b3c915c123 Mon Sep 17 00:00:00 2001 From: Rick Nitsche <2536635+nritsche@users.noreply.github.com> Date: Wed, 24 Feb 2021 08:45:49 -0800 Subject: [PATCH 07/15] fix(visAccumulate): Register base ds if ds id not set in frame (#926) --- lib/stages/visAccumulate.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/stages/visAccumulate.cpp b/lib/stages/visAccumulate.cpp index 331752e82..e406ee113 100644 --- a/lib/stages/visAccumulate.cpp +++ b/lib/stages/visAccumulate.cpp @@ -32,6 +32,7 @@ #include // for back_insert_iterator, begin, end, back_inserter #include // for lock_guard, mutex #include // for iota +#include // for optional #include // for match_results<>::_Base_type #include // for runtime_error, invalid_argument #include // for size_t, timespec @@ -249,7 +250,7 @@ void visAccumulate::main_thread() { frameID in_frame_id(in_buf); - dset_id_t ds_id_in = dset_id_t::null; + std::optional ds_id_in = std::nullopt; // Hold the gated datasets that are enabled; std::vector> enabled_gated_datasets; @@ -275,13 +276,13 @@ void visAccumulate::main_thread() { // Check if dataset ID changed dset_id_t ds_id_in_new = get_dataset_id(in_buf, in_frame_id); - if (ds_id_in_new != ds_id_in) { + if (!ds_id_in || ds_id_in_new != *ds_id_in) { ds_id_in = ds_id_in_new; // Register base dataset. If no dataset ID was was set in the incoming frame, // ds_id_in will be dset_id_t::null and thus cause a root dataset to // be registered. - base_dataset_id = dm.add_dataset(base_dataset_states, ds_id_in); + base_dataset_id = dm.add_dataset(base_dataset_states, *ds_id_in); DEBUG("Registered base dataset: {}", base_dataset_id) // Set the output dataset ID for the main visibility accumulation From a9b60c489c5498cc9a56835f14803f4af5fc6432 Mon Sep 17 00:00:00 2001 From: Adam Lanman Date: Wed, 24 Feb 2021 11:47:12 -0500 Subject: [PATCH 08/15] Fix compile option in readme (#922) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5e34756fa..b917c19f0 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ symbols. it is not part of the base compile, even when enabled. * `-DUSE_OMP=ON` Build stages using OpenMP. This requires a compiler supporting OpenMP (>= 3.0) * `-DOPENSSL_ROOT_DIR=` Only required for non-standard install locations of OpenSSL -* `-DBOOST_TESTS=ON` Build tests using The Boost Test Framework. +* `-DWITH_TESTS=ON` Build kotekans test library and C++ unit tests using The Boost Test Framework. pytest-cpp needs to be installed for pytest to find them. * `-DSUPERDEBUG=ON` Add extra debugging info and turn off all optimisation to improve coverage. * `-DSANITIZE=ON` Turn on extra Clang sanitizers (currently the address sanitizer) for finding issues. From 64216077c9518977d94a9990c9bf4fb7a06d6b08 Mon Sep 17 00:00:00 2001 From: james-s-willis <36906682+james-s-willis@users.noreply.github.com> Date: Wed, 24 Feb 2021 12:31:18 -0500 Subject: [PATCH 09/15] Double HFB receive buffer size (#930) --- config/chime_science_run_recv_hfb.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/chime_science_run_recv_hfb.yaml b/config/chime_science_run_recv_hfb.yaml index 6c5f458d9..798a846c1 100644 --- a/config/chime_science_run_recv_hfb.yaml +++ b/config/chime_science_run_recv_hfb.yaml @@ -37,7 +37,7 @@ hfb_pool: hfbbuf_10s: kotekan_buffer: hfb - num_frames: 2048 + num_frames: 4096 metadata_pool: hfb_pool # Kotekan stages From 7c00a75c69ec2d6d2dc6ac24c36f6ddcf66cd17d Mon Sep 17 00:00:00 2001 From: Andre Renard Date: Mon, 1 Mar 2021 12:44:56 -0500 Subject: [PATCH 10/15] Add config/dpdk option to do burst captures of ADC data (#911) * Allows for capturing bursts of full speed ADC data from one FPGA link/ADC upto the amount of available system memory. --- config/adc_record_burst.yaml | 107 ++++++++++++++++++++++++++++++++++ lib/dpdk/iceBoardStandard.hpp | 67 +++++++++++++-------- lib/stages/rawFileWrite.cpp | 25 +++++--- lib/stages/rawFileWrite.hpp | 4 +- 4 files changed, 171 insertions(+), 32 deletions(-) create mode 100644 config/adc_record_burst.yaml diff --git a/config/adc_record_burst.yaml b/config/adc_record_burst.yaml new file mode 100644 index 000000000..6d33d39e4 --- /dev/null +++ b/config/adc_record_burst.yaml @@ -0,0 +1,107 @@ +########################################## +# +# adc_record.yaml +# +# A config which allows for the recording of one +# stream of ADC data from ICEBoards on systems without enough disk IO to +# record continuously. Instead this records data to a RAM buffer and then +# stops capturing data while it's written to disk slowly +# +# Author: Andre Renard +# +########################################## +--- +type: config +# Logging level can be one of: +# OFF, ERROR, WARN, INFO, DEBUG, DEBUG2 (case insensitive) +# Note DEBUG and DEBUG2 require a build with (-DCMAKE_BUILD_TYPE=Debug) +log_level: INFO + +# Note that for ADC mode the "samples" variable +# doesn't really make sense here. Since there are +# 2048 ADC samples in the 2.56 us the other modes consider to be a "sample" +samples_per_data_set: 65536 +sample_block_size: 2048 + +# Set this value to the maximum allowed by the system RAM. +# Each frame is ~135 MB, and stores about 168ms of data. +buffer_depth: 105 +cpu_affinity: [2,3] +instrument_name: mcgill_lab_test + +# Telescope +telescope: + name: ICETelescope + require_gps: false + query_gps: true # If fpga_master provides GPS time set to true + gps_host: 127.0.0.1 # Set this to the fpga_master host + num_local_freq: 1 # This value doesn't apply to ADC capture + +# Pool +main_pool: + kotekan_metadata_pool: chimeMetadata + num_metadata_objects: 2 * buffer_depth + 10 + +# Buffers +adc_buffer: + kotekan_buffer: standard + num_frames: buffer_depth + frame_size: samples_per_data_set * sample_block_size + metadata_pool: main_pool + +lost_samples_buffer: + kotekan_buffer: standard + num_frames: buffer_depth + frame_size: samples_per_data_set + metadata_pool: main_pool + +# Stages + +# The core mapping here is setup for a 6 core CPU with 12 threads (vcores) +# and setup to use the first 4 real cores (8 vcores) +dpdk: + kotekan_stage: dpdkCore + # Format is index = lcore, value = cpu core + lcore_cpu_map: [0] + master_lcore_cpu: 1 + fpga_packet_size: 4160 + samples_per_packet: 2 + capture_n_frames: buffer_depth + alignment: samples_per_data_set * 100 + # Format is index = lcore, value = array of port IDs + # so [[0,1],[2,3]] maps lcore 0 to service ports 0 and 1, + # and lcore 1 to service ports 2 and 3. + lcore_port_map: + - [0] + # One handler must be given per port on the system. + handlers: + - dpdk_handler: iceBoardStandard + out_buf: adc_buffer + lost_samples_buf: lost_samples_buffer + # Uncomment if you have more than one uio port active + #- dpdk_handler: none + #- dpdk_handler: none + #- dpdk_handler: none + #- dpdk_handler: none + #- dpdk_handler: none + #- dpdk_handler: none + #- dpdk_handler: none + +zero_samples: + kotekan_stage: zeroSamples + out_buf: adc_buffer + lost_samples_buf: lost_samples_buffer + zero_value: 0 # Should be set to 255? + +metadata_dump: + kotekan_stage: chimeMetadataDump + in_buf: adc_buffer + +raw_file_write: + kotekan_stage: rawFileWrite + in_buf: adc_buffer + # *** Change this value for each run *** + base_dir: /data + file_name: adc_data + file_ext: raw + exit_after_n_files: buffer_depth diff --git a/lib/dpdk/iceBoardStandard.hpp b/lib/dpdk/iceBoardStandard.hpp index 88576d5d6..45032e4c7 100644 --- a/lib/dpdk/iceBoardStandard.hpp +++ b/lib/dpdk/iceBoardStandard.hpp @@ -55,9 +55,9 @@ class iceBoardStandard : public iceBoardHandler { protected: bool advance_frame(uint64_t new_seq, bool first_time = false); - void handle_lost_samples(int64_t lost_samples); + bool handle_lost_samples(int64_t lost_samples); - void copy_packet(struct rte_mbuf* mbuf); + bool copy_packet(struct rte_mbuf* mbuf); /// The output buffer struct Buffer* out_buf; @@ -76,6 +76,12 @@ class iceBoardStandard : public iceBoardHandler { /// Frame IDs int lost_samples_frame_id = 0; + + /// Number of frames captured + uint64_t num_frames_captured; + + /// Maximum number of frames to capture (used for burst captures), 0 = unlimited + uint64_t capture_n_frames; }; iceBoardStandard::iceBoardStandard(kotekan::Config& config, const std::string& unique_name, @@ -93,6 +99,9 @@ iceBoardStandard::iceBoardStandard(kotekan::Config& config, const std::string& u // We want to make sure the flag buffers are zeroed between uses. zero_frames(lost_samples_buf); + // Number of frames to capture before stopping, 0 = unlimited + capture_n_frames = config.get_default(unique_name, "capture_n_frames", 0); + // TODO Some parts of this function are common to the various ICEboard // handlers, and could likely be factored out. std::string endpoint_name = unique_name + "/port_data"; @@ -107,7 +116,7 @@ inline int iceBoardStandard::handle_packet(struct rte_mbuf* mbuf) { // Check if the packet is valid if (!iceBoardHandler::check_packet(mbuf)) - return 0; // Disgards the packet. + return 0; // Discards the packet. if (unlikely(!got_first_packet)) { if (likely(!iceBoardHandler::align_first_packet(mbuf))) @@ -115,7 +124,7 @@ inline int iceBoardStandard::handle_packet(struct rte_mbuf* mbuf) { // Setup the first buffer frame for copying data into if (!iceBoardStandard::advance_frame(last_seq, true)) - return 0; // This catches the exit condition. + return -1; // This catches the exit condition. } else { cur_seq = iceBoardHandler::get_mbuf_seq_num(mbuf); } @@ -132,10 +141,13 @@ inline int iceBoardStandard::handle_packet(struct rte_mbuf* mbuf) { // Handle lost packets if (unlikely(diff > samples_per_packet)) - iceBoardStandard::handle_lost_samples(diff - samples_per_packet); + if (unlikely(!iceBoardStandard::handle_lost_samples(diff - samples_per_packet))) + return -1; + // copy packet - iceBoardStandard::copy_packet(mbuf); + if (unlikely(!iceBoardStandard::copy_packet(mbuf))) + return -1; last_seq = cur_seq; @@ -152,15 +164,30 @@ inline bool iceBoardStandard::advance_frame(uint64_t new_seq, bool first_time) { // Advance the frame if (!first_time) { mark_frame_full(out_buf, unique_name.c_str(), out_frame_id); - - // Advance frame ID out_frame_id = (out_frame_id + 1) % out_buf->num_frames; + + // Advance the lost samples frame + mark_frame_full(lost_samples_buf, unique_name.c_str(), lost_samples_frame_id); + lost_samples_frame_id = (lost_samples_frame_id + 1) % lost_samples_buf->num_frames; } + // Check if we have captured enough frames + num_frames_captured++; + if (capture_n_frames != 0 && num_frames_captured > capture_n_frames) { + return false; + } + + // Get new output frame out_frame = wait_for_empty_frame(out_buf, unique_name.c_str(), out_frame_id); if (out_frame == nullptr) return false; + // Get new lost samples frame + lost_samples_frame = + wait_for_empty_frame(lost_samples_buf, unique_name.c_str(), lost_samples_frame_id); + if (lost_samples_frame == nullptr) + return false; + // Set metadata values. allocate_new_metadata_object(out_buf, out_frame_id); @@ -174,32 +201,24 @@ inline bool iceBoardStandard::advance_frame(uint64_t new_seq, bool first_time) { ice_set_stream_id_t(out_buf, out_frame_id, port_stream_id); set_fpga_seq_num(out_buf, out_frame_id, new_seq); - // Advance the lost samples frame - if (!first_time) { - mark_frame_full(lost_samples_buf, unique_name.c_str(), lost_samples_frame_id); - lost_samples_frame_id = (lost_samples_frame_id + 1) % lost_samples_buf->num_frames; - } - lost_samples_frame = - wait_for_empty_frame(lost_samples_buf, unique_name.c_str(), lost_samples_frame_id); - if (lost_samples_frame == nullptr) - return false; - return true; } // Note this function is almost identical to the handle_lost_samples function in the // iceboardshuffle except for the call to advance_frame(). It might be possible to // refactor some of this code. -inline void iceBoardStandard::handle_lost_samples(int64_t lost_samples) { +inline bool iceBoardStandard::handle_lost_samples(int64_t lost_samples) { int64_t lost_sample_location = last_seq + samples_per_packet - get_fpga_seq_num(out_buf, out_frame_id); uint64_t temp_seq = last_seq + samples_per_packet; - // TODO this could be made more efficent by breaking it down into blocks of memsets. + // TODO this could be made more efficient by breaking it down into blocks of memsets. while (lost_samples > 0) { if (unlikely(lost_sample_location * sample_size == out_buf->frame_size)) { - advance_frame(temp_seq); + if (!advance_frame(temp_seq)) { + return false; + } lost_sample_location = 0; } @@ -210,9 +229,10 @@ inline void iceBoardStandard::handle_lost_samples(int64_t lost_samples) { rx_lost_samples_total += 1; temp_seq += 1; } + return true; } -inline void iceBoardStandard::copy_packet(struct rte_mbuf* mbuf) { +inline bool iceBoardStandard::copy_packet(struct rte_mbuf* mbuf) { // Note this assumes that frame_size is divisable by samples_per_packet, // or the assert below will fail. @@ -223,7 +243,7 @@ inline void iceBoardStandard::copy_packet(struct rte_mbuf* mbuf) { if (unlikely(sample_location * sample_size == out_buf->frame_size)) { // If there are no new frames to fill, we are just dropping the packet if (!advance_frame(cur_seq)) - return; + return false; sample_location = 0; } @@ -232,6 +252,7 @@ inline void iceBoardStandard::copy_packet(struct rte_mbuf* mbuf) { copy_block(&mbuf, (uint8_t*)&out_frame[sample_location * sample_size], sample_size * samples_per_packet, &pkt_offset); + return true; } #endif diff --git a/lib/stages/rawFileWrite.cpp b/lib/stages/rawFileWrite.cpp index 2e8ccaa2d..1f5b05fce 100644 --- a/lib/stages/rawFileWrite.cpp +++ b/lib/stages/rawFileWrite.cpp @@ -1,9 +1,10 @@ #include "rawFileWrite.hpp" -#include "Config.hpp" // for Config -#include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate -#include "buffer.h" // for Buffer, get_metadata_container, mark_frame_empty, regis... -#include "bufferContainer.hpp" // for bufferContainer +#include "Config.hpp" // for Config +#include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate +#include "buffer.h" // for Buffer, get_metadata_container, mark_frame_empty, regis... +#include "bufferContainer.hpp" // for bufferContainer +#include "errors.h" #include "kotekanLogging.hpp" // for ERROR, INFO #include "metadata.h" // for metadataContainer #include "prometheusMetrics.hpp" // for Metrics, Gauge @@ -41,6 +42,7 @@ rawFileWrite::rawFileWrite(Config& config, const std::string& unique_name, _file_ext = config.get(unique_name, "file_ext"); _num_frames_per_file = config.get_default(unique_name, "num_frames_per_file", 1); _prefix_hostname = config.get_default(unique_name, "prefix_hostname", true); + _exit_after_n_files = config.get_default(unique_name, "exit_after_n_files", 0); } rawFileWrite::~rawFileWrite() {} @@ -48,8 +50,8 @@ rawFileWrite::~rawFileWrite() {} void rawFileWrite::main_thread() { int fd; - int file_num = 0; - int frame_id = 0; + uint32_t file_num = 0; + uint32_t frame_id = 0; uint32_t frame_ctr = 0; uint8_t* frame = nullptr; char hostname[64]; @@ -74,10 +76,10 @@ void rawFileWrite::main_thread() { if (!isFileOpen) { if (_prefix_hostname) { - snprintf(full_path, full_path_len, "%s/%s_%s_%07d.%s", _base_dir.c_str(), hostname, + snprintf(full_path, full_path_len, "%s/%s_%s_%07u.%s", _base_dir.c_str(), hostname, _file_name.c_str(), file_num, _file_ext.c_str()); } else { - snprintf(full_path, full_path_len, "%s/%s_%07d.%s", _base_dir.c_str(), + snprintf(full_path, full_path_len, "%s/%s_%07u.%s", _base_dir.c_str(), _file_name.c_str(), file_num, _file_ext.c_str()); } @@ -138,6 +140,13 @@ void rawFileWrite::main_thread() { mark_frame_empty(buf, unique_name.c_str(), frame_id); + // Check if we should exit after writing out a fixed number of files. + // Useful for some tests and burst modes. Will hopefully be replaced by frames + // which can contain a "final" signal. + if (_exit_after_n_files != 0 && file_num >= _exit_after_n_files) { + exit_kotekan(ReturnCode::CLEAN_EXIT); + } + frame_id = (frame_id + 1) % buf->num_frames; } } diff --git a/lib/stages/rawFileWrite.hpp b/lib/stages/rawFileWrite.hpp index dd0f3fd95..2627d38a1 100644 --- a/lib/stages/rawFileWrite.hpp +++ b/lib/stages/rawFileWrite.hpp @@ -20,7 +20,8 @@ * @conf base_dir String. Directory to write into. * @conf file_name String. Base filename to write. * @conf file_ext String. File extension. - * @conf num_frames_per_file Integer. No of frames to write into a single file. + * @conf num_frames_per_file Int. No of frames to write into a single file. + * @conf exit_after_n_files Int. Stop writing after this many files, Default 0 = unlimited files. * * @par Metrics * @metric kotekan_rawfilewrite_write_time_seconds @@ -41,6 +42,7 @@ class rawFileWrite : public kotekan::Stage { std::string _file_name; std::string _file_ext; uint32_t _num_frames_per_file; + uint32_t _exit_after_n_files; // Prefix file name with hostname or not bool _prefix_hostname; }; From 966b5dc849f89536450d82a85b2f1f998f4e6198 Mon Sep 17 00:00:00 2001 From: Andre Renard Date: Mon, 1 Mar 2021 17:25:58 -0500 Subject: [PATCH 11/15] Changes to support 16-element correlators with OpenCL (#932) * Use AMD's FindOpenCL to work with current ROCm versions * Add safe_swap frame function * Add namespaces to macro stage constructor * Adjust DPDK core settings to work with newer versions * Add fpga_dataset to iceBoardStandard packet capture mode * Add a stage to round-robin split frames into many out buffers * Add configuration for single ICEBoard systems (e.g. 16-element correlator) --- CMakeLists.txt | 2 +- cmake/FindOPENCL.cmake | 57 ++++++ config/single_iceboard_n2.yaml | 332 +++++++++++++++++++++++++++++++++ lib/core/Stage.hpp | 3 +- lib/core/buffer.c | 39 +++- lib/core/buffer.h | 15 ++ lib/dpdk/dpdkCore.cpp | 15 +- lib/dpdk/dpdkCore.hpp | 2 +- lib/dpdk/iceBoardStandard.hpp | 11 +- lib/opencl/CMakeLists.txt | 5 +- lib/stages/BaseWriter.cpp | 18 +- lib/stages/BufferSplit.cpp | 51 +++++ lib/stages/BufferSplit.hpp | 48 +++++ lib/stages/CMakeLists.txt | 1 + 14 files changed, 575 insertions(+), 24 deletions(-) create mode 100644 cmake/FindOPENCL.cmake create mode 100644 config/single_iceboard_n2.yaml create mode 100644 lib/stages/BufferSplit.cpp create mode 100644 lib/stages/BufferSplit.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 97438311e..7aba94b9a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,7 +117,7 @@ endif() if(${USE_OPENCL}) set(ENV{AMDAPPSDKROOT} /opt/rocm/opencl) - find_package(OpenCL REQUIRED) + find_package(OPENCL REQUIRED) set(USE_OPENCL ${OPENCL_FOUND}) endif() diff --git a/cmake/FindOPENCL.cmake b/cmake/FindOPENCL.cmake new file mode 100644 index 000000000..61d46da1a --- /dev/null +++ b/cmake/FindOPENCL.cmake @@ -0,0 +1,57 @@ +# ################################################################################################## +# +# MIT License +# +# Copyright (c) 2017 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# ################################################################################################## +find_path( + OPENCL_INCLUDE_DIRS + NAMES OpenCL/cl.h CL/cl.h + HINTS ${OPENCL_ROOT}/include $ENV{AMDAPPSDKROOT}/include $ENV{CUDA_PATH}/include + PATHS /usr/include /usr/local/include /usr/local/cuda/include /opt/cuda/include + /opt/rocm/opencl/include ${CMAKE_INSTALL_PREFIX}/opencl/include + DOC "OpenCL header file path") +mark_as_advanced(OPENCL_INCLUDE_DIRS) + +if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") + find_library( + OPENCL_LIBRARIES + NAMES OpenCL + HINTS ${OPENCL_ROOT}/lib $ENV{AMDAPPSDKROOT}/lib $ENV{CUDA_PATH}/lib + DOC "OpenCL dynamic library path" + PATH_SUFFIXES x86_64 x64 x86_64/sdk + PATHS /usr/lib /usr/local/cuda/lib /opt/cuda/lib /opt/rocm/opencl/lib + ${CMAKE_INSTALL_PREFIX}/opencl/lib) +else() + find_library( + OPENCL_LIBRARIES + NAMES OpenCL + HINTS ${OPENCL_ROOT}/lib $ENV{AMDAPPSDKROOT}/lib $ENV{CUDA_PATH}/lib + DOC "OpenCL dynamic library path" + PATH_SUFFIXES x86 Win32 + PATHS /usr/lib /usr/local/cuda/lib /opt/cuda/lib) +endif() +mark_as_advanced(OPENCL_LIBRARIES) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(OPENCL DEFAULT_MSG OPENCL_LIBRARIES OPENCL_INCLUDE_DIRS) + +if(NOT OPENCL_FOUND) + message(STATUS "FindOpenCL looked for libraries named: OpenCL") +endif() diff --git a/config/single_iceboard_n2.yaml b/config/single_iceboard_n2.yaml new file mode 100644 index 000000000..025560512 --- /dev/null +++ b/config/single_iceboard_n2.yaml @@ -0,0 +1,332 @@ +--- +type: config +# Logging level can be one of: +# OFF, ERROR, WARN, INFO, DEBUG, DEBUG2 (case insensitive) +# Note DEBUG and DEBUG2 require a build with (-DCMAKE_BUILD_TYPE=Debug) +log_level: debug + +buffer_depth: 16 +samples_per_data_set: 65536 +num_elements: 16 +num_links: 8 +num_freq: 1024 +num_local_freq: num_freq / num_links +num_freq_in_frame: num_freq / num_links +block_size: 16 +num_blocks: (num_elements / block_size) * (num_elements / block_size + 1) / 2 +instrument_name: bleien +num_ev: 0 + +telescope: + name: ICETelescope + require_gps: false + allow_default_frequency_map: true # Don't fall back on default map + +dataset_manager: + use_dataset_broker: False + +sizeof_int: 4 +sizeof_float: 4 +numa_node: 0 + +# Pool +main_pool: + kotekan_metadata_pool: chimeMetadata + num_metadata_objects: 15 * buffer_depth +# Buffers +gpu_input_buffers: + num_frames: buffer_depth + frame_size: samples_per_data_set * num_elements * num_local_freq + metadata_pool: main_pool + gpu_input_buffer_0: + kotekan_buffer: standard + gpu_input_buffer_1: + kotekan_buffer: standard + gpu_input_buffer_2: + kotekan_buffer: standard + gpu_input_buffer_3: + kotekan_buffer: standard + gpu_input_buffer_4: + kotekan_buffer: standard + gpu_input_buffer_5: + kotekan_buffer: standard + gpu_input_buffer_6: + kotekan_buffer: standard + gpu_input_buffer_7: + kotekan_buffer: standard + +lost_samples_buffers: + num_frames: buffer_depth + frame_size: samples_per_data_set + metadata_pool: main_pool + lost_samples_buffer_0: + kotekan_buffer: standard + lost_samples_buffer_1: + kotekan_buffer: standard + lost_samples_buffer_2: + kotekan_buffer: standard + lost_samples_buffer_3: + kotekan_buffer: standard + lost_samples_buffer_4: + kotekan_buffer: standard + lost_samples_buffer_5: + kotekan_buffer: standard + lost_samples_buffer_6: + kotekan_buffer: standard + lost_samples_buffer_7: + kotekan_buffer: standard + +merged_network_buf: + kotekan_buffer: standard + num_frames: buffer_depth * 8 + frame_size: samples_per_data_set * num_elements * num_local_freq + metadata_pool: main_pool + +gpu_output_buffers: + num_frames: buffer_depth * num_links + frame_size: num_local_freq * num_blocks * (block_size*block_size) * 2 * sizeof_int + metadata_pool: main_pool + gpu_output_buffer: + kotekan_buffer: standard + +gpu_output_split_buffer: + num_frames: buffer_depth + frame_size: num_local_freq * num_blocks * (block_size*block_size) * 2 * sizeof_int + metadata_pool: main_pool + gpu_output_split_buffer_0: + kotekan_buffer: standard + gpu_output_split_buffer_1: + kotekan_buffer: standard + gpu_output_split_buffer_2: + kotekan_buffer: standard + gpu_output_split_buffer_3: + kotekan_buffer: standard + gpu_output_split_buffer_4: + kotekan_buffer: standard + gpu_output_split_buffer_5: + kotekan_buffer: standard + gpu_output_split_buffer_6: + kotekan_buffer: standard + gpu_output_split_buffer_7: + kotekan_buffer: standard + +cpu_affinity: [4,5,10,11] # other cores are on DPDK +dpdk: + kotekan_stage: dpdkCore + # Format is index = lcore, value = cpu core + lcore_cpu_map: [0,1,2,3,4,5,6,7] + master_lcore_cpu: 8 + fpga_packet_size: 4680 + alignment: samples_per_data_set + # Format is index = lcore, value = array of port IDs + # so [[0,1],[2,3]] maps lcore 0 to service ports 0 and 1, + # and lcore 1 to service ports 2 and 3. + lcore_port_map: + - [0] + - [1] + - [2] + - [3] + - [4] + - [5] + - [6] + - [7] + # One handler must be given per port on the system. + handlers: + - dpdk_handler: iceBoardStandard + out_buf: gpu_input_buffer_0 + lost_samples_buf: lost_samples_buffer_0 + - dpdk_handler: iceBoardStandard + out_buf: gpu_input_buffer_1 + lost_samples_buf: lost_samples_buffer_1 + - dpdk_handler: iceBoardStandard + out_buf: gpu_input_buffer_2 + lost_samples_buf: lost_samples_buffer_2 + - dpdk_handler: iceBoardStandard + out_buf: gpu_input_buffer_3 + lost_samples_buf: lost_samples_buffer_3 + - dpdk_handler: iceBoardStandard + out_buf: gpu_input_buffer_4 + lost_samples_buf: lost_samples_buffer_4 + - dpdk_handler: iceBoardStandard + out_buf: gpu_input_buffer_5 + lost_samples_buf: lost_samples_buffer_5 + - dpdk_handler: iceBoardStandard + out_buf: gpu_input_buffer_6 + lost_samples_buf: lost_samples_buffer_6 + - dpdk_handler: iceBoardStandard + out_buf: gpu_input_buffer_7 + lost_samples_buf: lost_samples_buffer_7 + +zero_samples: + zero_0: + kotekan_stage: zeroSamples + out_buf: gpu_input_buffer_0 + lost_samples_buf: lost_samples_buffer_0 + zero_1: + kotekan_stage: zeroSamples + out_buf: gpu_input_buffer_1 + lost_samples_buf: lost_samples_buffer_1 + zero_2: + kotekan_stage: zeroSamples + out_buf: gpu_input_buffer_2 + lost_samples_buf: lost_samples_buffer_2 + zero_3: + kotekan_stage: zeroSamples + out_buf: gpu_input_buffer_3 + lost_samples_buf: lost_samples_buffer_3 + zero_4: + kotekan_stage: zeroSamples + out_buf: gpu_input_buffer_4 + lost_samples_buf: lost_samples_buffer_4 + zero_5: + kotekan_stage: zeroSamples + out_buf: gpu_input_buffer_5 + lost_samples_buf: lost_samples_buffer_5 + zero_6: + kotekan_stage: zeroSamples + out_buf: gpu_input_buffer_6 + lost_samples_buf: lost_samples_buffer_6 + zero_7: + kotekan_stage: zeroSamples + out_buf: gpu_input_buffer_7 + lost_samples_buf: lost_samples_buffer_7 + +merge: + kotekan_stage: bufferMerge + in_bufs: + - gpu_input_buffer_0 + - gpu_input_buffer_1 + - gpu_input_buffer_2 + - gpu_input_buffer_3 + - gpu_input_buffer_4 + - gpu_input_buffer_5 + - gpu_input_buffer_6 + - gpu_input_buffer_7 + out_buf: merged_network_buf + +gpu: + num_data_sets: 1 + profiling: true + kernel_path: "/var/lib/kotekan/opencl/kernels/" + log_level: WARN + frame_arrival_period: samples_per_data_set / 390625 / num_links + commands: + - name: clInputData + - name: clPresumZero + - name: clOutputDataZero + - name: clPresumKernel + - name: clKVCorr + - name: clOutputData + gpu_0: + kotekan_stage: clProcess + gpu_id: 0 + in_buffers: + network_buf: merged_network_buf + out_buffers: + output_buf: gpu_output_buffer + +split: + kotekan_stage: BufferSplit + in_buf: gpu_output_buffer + out_bufs: + - gpu_output_split_buffer_0 + - gpu_output_split_buffer_1 + - gpu_output_split_buffer_2 + - gpu_output_split_buffer_3 + - gpu_output_split_buffer_4 + - gpu_output_split_buffer_5 + - gpu_output_split_buffer_6 + - gpu_output_split_buffer_7 + +metadata: + #kotekan_stage: chimeMetadataDump + in_buf: gpu_output_buffer + +# Metadata pool +vis_pool: + kotekan_metadata_pool: VisMetadata + num_metadata_objects: 20 * buffer_depth * num_freq_in_frame + +# Buffers +vis_buffers: + metadata_pool: vis_pool + num_frames: buffer_depth * num_freq_in_frame + visbuf_5s_merged: + kotekan_buffer: vis + visbuf_5s_0: + kotekan_buffer: vis + visbuf_5s_1: + kotekan_buffer: vis + visbuf_5s_2: + kotekan_buffer: vis + visbuf_5s_3: + kotekan_buffer: vis + visbuf_5s_4: + kotekan_buffer: vis + visbuf_5s_5: + kotekan_buffer: vis + visbuf_5s_6: + kotekan_buffer: vis + visbuf_5s_7: + kotekan_buffer: vis + +vis_accumulate_processes: + log_level: WARN + integration_time: 5.0 + acc_0: + kotekan_stage: visAccumulate + in_buf: gpu_output_split_buffer_0 + out_buf: visbuf_5s_0 + log_level: INFO + acc_1: + kotekan_stage: visAccumulate + in_buf: gpu_output_split_buffer_1 + out_buf: visbuf_5s_1 + acc_2: + kotekan_stage: visAccumulate + in_buf: gpu_output_split_buffer_2 + out_buf: visbuf_5s_2 + acc_3: + kotekan_stage: visAccumulate + in_buf: gpu_output_split_buffer_3 + out_buf: visbuf_5s_3 + acc_4: + kotekan_stage: visAccumulate + in_buf: gpu_output_split_buffer_4 + out_buf: visbuf_5s_4 + acc_5: + kotekan_stage: visAccumulate + in_buf: gpu_output_split_buffer_5 + out_buf: visbuf_5s_5 + acc_6: + kotekan_stage: visAccumulate + in_buf: gpu_output_split_buffer_6 + out_buf: visbuf_5s_6 + acc_7: + kotekan_stage: visAccumulate + in_buf: gpu_output_split_buffer_7 + out_buf: visbuf_5s_7 + +merge_acc: + kotekan_stage: bufferMerge + in_bufs: + - visbuf_5s_0 + - visbuf_5s_1 + - visbuf_5s_2 + - visbuf_5s_3 + - visbuf_5s_4 + - visbuf_5s_5 + - visbuf_5s_6 + - visbuf_5s_7 + out_buf: visbuf_5s_merged + +vis_writer: + kotekan_stage: VisWriter + in_buf: visbuf_5s_merged + node_mode: true + file_type: hdf5 + log_level: INFO + root_path: /data/archive/ + freq_ids: [] + write_ev: False + file_length: 512 diff --git a/lib/core/Stage.hpp b/lib/core/Stage.hpp index b7ea4086b..f51379229 100644 --- a/lib/core/Stage.hpp +++ b/lib/core/Stage.hpp @@ -105,7 +105,8 @@ class Stage : public kotekanLogging { /// Helper defined to reduce the boiler plate needed to crate the /// standarized constructor in sub classes #define STAGE_CONSTRUCTOR(T) \ - T::T(Config& config, const std::string& unique_name, bufferContainer& buffer_container) : \ + T::T(kotekan::Config& config, const std::string& unique_name, \ + kotekan::bufferContainer& buffer_container) : \ Stage(config, unique_name, buffer_container, std::bind(&T::main_thread, this)) #endif /* KOTEKAN_STAGE_H */ diff --git a/lib/core/buffer.c b/lib/core/buffer.c index dd92c4a37..4afe14b60 100644 --- a/lib/core/buffer.c +++ b/lib/core/buffer.c @@ -721,9 +721,7 @@ void pass_metadata(struct Buffer* from_buf, int from_ID, struct Buffer* to_buf, return; } - struct metadataContainer* metadata_container = NULL; - - metadata_container = from_buf->metadata[from_ID]; + struct metadataContainer* metadata_container = from_buf->metadata[from_ID]; CHECK_ERROR_F(pthread_mutex_lock(&to_buf->lock)); @@ -842,6 +840,41 @@ void swap_frames(struct Buffer* from_buf, int from_frame_id, struct Buffer* to_b to_buf->frames[to_frame_id] = temp_frame; } +void safe_swap_frame(struct Buffer* src_buf, int src_frame_id, struct Buffer* dest_buf, + int dest_frame_id) { + assert(src_buf != dest_buf); + assert(src_buf != NULL); + assert(dest_buf != NULL); + assert(src_frame_id >= 0); + assert(src_frame_id < src_buf->num_frames); + assert(dest_frame_id >= 0); + assert(dest_frame_id < dest_buf->num_frames); + + // Buffer sizes must match exactly + if (src_buf->frame_size != dest_buf->frame_size) { + FATAL_ERROR_F("Buffer sizes must match for direct copy (%s.frame_size != %s.frame_size)", + src_buf->buffer_name, dest_buf->buffer_name); + } + + if (get_num_producers(dest_buf) > 1) { + FATAL_ERROR_F("Cannot swap/copy frames into dest buffer %s with more than one producer", + dest_buf->buffer_name); + } + + int num_consumers = get_num_consumers(src_buf); + + // Copy or transfer the data part. + if (num_consumers == 1) { + // Swap the frames + uint8_t* temp_frame = src_buf->frames[src_frame_id]; + src_buf->frames[src_frame_id] = dest_buf->frames[dest_frame_id]; + dest_buf->frames[dest_frame_id] = temp_frame; + } else if (num_consumers > 1) { + // Copy the frame data over, leaving the source intact + memcpy(dest_buf->frames[dest_frame_id], src_buf->frames[src_frame_id], src_buf->frame_size); + } +} + uint8_t* buffer_malloc(ssize_t len, int numa_node) { uint8_t* frame = NULL; diff --git a/lib/core/buffer.h b/lib/core/buffer.h index a0ec11ddc..a3e33c19f 100644 --- a/lib/core/buffer.h +++ b/lib/core/buffer.h @@ -550,6 +550,21 @@ void pass_metadata(struct Buffer* from_buf, int from_frame_id, struct Buffer* to void copy_metadata(struct Buffer* from_buf, int from_frame_id, struct Buffer* to_buf, int to_frame_id); +/** + * @brief Swaps a frame or performs a deep copy depending on the number of consumers on the + * source buffer. + * + * Like @c swap_frames(), but doesn't fail if there is more than one consumer on the source buffer. + * Does not pass or copy metadata. + * + * @param[in] src_buf The source buffer + * @param[in] src_frame_id The source frame ID + * @param[in] dest_buf The destination buffer + * @param[in] dest_frame_id The destination frame ID + */ +void safe_swap_frame(struct Buffer* src_buf, int src_frame_id, struct Buffer* dest_buf, + int dest_frame_id); + /** * @brief Tells the buffers to stop returning full/empty frames to consumers/producers * diff --git a/lib/dpdk/dpdkCore.cpp b/lib/dpdk/dpdkCore.cpp index a2ae7e28f..f6ff95227 100644 --- a/lib/dpdk/dpdkCore.cpp +++ b/lib/dpdk/dpdkCore.cpp @@ -19,6 +19,7 @@ #include // for unlikely #include // for RTE_PKTMBUF_HEADROOM #include // for rte_eal_init +#include // for rte_strerror, per_lcore__rte_errno, rte_errno #include // for ether_addr #include // for rte_eal_mp_remote_launch, rte_eal_mp_wait_lcore, SKIP... #include // for rte_lcore_count, rte_lcore_id @@ -59,7 +60,7 @@ dpdkCore::dpdkCore(Config& config, const string& unique_name, bufferContainer& b tx_ring_size = config.get_default(unique_name, "tx_ring_size", 512); num_mem_channels = config.get_default(unique_name, "num_mem_channels", 4); - init_mem_alloc = config.get_default(unique_name, "init_mem_alloc", 256); + init_mem_alloc = config.get_default(unique_name, "init_mem_alloc", "256"); // Setup the lcore mappings // Basically this is mapping the DPDK EAL framework way of assigning threads @@ -146,7 +147,8 @@ dpdkCore::dpdkCore(Config& config, const string& unique_name, bufferContainer& b mbuf_size, mbuf_cache_size, sizeof(struct rte_pktmbuf_pool_private), rte_pktmbuf_pool_init, nullptr, rte_pktmbuf_init, nullptr, node_id, 0); if (pool == nullptr) { - throw std::runtime_error("Cannot create DPDK mbuf pool."); + throw std::runtime_error("Cannot create DPDK mbuf pool: " + + std::string(rte_strerror(rte_errno))); } } mbuf_pools.push_back(pool); @@ -169,7 +171,7 @@ dpdkCore::dpdkCore(Config& config, const string& unique_name, bufferContainer& b void dpdkCore::create_handlers(bufferContainer& buffer_container) { // Create the handlers // TODO This could likely be refactored out of this system. - // The one problem is that we are using header only builds for efficency, + // The one problem is that we are using header only builds for efficiency, // so the normal factory model doesn't work here. vector handlers_block = config.get>(unique_name, "handlers"); uint32_t port = 0; @@ -230,10 +232,9 @@ void dpdkCore::dpdk_init(vector lcore_cpu_map, uint32_t master_lcore_cpu) { char* arg4 = (char*)malloc(dpdk_lcore_map.length() + 1); strncpy(arg4, dpdk_lcore_map.c_str(), dpdk_lcore_map.length() + 1); // Initial memory allocation - char arg5[] = "-m"; - char* arg6 = (char*)malloc(std::to_string(init_mem_alloc).length() + 1); - strncpy(arg6, std::to_string(init_mem_alloc).c_str(), - std::to_string(init_mem_alloc).length() + 1); + char arg5[] = "--socket-mem"; + char* arg6 = (char*)malloc(init_mem_alloc.length() + 1); + strncpy(arg6, init_mem_alloc.c_str(), init_mem_alloc.length() + 1); // Generate final options string for EAL initialization char* argv2[] = {&arg0[0], &arg1[0], &arg2[0], &arg3[0], &arg4[0], &arg5[0], &arg6[0], nullptr}; int argc2 = (int)(sizeof(argv2) / sizeof(argv2[0])) - 1; diff --git a/lib/dpdk/dpdkCore.hpp b/lib/dpdk/dpdkCore.hpp index 24fedd24f..5a09c4897 100644 --- a/lib/dpdk/dpdkCore.hpp +++ b/lib/dpdk/dpdkCore.hpp @@ -226,7 +226,7 @@ class dpdkCore : public kotekan::Stage { uint32_t num_mem_channels; /// Initial memory allocation in MB - uint32_t init_mem_alloc; + std::string init_mem_alloc; /// One of these exists per system port dpdkRXhandler** handlers; diff --git a/lib/dpdk/iceBoardStandard.hpp b/lib/dpdk/iceBoardStandard.hpp index 45032e4c7..87077df2a 100644 --- a/lib/dpdk/iceBoardStandard.hpp +++ b/lib/dpdk/iceBoardStandard.hpp @@ -42,6 +42,9 @@ * @buffer_format unit8_t array of flags * @buffer_metadata none * + * @conf fpga_dataset String. The dataset ID for the data being received from + * the F-engine. + * * @author Andre Renard */ class iceBoardStandard : public iceBoardHandler { @@ -62,7 +65,7 @@ class iceBoardStandard : public iceBoardHandler { /// The output buffer struct Buffer* out_buf; - /// The current frame + /// The current frame. uint8_t* out_frame; /// The ID of the current frame @@ -71,6 +74,9 @@ class iceBoardStandard : public iceBoardHandler { /// The flag buffer tracking lost samples struct Buffer* lost_samples_buf; + // Parameters saved from the config files + dset_id_t fpga_dataset; + /// The active lost sample frame uint8_t* lost_samples_frame; @@ -99,6 +105,8 @@ iceBoardStandard::iceBoardStandard(kotekan::Config& config, const std::string& u // We want to make sure the flag buffers are zeroed between uses. zero_frames(lost_samples_buf); + fpga_dataset = config.get_default("/fpga_dataset", "id", dset_id_t::null); + // Number of frames to capture before stopping, 0 = unlimited capture_n_frames = config.get_default(unique_name, "capture_n_frames", 0); @@ -200,6 +208,7 @@ inline bool iceBoardStandard::advance_frame(uint64_t new_seq, bool first_time) { ice_set_stream_id_t(out_buf, out_frame_id, port_stream_id); set_fpga_seq_num(out_buf, out_frame_id, new_seq); + set_dataset_id(out_buf, out_frame_id, fpga_dataset); return true; } diff --git a/lib/opencl/CMakeLists.txt b/lib/opencl/CMakeLists.txt index 5fe078781..b3dbd9c16 100644 --- a/lib/opencl/CMakeLists.txt +++ b/lib/opencl/CMakeLists.txt @@ -30,10 +30,11 @@ add_library( target_link_libraries( kotekan_opencl - INTERFACE ${OpenCL_LIBRARY} + INTERFACE ${OPENCL_LIBRARIES} PRIVATE libexternal kotekan_libs) + target_include_directories(kotekan_opencl PUBLIC .) -target_include_directories(kotekan_opencl SYSTEM INTERFACE ${OpenCL_INCLUDE_DIR}) +target_include_directories(kotekan_opencl SYSTEM INTERFACE ${OPENCL_INCLUDE_DIRS}) add_custom_target( kernel_copy diff --git a/lib/stages/BaseWriter.cpp b/lib/stages/BaseWriter.cpp index ba2f9b991..77be0c7f1 100644 --- a/lib/stages/BaseWriter.cpp +++ b/lib/stages/BaseWriter.cpp @@ -8,12 +8,14 @@ #include "datasetManager.hpp" // for dset_id_t, fingerprint_t, datasetManager #include "datasetState.hpp" // for metadataState, _factory_aliasdatasetState #include "factory.hpp" // for FACTORY -#include "kotekanLogging.hpp" // for FATAL_ERROR, INFO, WARN, DEBUG, logLevel +#include "kotekanLogging.hpp" // for INFO, WARN, FATAL_ERROR, DEBUG, logLevel #include "prometheusMetrics.hpp" // for Counter, Metrics, MetricFamily, Gauge #include "restServer.hpp" // for HTTP_RESPONSE, connectionInstance, restServer #include "version.h" // for get_git_commit_hash #include "visFile.hpp" // for visFileBundle, _factory_aliasvisFile +#include "fmt.hpp" // for format + #include // for copy, copy_backward, equal, max #include // for atomic_bool #include // for deque @@ -50,12 +52,15 @@ BaseWriter::BaseWriter(Config& config, const std::string& unique_name, acq_timeout = config.get_default(unique_name, "acq_timeout", 300); ignore_version = config.get_default(unique_name, "ignore_version", false); + // Get the list of buffers that this stage should connect to + in_buf = get_buffer("in_buf"); + register_consumer(in_buf, unique_name.c_str()); + // Get the type of the file we are writing // TODO: we may want to validate here rather than at creation time file_type = config.get_default(unique_name, "file_type", "hdf5fast"); if (!FACTORY(visFile)::exists(file_type)) { - FATAL_ERROR("Unknown file type '{}'", file_type); - return; + throw std::runtime_error(fmt::format("Unknown file type '{}'", file_type)); } file_length = config.get_default(unique_name, "file_length", 1024); @@ -78,15 +83,12 @@ BaseWriter::BaseWriter(Config& config, const std::string& unique_name, auto t = config.get_default>(unique_name, "critical_states", {}); for (const auto& state : t) { if (!FACTORY(datasetState)::exists(state)) { - FATAL_ERROR("Unknown datasetState type '{}' given as `critical_state`", state); + throw std::runtime_error( + fmt::format("Unknown datasetState type '{}' given as `critical_state`", state)); return; } critical_state_types.insert(state); } - - // Get the list of buffers that this stage should connect to - in_buf = get_buffer("in_buf"); - register_consumer(in_buf, unique_name.c_str()); } void BaseWriter::main_thread() { diff --git a/lib/stages/BufferSplit.cpp b/lib/stages/BufferSplit.cpp new file mode 100644 index 000000000..8b57ce330 --- /dev/null +++ b/lib/stages/BufferSplit.cpp @@ -0,0 +1,51 @@ +#include "BufferSplit.hpp" + +#include "StageFactory.hpp" // for REGISTER_KOTEKAN_STAGE, StageMakerTemplate +#include "buffer.h" // for mark_frame_empty, mark_frame_full, pass_metadata, register_c... +#include "visUtil.hpp" // for frameID, modulo + +#include // for max +#include // for atomic_bool +#include // for allocator_traits<>::value_type +#include // for uint8_t, uint32_t + +REGISTER_KOTEKAN_STAGE(BufferSplit); + +STAGE_CONSTRUCTOR(BufferSplit) { + in_buf = get_buffer("in_buf"); + register_consumer(in_buf, unique_name.c_str()); + + out_bufs = get_buffer_array("out_bufs"); + for (struct Buffer* out_buf : out_bufs) + register_producer(out_buf, unique_name.c_str()); +} + +BufferSplit::~BufferSplit() {} + +void BufferSplit::main_thread() { + + std::vector output_frame_ids; + for (auto& out_buf : out_bufs) + output_frame_ids.push_back(frameID(out_buf)); + + frameID input_frame_id(in_buf); + + while (!stop_thread) { + for (uint32_t i = 0; i < out_bufs.size(); ++i) { + uint8_t* input = wait_for_full_frame(in_buf, unique_name.c_str(), input_frame_id); + if (input == nullptr) + break; + uint8_t* output = + wait_for_empty_frame(out_bufs.at(i), unique_name.c_str(), output_frame_ids.at(i)); + if (output == nullptr) + break; + + safe_swap_frame(in_buf, input_frame_id, out_bufs.at(i), output_frame_ids.at(i)); + + pass_metadata(in_buf, input_frame_id, out_bufs.at(i), output_frame_ids.at(i)); + + mark_frame_empty(in_buf, unique_name.c_str(), input_frame_id++); + mark_frame_full(out_bufs.at(i), unique_name.c_str(), output_frame_ids.at(i)++); + } + } +} diff --git a/lib/stages/BufferSplit.hpp b/lib/stages/BufferSplit.hpp new file mode 100644 index 000000000..273788815 --- /dev/null +++ b/lib/stages/BufferSplit.hpp @@ -0,0 +1,48 @@ +#ifndef BUFFER_SPLIT_HPP +#define BUFFER_SPLIT_HPP + +#include "Config.hpp" // for Config +#include "Stage.hpp" // for Stage +#include "bufferContainer.hpp" // for bufferContainer + +#include // for string +#include // for vector + +/** + * @brief Splits the input buffer into multiple output frames in a round robin fashion. + * + * Uses zero copy operations if this stage is the only consumer, if not it uses a deep copy. + * This stage must be the only producer of the output buffers + * + * @par Buffers + * @buffer in_buf The source buffer + * @buffer_format Matches the output buffers + * @buffer_metadata Matches the output buffers + * + * @buffer out_bufs Array of buffers to fill with frames from in_buf + * @buffer_format any, but all must be the same type and match in_buf + * @buffer_metadata any, but all must be the same type match in_buf + * + * @author Andre Renard + */ +class BufferSplit : public kotekan::Stage { +public: + /// Constructor + BufferSplit(kotekan::Config& config, const std::string& unique_name, + kotekan::bufferContainer& buffer_container); + + /// Destructor + ~BufferSplit(); + + /// Thread for splitting the frames over the output buffers. + void main_thread() override; + +private: + /// Input buffer + struct Buffer* in_buf; + + /// The output buffers to put frames into + std::vector out_bufs; +}; + +#endif \ No newline at end of file diff --git a/lib/stages/CMakeLists.txt b/lib/stages/CMakeLists.txt index 2737f9cba..cc36a02d8 100644 --- a/lib/stages/CMakeLists.txt +++ b/lib/stages/CMakeLists.txt @@ -46,6 +46,7 @@ add_library( bufferCopy.cpp bufferSwitch.cpp bufferDelay.cpp + BufferSplit.cpp InputSubset.cpp # RFI Pipeline Processes rfiVDIF.cpp From b0ef2503744321c414a9ada3cf64d726dae132e3 Mon Sep 17 00:00:00 2001 From: james-s-willis <36906682+james-s-willis@users.noreply.github.com> Date: Tue, 2 Mar 2021 09:12:56 -0500 Subject: [PATCH 12/15] Debugging upchan kernel. (#925) * Adds new square wave mode to testDataGen, and new configuration file for generating fake raw HFB data. --- config/tests/chime_frb_verification.yaml | 2 +- config/tests/chime_verify_upchan.yaml | 300 +++++++++++++++++++++++ lib/stages/testDataGen.cpp | 15 +- 3 files changed, 315 insertions(+), 2 deletions(-) create mode 100644 config/tests/chime_verify_upchan.yaml diff --git a/config/tests/chime_frb_verification.yaml b/config/tests/chime_frb_verification.yaml index 1da740b09..93773da96 100644 --- a/config/tests/chime_frb_verification.yaml +++ b/config/tests/chime_frb_verification.yaml @@ -16,7 +16,7 @@ log_level: info num_elements: 2048 num_local_freq: 1 num_data_sets: 1 -samples_per_data_set: 49152 +samples_per_data_set: 4608 buffer_depth: 2 baseband_buffer_depth: 120 # 48Gb, 15s buffer. vbuffer_depth: 32 diff --git a/config/tests/chime_verify_upchan.yaml b/config/tests/chime_verify_upchan.yaml new file mode 100644 index 000000000..5a4d05424 --- /dev/null +++ b/config/tests/chime_verify_upchan.yaml @@ -0,0 +1,300 @@ +########################################## +# +# Upchannelisation verification +# +# Runs GPU FRB beamformer pipeline to test the upchannelizer. A square wave can be injected at the start or directly into the upchan kernel. +# The output from the upchan kernel is passed to HFBAccumulate and integration is written to a raw file. +# This uses 4608 samples for a quick run, change it to 49152 for the standard operation setting +# +# Author: James Willis +# +########################################## +--- +type: config +# Logging level can be one of: +# OFF, ERROR, WARN, INFO, DEBUG, DEBUG2 (case insensitive) +# Note DEBUG and DEBUG2 require a build with (-DCMAKE_BUILD_TYPE=Debug) +log_level: info +num_elements: 2048 +num_local_freq: 1 +num_data_sets: 1 +samples_per_data_set: 4608 +buffer_depth: 2 +baseband_buffer_depth: 120 # 48Gb, 15s buffer. +vbuffer_depth: 32 +num_links: 4 +timesamples_per_packet: 2 +cpu_affinity: [2,3,8,9] +block_size: 32 +num_gpus: 4 +link_map: [0,1,2,3] +freq_ids: [3, 777, 554] + +# Constants +sizeof_float: 4 +sizeof_int: 4 +sizeof_short: 2 + +dataset_manager: + use_dataset_broker: False + ds_broker_host: "127.0.0.1" + ds_broker_port: 12050 + +#FRB global options +downsample_time: 3 +downsample_freq: 8 +factor_upchan: 128 +factor_upchan_out: 16 +num_frb_total_beams: 1024 +frb_missing_gains: [1.0,0.0] +frb_scaling: 1.0 +reorder_map: [32,33,34,35,40,41,42,43,48,49,50,51,56,57,58,59,96,97,98,99, + 104,105,106,107,112,113,114,115,120,121,122,123,67,66,65,64, + 75,74,73,72,83,82,81,80,91,90,89,88,3,2,1,0,11,10,9,8,19,18, + 17,16,27,26,25,24,152,153,154,155,144,145,146,147,136,137,138, + 139,128,129,130,131,216,217,218,219,208,209,210,211,200,201, + 202,203,192,193,194,195,251,250,249,248,243,242,241,240,235, + 234,233,232,227,226,225,224,187,186,185,184,179,178,177,176, + 171,170,169,168,163,162,161,160,355,354,353,352,363,362,361, + 360,371,370,369,368,379,378,377,376,291,290,289,288,299,298, + 297,296,307,306,305,304,315,314,313,312,259,258,257,256,264, + 265,266,267,272,273,274,275,280,281,282,283,323,322,321,320, + 331,330,329,328,339,338,337,336,347,346,345,344,408,409,410, + 411,400,401,402,403,392,393,394,395,384,385,386,387,472,473, + 474,475,464,465,466,467,456,457,458,459,448,449,450,451,440, + 441,442,443,432,433,434,435,424,425,426,427,416,417,418,419, + 504,505,506,507,496,497,498,499,488,489,490,491,480,481,482, + 483,36,37,38,39,44,45,46,47,52,53,54,55,60,61,62,63,100,101, + 102,103,108,109,110,111,116,117,118,119,124,125,126,127,71,70, + 69,68,79,78,77,76,87,86,85,84,95,94,93,92,7,6,5,4,15,14,13,12, + 23,22,21,20,31,30,29,28,156,157,158,159,148,149,150,151,140, + 141,142,143,132,133,134,135,220,221,222,223,212,213,214,215, + 204,205,206,207,196,197,198,199,255,254,253,252,247,246,245, + 244,239,238,237,236,231,230,229,228,191,190,189,188,183,182, + 181,180,175,174,173,172,167,166,165,164,359,358,357,356,367, + 366,365,364,375,374,373,372,383,382,381,380,295,294,293,292, + 303,302,301,300,311,310,309,308,319,318,317,316,263,262,261, + 260,268,269,270,271,276,277,278,279,284,285,286,287,327,326, + 325,324,335,334,333,332,343,342,341,340,351,350,349,348,412, + 413,414,415,404,405,406,407,396,397,398,399,388,389,390,391, + 476,477,478,479,468,469,470,471,460,461,462,463,452,453,454, + 455,444,445,446,447,436,437,438,439,428,429,430,431,420,421, + 422,423,508,509,510,511,500,501,502,503,492,493,494,495,484, + 485,486,487] + +num_beams: 10 +# Pool +main_pool: + kotekan_metadata_pool: chimeMetadata + num_metadata_objects: 30 * buffer_depth + 5 * baseband_buffer_depth + +hfb_pool: + kotekan_metadata_pool: HFBMetadata + num_metadata_objects: 30 * buffer_depth + 5 * baseband_buffer_depth + +# Buffers +network_buffers: + num_frames: baseband_buffer_depth + frame_size: samples_per_data_set * num_elements * num_local_freq * num_data_sets + metadata_pool: main_pool + network_buffer_0: + kotekan_buffer: standard + +upchan_test: + num_frames: baseband_buffer_depth + frame_size: num_elements * (samples_per_data_set + 64) * sizeof_float * 2 + metadata_pool: main_pool + upchan_in: + kotekan_buffer: standard + +gpu_beamform_output_buffers: + num_frames: buffer_depth + frame_size: num_data_sets * (samples_per_data_set/downsample_time/downsample_freq) * num_frb_total_beams * sizeof_float + metadata_pool: main_pool + gpu_beamform_output_buffer_0: + kotekan_buffer: standard + +gain_frb_buffer: + num_frames: buffer_depth + frame_size: 2048 * 2 * sizeof_float + metadata_pool: main_pool + kotekan_buffer: standard + +gain_tracking_buffer: + num_frames: buffer_depth + frame_size: 2048 * 2 * num_beams * sizeof_float + metadata_pool: main_pool + kotekan_buffer: standard + +gpu_beamform_hfb_output_buffers: + num_frames: buffer_depth + frame_size: num_data_sets * factor_upchan * num_frb_total_beams * sizeof_float + metadata_pool: main_pool + gpu_beamform_hfb_output_buffer_0: + kotekan_buffer: standard + +beamform_hfb_output_buffers: + num_frames: buffer_depth + metadata_pool: main_pool + beamform_hfb_output_buffer_0: + metadata_pool: hfb_pool + kotekan_buffer: hfb + beamform_hfb_output_buffer_read: + metadata_pool: hfb_pool + kotekan_buffer: hfb + +lost_samples_buffer: + kotekan_buffer: standard + num_frames: 2 * buffer_depth + frame_size: samples_per_data_set * num_local_freq * num_data_sets + metadata_pool: main_pool + +compressed_lost_samples_buffer: + kotekan_buffer: standard + num_frames: 2 * buffer_depth + frame_size: samples_per_data_set * num_data_sets / (factor_upchan * downsample_time) * sizeof_int + metadata_pool: main_pool + +compressed_lost_samples_buffer_cpy: + kotekan_buffer: standard + num_frames: 2 * buffer_depth + frame_size: samples_per_data_set * num_data_sets / (factor_upchan * downsample_time) * sizeof_int + metadata_pool: main_pool + +cpu_affinity: [2,3,8,9] + +gen_data: + test_data_gen_network: + type: square + value: 153 + kotekan_stage: testDataGen + out_buf: network_buffer_0 + test_data_gen_lost_samples: + type: const + value: 0 + kotekan_stage: testDataGen + out_buf: lost_samples_buffer + test_data_gen_upchan: + type: square + value: 153 + kotekan_stage: testDataGen + out_buf: upchan_in + +compress_lost_samples: + kotekan_stage: compressLostSamples + compression_factor: factor_upchan * downsample_time + zero_all_in_group: true + in_buf: lost_samples_buffer + out_buf: compressed_lost_samples_buffer + +copy: + kotekan_stage: bufferCopy + in_buf: compressed_lost_samples_buffer + out_bufs: + - compressed_lost_samples_buffer_cpy + +gpu: + kernel_path: "../lib/hsa/kernels/" + commands: &command_list + - name: hsaInputData + - name: hsaHostToDeviceCopy + in_buf: hfb_compressed_lost_samples_buf + gpu_memory_name: hfb_compressed_lost_samples + # - name: hsaHostToDeviceCopy + # in_buf: upchan_buf + # gpu_memory_name: upchan + - name: hsaOutputDataZero + - name: hsaAsyncCopyGain + - name: hsaBarrier + - name: hsaBeamformReorder + - name: hsaBeamformKernel + - name: hsaBeamformTranspose +# - name: hsaKVFFT + - name: hsaBeamformUpchanHFB + - name: hsaBeamformHFBSum + - name: hsaBeamformOutputData + - name: hsaBeamformHFBOutputData + enable_delay: true + cpu_affinity: [2, 3, 8, 9] + delay_max_fraction: 2.0 + block_size: 32 + buffer_depth: 1 + n_intg: 24576 + frame_arrival_period: samples_per_data_set / 390625 + gpu_0: + kotekan_stage: hsaProcess + gpu_id: 0 + ew_spacing: [0.0, 0.1, 0.2, 0.3] + northmost_beam: 90.0 + commands: *command_list + in_buffers: + network_buf: network_buffer_0 + gain_frb_buf: gain_frb_buffer + hfb_compressed_lost_samples_buf: compressed_lost_samples_buffer_cpy + # upchan_buf: upchan_in + out_buffers: + beamform_output_buf: gpu_beamform_output_buffer_0 + beamform_hfb_output_buf: gpu_beamform_hfb_output_buffer_0 + +hyper_fine_beam: + num_frames_to_integrate: 4 + good_samples_threshold: 0.95 + integrate_hfb_data: + kotekan_stage: HFBAccumulate + log_level: debug + hfb_input_buf: gpu_beamform_hfb_output_buffer_0 + compressed_lost_samples_buf: compressed_lost_samples_buffer + hfb_output_buf: beamform_hfb_output_buffer_0 + +hfb: + file_length: 1 + file_type: hfbraw + root_path: . + kotekan_stage: HFBWriter + in_buf: beamform_hfb_output_buffer_0 + instrument_name: chimeHFB + +read_gain: + kotekan_stage: ReadGain + updatable_config: + gain_frb: /updatable_config/frb_gain + gain_tracking: /updatable_config/tracking_gain + in_buf: network_buffer_0 + gain_frb_buf: gain_frb_buffer + gain_tracking_buf: gain_tracking_buffer + +updatable_config: + frb_gain: + kotekan_update_endpoint: json + frb_gain_dir: /mnt/frb-archiver/GainFiles/Latest_FRB + tracking_gain: + 0: + kotekan_update_endpoint: json + gain_dir: /mnt/frb-archiver/daily_gain_solutions/Latest_PSR + 1: + kotekan_update_endpoint: json + gain_dir: /mnt/frb-archiver/daily_gain_solutions/Latest_PSR + 2: + kotekan_update_endpoint: json + gain_dir: /mnt/frb-archiver/daily_gain_solutions/Latest_PSR + 3: + kotekan_update_endpoint: json + gain_dir: /mnt/frb-archiver/daily_gain_solutions/Latest_PSR + 4: + kotekan_update_endpoint: json + gain_dir: /mnt/frb-archiver/daily_gain_solutions/Latest_PSR + 5: + kotekan_update_endpoint: json + gain_dir: /mnt/frb-archiver/daily_gain_solutions/Latest_PSR + 6: + kotekan_update_endpoint: json + gain_dir: /mnt/frb-archiver/daily_gain_solutions/Latest_PSR + 7: + kotekan_update_endpoint: json + gain_dir: /mnt/frb-archiver/daily_gain_solutions/Latest_PSR + 8: + kotekan_update_endpoint: json + gain_dir: /mnt/frb-archiver/daily_gain_solutions/Latest_PSR + 9: + kotekan_update_endpoint: json + gain_dir: /mnt/frb-archiver/daily_gain_solutions/Latest_PSR diff --git a/lib/stages/testDataGen.cpp b/lib/stages/testDataGen.cpp index 82bb4a510..c775a7923 100644 --- a/lib/stages/testDataGen.cpp +++ b/lib/stages/testDataGen.cpp @@ -45,7 +45,8 @@ testDataGen::testDataGen(Config& config, const std::string& unique_name, buf = get_buffer("out_buf"); register_producer(buf, unique_name.c_str()); type = config.get(unique_name, "type"); - assert(type == "const" || type == "random" || type == "ramp" || type == "tpluse"); + assert(type == "const" || type == "random" || type == "ramp" || type == "tpluse" + || type == "square"); if (type == "const" || type == "random" || type == "ramp") value = config.get(unique_name, "value"); _pathfinder_test_mode = config.get_default(unique_name, "pathfinder_test_mode", false); @@ -155,6 +156,18 @@ void testDataGen::main_thread() { frame[j] = temp_output; } else if (type == "tpluse") { frame[j] = seq_num + j / num_elements + j % num_elements; + } else if (type == "square") { + unsigned char new_real; + unsigned char new_imaginary; + if ((j / num_elements) % 8 < 4) { + new_real = 0; + new_imaginary = 0; + } else { + new_real = 4; + new_imaginary = 0; + } + temp_output = ((new_real << 4) & 0xF0) + (new_imaginary & 0x0F); + frame[j] = temp_output; } } DEBUG("Generated a {:s} test data set in {:s}[{:d}]", type, buf->buffer_name, frame_id); From e3b9edee6ba24cdab9e964ff0b2fdceb783d7e79 Mon Sep 17 00:00:00 2001 From: james-s-willis <36906682+james-s-willis@users.noreply.github.com> Date: Thu, 11 Mar 2021 17:19:10 -0500 Subject: [PATCH 13/15] Set the output dataset ID for all datasets when the incoming dataset ID changes. (#934) --- lib/stages/visAccumulate.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/stages/visAccumulate.cpp b/lib/stages/visAccumulate.cpp index e406ee113..388dd4ddc 100644 --- a/lib/stages/visAccumulate.cpp +++ b/lib/stages/visAccumulate.cpp @@ -285,8 +285,10 @@ void visAccumulate::main_thread() { base_dataset_id = dm.add_dataset(base_dataset_states, *ds_id_in); DEBUG("Registered base dataset: {}", base_dataset_id) - // Set the output dataset ID for the main visibility accumulation - gated_datasets.at(0).output_dataset_id = base_dataset_id; + // Set the output dataset ID for all datasets + for (auto& state : gated_datasets) { + state.output_dataset_id = register_gate_dataset(*state.spec.get()); + } } int32_t* input = (int32_t*)in_frame; From e1186cad901692d4f70f1fa44f55d9b5f126e2a1 Mon Sep 17 00:00:00 2001 From: james-s-willis <36906682+james-s-willis@users.noreply.github.com> Date: Mon, 15 Mar 2021 13:32:52 -0400 Subject: [PATCH 14/15] Add acquisition type to file writer (#933) * Prefix raw absorber file name with 'hfb_'. * Move acquisition and file name format to Vis/HFBWriters. * Use named arguments when setting acq and file names. --- config/chime_science_run_recv_hfb.yaml | 2 +- config/tests/chime_science_run_recv_hfb.yaml | 2 +- config/tests/test_HFBRawReader.yaml | 2 +- config/tests/test_hfb_weights.yaml | 2 +- lib/stages/BaseWriter.cpp | 5 +---- lib/stages/BaseWriter.hpp | 6 ++++++ lib/stages/HFBWriter.cpp | 5 ++++- lib/stages/VisWriter.cpp | 5 ++++- lib/utils/visFile.cpp | 5 +++-- lib/utils/visFile.hpp | 21 ++++++++++---------- 10 files changed, 32 insertions(+), 23 deletions(-) diff --git a/config/chime_science_run_recv_hfb.yaml b/config/chime_science_run_recv_hfb.yaml index 798a846c1..f4c102b5c 100644 --- a/config/chime_science_run_recv_hfb.yaml +++ b/config/chime_science_run_recv_hfb.yaml @@ -54,7 +54,7 @@ write_hfb: kotekan_stage: HFBWriter in_buf: hfbbuf_10s - instrument_name: chimeHFB + instrument_name: chime buffer_status: kotekan_stage: bufferStatus diff --git a/config/tests/chime_science_run_recv_hfb.yaml b/config/tests/chime_science_run_recv_hfb.yaml index a3e3e30e0..99aec03ab 100644 --- a/config/tests/chime_science_run_recv_hfb.yaml +++ b/config/tests/chime_science_run_recv_hfb.yaml @@ -55,7 +55,7 @@ write_hfb: kotekan_stage: HFBWriter in_buf: hfbbuf_10s - instrument_name: chimeHFB + instrument_name: chime #buffer_status: # kotekan_stage: bufferStatus diff --git a/config/tests/test_HFBRawReader.yaml b/config/tests/test_HFBRawReader.yaml index 4ca191cf2..3c47e7471 100644 --- a/config/tests/test_HFBRawReader.yaml +++ b/config/tests/test_HFBRawReader.yaml @@ -37,7 +37,7 @@ hfb_pool: read_raw: log_level: debug - infile: 20210209T183459Z_chimeHFB_corr/00000000_0000 + infile: 20210209T183459Z_chime_hfb/hfb_00000000_0000 kotekan_stage: HFBRawReader max_read_rate: 0.0 out_buf: read_buffer diff --git a/config/tests/test_hfb_weights.yaml b/config/tests/test_hfb_weights.yaml index 9fa385f19..600a374cb 100644 --- a/config/tests/test_hfb_weights.yaml +++ b/config/tests/test_hfb_weights.yaml @@ -102,4 +102,4 @@ write_hfb: kotekan_stage: HFBWriter in_buf: beamform_hfb_output_buffer_0 - instrument_name: chimeHFB + instrument_name: chime diff --git a/lib/stages/BaseWriter.cpp b/lib/stages/BaseWriter.cpp index 77be0c7f1..6784c9170 100644 --- a/lib/stages/BaseWriter.cpp +++ b/lib/stages/BaseWriter.cpp @@ -143,15 +143,12 @@ void BaseWriter::init_acq(dset_id_t ds_id) { return; } - // TODO: chunk ID is not really supported now. Just set it to zero. - uint32_t chunk_id = 0; - // Construct metadata auto metadata = make_metadata(ds_id); try { acq.file_bundle = std::make_unique( - file_type, root_path, instrument_name, metadata, chunk_id, file_length, window, + file_type, root_path, acq_fmt, file_fmt, metadata, file_length, window, kotekan::logLevel(_member_log_level), ds_id, file_length); } catch (std::exception& e) { FATAL_ERROR("Failed creating file bundle for new acquisition: {:s}", e.what()); diff --git a/lib/stages/BaseWriter.hpp b/lib/stages/BaseWriter.hpp index 3342606ee..7cabeb2bf 100644 --- a/lib/stages/BaseWriter.hpp +++ b/lib/stages/BaseWriter.hpp @@ -142,6 +142,12 @@ class BaseWriter : public kotekan::Stage { // Parameters saved from the config files std::string instrument_name; + // Acquisition name format + std::string acq_fmt; + + // File name format + std::string file_fmt; + private: /// Construct the set of metadata virtual std::map make_metadata(dset_id_t ds_id) = 0; diff --git a/lib/stages/HFBWriter.cpp b/lib/stages/HFBWriter.cpp index a1d6b83b8..3d16e64d3 100644 --- a/lib/stages/HFBWriter.cpp +++ b/lib/stages/HFBWriter.cpp @@ -44,7 +44,10 @@ REGISTER_KOTEKAN_STAGE(HFBWriter); HFBWriter::HFBWriter(kotekan::Config& config, const std::string& unique_name, kotekan::bufferContainer& buffer_container) : - BaseWriter(config, unique_name, buffer_container){}; + BaseWriter(config, unique_name, buffer_container) { + acq_fmt = "{acq_start:%Y%m%dT%H%M%SZ}_" + instrument_name + "_hfb"; + file_fmt = "hfb_{seconds_since_start:08d}_0000"; +}; /// Construct the set of metadata std::map HFBWriter::make_metadata(dset_id_t ds_id) { diff --git a/lib/stages/VisWriter.cpp b/lib/stages/VisWriter.cpp index 6a949a548..1f0e79949 100644 --- a/lib/stages/VisWriter.cpp +++ b/lib/stages/VisWriter.cpp @@ -43,7 +43,10 @@ REGISTER_KOTEKAN_STAGE(VisWriter); VisWriter::VisWriter(kotekan::Config& config, const std::string& unique_name, kotekan::bufferContainer& buffer_container) : - BaseWriter(config, unique_name, buffer_container){}; + BaseWriter(config, unique_name, buffer_container) { + acq_fmt = "{acq_start:%Y%m%dT%H%M%SZ}_" + instrument_name + "_corr"; + file_fmt = "{seconds_since_start:08d}_0000"; +}; /// Construct the set of metadata std::map VisWriter::make_metadata(dset_id_t ds_id) { diff --git a/lib/utils/visFile.cpp b/lib/utils/visFile.cpp index cbb2690ee..519600a8d 100644 --- a/lib/utils/visFile.cpp +++ b/lib/utils/visFile.cpp @@ -95,7 +95,7 @@ void visFileBundle::add_file(time_ctype first_time) { // Start the acq and create the directory if required if (acq_name.empty()) { // Format the time (annoyingly you still have to use streams for this) - acq_name = fmt::format("{:%Y%m%dT%H%M%SZ}_{:s}_corr", *std::gmtime(&t), instrument_name); + acq_name = fmt::format(acq_fmt, fmt::arg("acq_start", *std::gmtime(&t))); // Set the acq fields on the instance acq_start_time = first_time.ctime; @@ -106,7 +106,8 @@ void visFileBundle::add_file(time_ctype first_time) { // Construct the name of the new file uint32_t time_since_start = (uint32_t)(first_time.ctime - acq_start_time); - std::string file_name = fmt::format(fmt("{:08d}_{:04d}"), time_since_start, freq_chunk); + std::string file_name = + fmt::format(file_fmt, fmt::arg("seconds_since_start", time_since_start)); // Create the file, create room for the first sample and add into the file map auto file = mk_file(file_name, acq_name, root_path); diff --git a/lib/utils/visFile.hpp b/lib/utils/visFile.hpp index 7e04b4a8e..6445ed68f 100644 --- a/lib/utils/visFile.hpp +++ b/lib/utils/visFile.hpp @@ -125,9 +125,9 @@ class visFileBundle : public kotekan::kotekanLogging { * Initialise the file bundle * @param type Type of the files to write. * @param root_path Directory to write into. - * @param instrument_name Instrument name (e.g. chime) + * @param acq_fmt Acquistion name format (e.g. chime_corr) + * @param file_fmt File name format (e.g. {"hfb_", "freq_chunk"}) * @param metadata Textual metadata to write into the files. - * @param freq_chunk ID of the frequency chunk being written * @param rollover Maximum time length of file. * @param window_size Number of "active" timesamples to keep. * @param log_level kotekan log level for any logging generated by the visFileBundle instance @@ -136,9 +136,8 @@ class visFileBundle : public kotekan::kotekanLogging { * @warning The directory will not be created if it doesn't exist. **/ template - visFileBundle(const std::string& type, const std::string& root_path, - const std::string& instrument_name, - const std::map& metadata, int freq_chunk, + visFileBundle(const std::string& type, const std::string& root_path, const std::string& acq_fmt, + const std::string& file_fmt, const std::map& metadata, size_t rollover, size_t window_size, const kotekan::logLevel log_level, InitArgs... args); @@ -173,8 +172,8 @@ class visFileBundle : public kotekan::kotekanLogging { const std::string root_path; - const std::string instrument_name; - const int freq_chunk; + const std::string acq_fmt; + const std::string file_fmt; size_t rollover; size_t window_size; @@ -191,13 +190,13 @@ class visFileBundle : public kotekan::kotekanLogging { // parameter pack into the lambda perfectly. template inline visFileBundle::visFileBundle(const std::string& type, const std::string& root_path, - const std::string& instrument_name, + const std::string& acq_fmt, const std::string& file_fmt, const std::map& metadata, - int freq_chunk, size_t rollover, size_t window_size, + size_t rollover, size_t window_size, const kotekan::logLevel log_level, InitArgs... args) : root_path(root_path), - instrument_name(instrument_name), - freq_chunk(freq_chunk), + acq_fmt(acq_fmt), + file_fmt(file_fmt), rollover(rollover), window_size(window_size) { From 84299168b56a7a2145562e5b764886d125233669 Mon Sep 17 00:00:00 2001 From: james-s-willis <36906682+james-s-willis@users.noreply.github.com> Date: Tue, 23 Mar 2021 18:46:40 -0400 Subject: [PATCH 15/15] Missing assignment of offset into dataset ID array. (#936) --- lib/stages/Transpose.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/stages/Transpose.cpp b/lib/stages/Transpose.cpp index be9181d76..66e4fc5ca 100644 --- a/lib/stages/Transpose.cpp +++ b/lib/stages/Transpose.cpp @@ -62,8 +62,6 @@ void Transpose::main_thread() { uint32_t frames_so_far = 0; // frequency and time indices within chunk uint32_t fi = 0, ti = 0; - // offset for copying into buffer - uint32_t offset = 0; // Flags to indicate incomplete chunks bool t_edge = false, f_edge = false; @@ -144,6 +142,9 @@ void Transpose::main_thread() { auto [frame_size, fpga_seq_total, frame_ds_id] = get_frame_data(); (void)frame_size; + // Offset into dset_id list + uint32_t offset = fi * write_t; + // Parse the dataset ID if (fpga_seq_total == 0 && frame_ds_id == dset_id_t::null) { DEBUG2("Got an empty frame.");