Merge pull request #159 from TGSAI/readerExample

Statically build+link example with open data access

Author: BrianMichell

examples/seismic_reader/CMakeLists.txt

@@ -0,0 +1,97 @@
+cmake_minimum_required(VERSION 3.24)
+project(SeismicReader VERSION 1.0.0 LANGUAGES CXX)
+
+# Set the C++ standard to C++17
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+# Specify MDIO installation directory (set by bootstrap.sh)
+set(MDIO_INSTALL_DIR "${CMAKE_SOURCE_DIR}/inst")
+
+# Add library directories
+link_directories(
+    ${MDIO_INSTALL_DIR}/lib
+    ${MDIO_INSTALL_DIR}/lib/drivers
+)
+
+# CURL is built and installed by the MDIO installer; no need for separate find_package.
+
+# Define MDIO linker flags - corrected format to match SCons configuration
+set(MDIO_LINK_FLAGS
+    "-Wl,-rpath,${MDIO_INSTALL_DIR}/lib,-rpath,${MDIO_INSTALL_DIR}/lib/drivers,--whole-archive,-L${MDIO_INSTALL_DIR}/lib,-L${MDIO_INSTALL_DIR}/lib/drivers,\
+-lnlohmann_json_schema_validator,\
+-ltensorstore_driver_zarr_bzip2_compressor,\
+-ltensorstore_driver_zarr_driver,\
+-ltensorstore_driver_zarr_spec,\
+-ltensorstore_driver_zarr_zlib_compressor,\
+-ltensorstore_driver_zarr_zstd_compressor,\
+-ltensorstore_driver_zarr_blosc_compressor,\
+-ltensorstore_kvstore_gcs_http,\
+-ltensorstore_kvstore_gcs_gcs_resource,\
+-ltensorstore_kvstore_gcs_validate,\
+-ltensorstore_kvstore_gcs_http_gcs_resource,\
+-ltensorstore_kvstore_s3,\
+-ltensorstore_kvstore_s3_aws_credentials_resource,\
+-ltensorstore_kvstore_s3_credentials_default_credential_provider,\
+-ltensorstore_kvstore_s3_credentials_environment_credential_provider,\
+-ltensorstore_kvstore_s3_credentials_file_credential_provider,\
+-ltensorstore_kvstore_s3_credentials_ec2_credential_provider,\
+-ltensorstore_kvstore_s3_s3_metadata,\
+-ltensorstore_kvstore_s3_s3_resource,\
+-ltensorstore_driver_json,\
+-ltensorstore_internal_cache_cache_pool_resource,\
+-ltensorstore_internal_data_copy_concurrency_resource,\
+-ltensorstore_kvstore_file,\
+-ltensorstore_internal_file_io_concurrency_resource,\
+-ltensorstore_internal_cache_kvs_backed_chunk_cache,\
+-labsl,\
+-lblosc,\
+-ltensorstore,\
+-lre2,\
+-lriegeli,\
+-ltinyxml2_tinyxml2,\
+-lcurl,\
+-lopenssl,\
+--no-whole-archive,\
+-lpthread,\
+-lm"
+)
+
+# Debug: Print out the MDIO_LINK_FLAGS
+message(STATUS "MDIO_LINK_FLAGS: ${MDIO_LINK_FLAGS}")
+
+# Create the executable target. (Assumes main.cpp exists in your project.)
+add_executable(read main.cc)
+
+# Append the linker flags to the target's link flags.
+set_target_properties(read PROPERTIES LINK_FLAGS "${MDIO_LINK_FLAGS}")
+
+# Add compile definitions
+target_compile_definitions(read PRIVATE HAVE_MDIO MAX_NUM_SLICES=32)
+
+# Add MDIO and third-party include directories for target 'read'
+# Collect all immediate subdirectories from the MDIO include directory.
+file(GLOB CHILD_DIRS LIST_DIRECTORIES true "${MDIO_INSTALL_DIR}/include/*")
+
+# Also include the top-level include directory so that headers like "mdio/mdio.h" are found.
+list(INSERT CHILD_DIRS 0 "${MDIO_INSTALL_DIR}/include")
+
+# Remove any unwanted directories (for example, the gtest-src directory)
+foreach(dir ${CHILD_DIRS})
+    get_filename_component(basename ${dir} NAME)
+    if(basename MATCHES "gtest-src")
+        list(REMOVE_ITEM CHILD_DIRS "${dir}")
+    endif()
+endforeach()
+
+# Append additional directories that the installer uses but might not be one-level deep.
+list(APPEND CHILD_DIRS 
+    "${MDIO_INSTALL_DIR}/include/nlohmann_json-src/include"
+    "${MDIO_INSTALL_DIR}/include/half-src/include"
+)
+
+target_include_directories(read PRIVATE ${CHILD_DIRS})
+
+# Debug: Print out the include directories for target 'read'
+get_target_property(READ_INCLUDES read INCLUDE_DIRECTORIES)
+message(STATUS "Target 'read' include directories: ${READ_INCLUDES}")

examples/seismic_reader/README.md

@@ -0,0 +1,64 @@
+# Seismic data reader
+
+The purpose of this example is to demonstrate one method of integrating the **MDIO** C++ library into a seismic data reader into an existing project.
+
+Due to the variety of target build systems and differing complexities of integrating the library with existing CMake projects, this example will also expose a way to build MDIO and its dependencies as a set of shared and linkable libraries and integrate it that way.
+
+## Table of contents
+- [Concepts demonstrated](#concepts-demonstrated)
+- [Overview](#overview)
+- [Running](#running)
+- [Glossary](#glossary)
+
+## Concepts demonstrated
+- Index-based slicing
+- Value-based slicing
+- Dimension coordinates
+- Coordinates
+- Caching
+
+Chunk-aligned opreations were briefly discussed but not examined in great detail for this example.
+
+## Overview
+
+This example assumes that you are able to build the main branch of **MDIO**.
+
+This example will perform the following steps:
+
+1. Clone the installer to this directory.
+2. Run the installer, building and installing several archives and shared files.
+3. Link the installed files in the `CMakeLists.txt`.
+4. Demonstrate opening a Dataset with a configurable cache.
+5. Demonstrate acquiring the corner points for the UTM grid on the Open Poseidon dataset from S3.
+    - Calculates the latitude and longitude coordinates for the corner points.
+    - Provides a web link to display the surface area on a map.
+6. Demonstrate acquiring the inline and crossline extents.
+7. Demonstrate an index-based slice for chunk-aligned, tracewise processing.
+    - Calculates basic statistics for a small section of the dataset.
+    - Tracks the highest (peak) and lowest (trough) amplitudes and their actual inline/crossline coordinate pairs.
+8. Demonstrate a value-based slice for more targeted analysis.
+    - Corolates the cdp-x and cdp-y coordinate pair for the peak and trough values.
+    - Provides a web link to display each one's location on a map.
+
+## Running
+
+```bash
+$ ./bootstrap.sh
+$ cd build
+$ cmake ..
+$ make -j
+```
+
+The bootstrap shell script will build and install the mdio-cpp library as a set of shared and static objects.
+
+It will then set up the build directory if it completes without error.
+
+After the program has finished building it can be run with `./read`.
+
+## Glossary
+- Dimension coordinate: A 1-dimensional Variable that describes a dimension of the dataset. 
+- Coordinate: A Variable that helps describe data outside of its natural (logical) domain. May be greater than 1-dimensional.
+- Slicing: The act of subsetting a dataset along one or more dimension coordinates. A subset of a dataset is still considered a dataset.
+- Index-based slicing (*isel*): Subsetting a dataset based on the logical indices of its *dimension coordinate*(s).
+- Value-based slicing (*sel*): Subsetting a dataset based on the values contained by its *dimension coordinate*(s).
+- Chunk-aligned: Slicing the data along its logical chunk boundries for efficient I/O performance.

examples/seismic_reader/bootstrap.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# bootstrap.sh
+#
+# This script bootstraps the MDIO installation.
+# It clones the mdio-cpp-installer repository and installs MDIO
+# into the current directory (in the "inst" folder).
+#
+# Instruction from the installer:
+#   $ ./install.sh [install_directory] [mdio_tag]
+# (the mdio_tag is ignored)
+# The hidden --curl flag is also used.
+
+# Define installation directory as the "inst" folder in the current directory
+INST_DIR="$(pwd)/inst"
+
+echo "Installing MDIO in: $INST_DIR"
+
+# Clone the installer repo if it does not already exist
+if [ ! -d "mdio-cpp-installer" ]; then
+    echo "Cloning mdio-cpp-installer repository..."
+    git clone https://github.com/BrianMichell/mdio-cpp-installer.git || { 
+        echo "Failed to clone repository."; exit 1; 
+    }
+fi
+
+# Change into the installer directory
+cd mdio-cpp-installer || { 
+    echo "Failed to change directory into mdio-cpp-installer."; exit 1; 
+}
+
+# Make sure the installer script is executable
+chmod +x install.sh
+
+# Run the installer: Pass the installation directory, a dummy tag,
+# and the hidden "--curl" flag.
+echo "Running MDIO installer..."
+./install.sh "$INST_DIR" dummy_tag --curl || { 
+    echo "MDIO installation failed."; exit 1; 
+}
+
+echo "MDIO installation completed successfully."
+
+# Return to the original directory
+cd ..
+
+mkdir build

examples/seismic_reader/main.cc

@@ -0,0 +1,173 @@
+// Copyright 2025 TGS
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//    http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "main.hh"
+
+int main() {
+    const std::string path = "s3://tgs-opendata-poseidon/full_stack_agc.mdio";
+    const uint64_t cache_size_bytes = 1024ULL * 1024ULL * 1024ULL * 5ULL;  // Use a 5GiB cache.
+    mdio::Future<mdio::Dataset> dsFut = OpenDataset(path, cache_size_bytes);
+
+    if (!dsFut.status().ok()) {
+        std::cerr << "Failed to open dataset: " << dsFut.status().message() << std::endl;
+        return 1;
+    }
+
+    mdio::Dataset ds = dsFut.value();
+    std::cout << "Dataset opened successfully" << std::endl;
+    std::cout << ds << std::endl;
+
+    auto cdpsFut = GetUTMCoords(ds);
+    if (!cdpsFut.status().ok()) {
+        std::cerr << "Failed to get UTM coordinates: " << cdpsFut.status().message() << std::endl;
+        return 1;
+    }
+
+    auto cdps = cdpsFut.value();
+    auto cdp_x = cdps.first.value();
+    auto cdp_y = cdps.second.value();
+
+    auto cdp_x_extents = GetExtents<mdio::dtypes::float64_t>(cdp_x);
+    auto cdp_y_extents = GetExtents<mdio::dtypes::float64_t>(cdp_y);
+
+    std::cout << "========CDP Coordinates=========" << std::endl;
+    std::cout << "CDP X extents: " << cdp_x_extents << std::endl;
+    std::cout << "CDP Y extents: " << cdp_y_extents << std::endl;
+    utm::print_corners(cdp_x_extents, cdp_y_extents);
+    std::cout << std::endl;
+    // This displays the maximum area of the extents on a web map.
+    // The surveys actual polygon is not computed in this example.
+    utm::web_display(cdp_x_extents, cdp_y_extents);
+    std::cout << std::endl;
+    std::cout << "=================================" << std::endl << std::endl;
+
+    auto linesFut = GetInlineCrossline(ds);
+    if (!linesFut.status().ok()) {
+        std::cerr << "Failed to get inline and crossline coordinates: " << linesFut.status().message() << std::endl;
+        return 1;
+    }
+
+    auto lines = linesFut.value();
+    auto il = lines.first.value();
+    auto xl = lines.second.value();
+
+    auto il_extents = GetExtents<mdio::dtypes::uint16_t>(il);
+    auto xl_extents = GetExtents<mdio::dtypes::uint16_t>(xl);
+
+    std::cout << "Inline extents: " << il_extents << std::endl;
+    std::cout << "Crossline extents: " << xl_extents << std::endl;
+
+    auto statsResult = stats::CalculateVolumeStatistics<mdio::dtypes::float32_t, mdio::dtypes::uint16_t>(ds);
+    if (!statsResult.status().ok()) {
+        std::cerr << "Failed to get volume statistics: " << statsResult.status().message() << std::endl;
+        return 1;
+    }
+
+    auto stats = statsResult.value();
+    std::cout << stats << std::endl;
+
+    // Now that we have the statistics, lets pinpoint where our peak and trough amplitudes are located on the world.
+    mdio::ValueDescriptor<mdio::dtypes::uint16_t> il_peak = {"inline", stats.peak_amplitude_inline};
+    mdio::ValueDescriptor<mdio::dtypes::uint16_t> xl_peak = {"crossline", stats.peak_amplitude_crossline};
+    mdio::ValueDescriptor<mdio::dtypes::uint16_t> il_trough = {"inline", stats.trough_amplitude_inline};
+    mdio::ValueDescriptor<mdio::dtypes::uint16_t> xl_trough = {"crossline", stats.trough_amplitude_crossline};
+
+    auto peakSlicedDatasetRes = ds.sel(il_peak, xl_peak);
+    if (!peakSlicedDatasetRes.status().ok()) {
+        std::cerr << "Failed to slice dataset: " << peakSlicedDatasetRes.status().message() << std::endl;
+        return 1;
+    }
+
+    auto peakSlicedDataset = peakSlicedDatasetRes.value();
+    // We can print the dataset and see that the inline and crossline indices are not the same as their values.
+    // std::cout << "Peak sliced dataset: " << peakSlicedDataset << std::endl;
+
+    // We can re-use the same function (and variable) to get the cdp coordinates of the peak amplitude now.
+    cdpsFut = GetUTMCoords(peakSlicedDataset);
+    if (!cdpsFut.status().ok()) {
+        std::cerr << "Failed to get UTM coordinates of peak amplitude: " << cdpsFut.status().message() << std::endl;
+        return 1;
+    }
+
+    cdps = cdpsFut.value();
+    cdp_x = cdps.first.value();
+    cdp_y = cdps.second.value();
+    cdp_x_extents = GetExtents<mdio::dtypes::float64_t>(cdp_x);
+    cdp_y_extents = GetExtents<mdio::dtypes::float64_t>(cdp_y);
+
+    std::cout << "========Peak Amplitude=========" << std::endl;
+    utm::print_corners(cdp_x_extents, cdp_y_extents);
+    utm::web_display(cdp_x_extents, cdp_y_extents);
+    std::cout << std::endl;
+    std::cout << "=================================" << std::endl << std::endl;
+
+    auto troughSlicedDatasetRes = ds.sel(il_trough, xl_trough);
+    if (!troughSlicedDatasetRes.status().ok()) {
+        std::cerr << "Failed to slice dataset: " << troughSlicedDatasetRes.status().message() << std::endl;
+        return 1;
+    }
+
+    auto troughSlicedDataset = troughSlicedDatasetRes.value();
+    // The trough sliced dataset should show a different inline/crossline index pair than the peak sliced dataset.
+    // std::cout << "Trough sliced dataset: " << troughSlicedDataset << std::endl;
+
+    cdpsFut = GetUTMCoords(troughSlicedDataset);
+    if (!cdpsFut.status().ok()) {
+        std::cerr << "Failed to get UTM coordinates of trough amplitude: " << cdpsFut.status().message() << std::endl;
+        return 1;
+    }
+
+    cdps = cdpsFut.value();
+    cdp_x = cdps.first.value();
+    cdp_y = cdps.second.value();
+    cdp_x_extents = GetExtents<mdio::dtypes::float64_t>(cdp_x);
+    cdp_y_extents = GetExtents<mdio::dtypes::float64_t>(cdp_y);
+
+    std::cout << "========Trough Amplitude=========" << std::endl;
+    utm::print_corners(cdp_x_extents, cdp_y_extents);
+    std::cout << std::endl;
+    utm::web_display(cdp_x_extents, cdp_y_extents);
+    std::cout << std::endl;
+    std::cout << "=================================" << std::endl << std::endl;
+    return 0;
+}
+
+
+mdio::Future<mdio::Dataset> OpenDataset(const std::string& path, uint64_t cache_size_bytes) {
+    auto cacheJson = nlohmann::json::parse(R"({"cache_pool": {"total_bytes_limit": 1073741824}})");  // 1GiB default cache size.
+    cacheJson["cache_pool"]["total_bytes_limit"] = cache_size_bytes;
+    auto spec = mdio::Context::Spec::FromJson(cacheJson);
+    auto ctx = mdio::Context(spec.value());
+    return mdio::Dataset::Open(path, mdio::constants::kOpen, ctx);
+}
+
+mdio::Result<std::pair<mdio::Future<mdio::VariableData<mdio::dtypes::float64_t>>, mdio::Future<mdio::VariableData<mdio::dtypes::float64_t>>>> GetUTMCoords(mdio::Dataset& ds) {
+    MDIO_ASSIGN_OR_RETURN(auto cdp_x, ds.variables.get<mdio::dtypes::float64_t>("cdp-x"));
+    MDIO_ASSIGN_OR_RETURN(auto cdp_y, ds.variables.get<mdio::dtypes::float64_t>("cdp-y"));
+
+    auto cdp_x_fut = cdp_x.Read();
+    auto cdp_y_fut = cdp_y.Read();
+
+    return std::make_pair(cdp_x_fut, cdp_y_fut);
+}
+
+mdio::Result<std::pair<mdio::Future<mdio::VariableData<mdio::dtypes::uint16_t>>, mdio::Future<mdio::VariableData<mdio::dtypes::uint16_t>>>> GetInlineCrossline(mdio::Dataset& ds) {
+    MDIO_ASSIGN_OR_RETURN(auto il, ds.variables.get<mdio::dtypes::uint16_t>("inline"));
+    MDIO_ASSIGN_OR_RETURN(auto xl, ds.variables.get<mdio::dtypes::uint16_t>("crossline"));
+
+    auto il_fut = il.Read();
+    auto xl_fut = xl.Read();
+
+    return std::make_pair(il_fut, xl_fut);
+}