Merge branch 'main' into slice_stride

Author: BrianMichell

examples/real_data_example/CMakeLists.txt

@@ -32,6 +32,6 @@ target_link_libraries(real_data_example PRIVATE
   mdio
   ${mdio_INTERNAL_DEPS}
   ${mdio_INTERNAL_S3_DRIVER_DEPS}
-  PNG::PNG
+  ${mdio_INTERNAL_GCS_DRIVER_DEPS}
   indicators
 )

examples/real_data_example/README.md

@@ -32,7 +32,7 @@ This tool allows users to extract specific slices from 3D seismic data stored in
 - `--xline_range`: Crossline range in format {crossline,start,end,step} (default: "{crossline,500,700,1}")
 - `--depth_range`: Optional depth range in format {depth,start,end,step}
 - `--variable_name`: Name of the seismic variable to extract (default: "seismic")
-- `--print_dataset`: Print the dataset URL and return without processing
+- `--print_dataset`: Print the dataset metadata and return without processing
 
 ### Example
 

examples/real_data_example/src/real_data_example.cc

@@ -24,7 +24,6 @@
 #include "interpolation.h"
 #include "progress.h"
 #include "seismic_numpy.h"
-#include "seismic_png.h"
 #include "tensorstore/tensorstore.h"
 
 #define MDIO_RETURN_IF_ERROR(...) TENSORSTORE_RETURN_IF_ERROR(__VA_ARGS__)
@@ -39,7 +38,7 @@ ABSL_FLAG(std::string, depth_range, "",
           "Optional depth range in format {depth,start,end,step}");
 ABSL_FLAG(std::string, variable_name, "seismic",
           "Name of the seismic variable");
-ABSL_FLAG(bool, print_dataset, false, "Print the dataset URL and return");
+ABSL_FLAG(bool, print_dataset, false, "Print the dataset metadata and return");
 ABSL_FLAG(std::string, dataset_path,
           "s3://tgs-opendata-poseidon/full_stack_agc.mdio",
           "The path to the dataset");
@@ -126,5 +125,11 @@ int main(int argc, char* argv[]) {
   auto desc2 = ParseRange(crossline_range);
   auto desc3 = ParseRange(depth_range);
 
-  return Run<float>(desc1, desc2, desc3).ok() ? 0 : 1;
-}
+  auto runResult = Run<float>(desc1, desc2, desc3);
+  if (!runResult.ok()) {
+    std::cerr << "Error: " << runResult.ToString() << std::endl;
+    return 1;
+  }
+
+  return 0;
+}

examples/seismic_reader/CMakeLists.txt

@@ -0,0 +1,97 @@
+cmake_minimum_required(VERSION 3.24)
+project(SeismicReader VERSION 1.0.0 LANGUAGES CXX)
+
+# Set the C++ standard to C++17
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+# Specify MDIO installation directory (set by bootstrap.sh)
+set(MDIO_INSTALL_DIR "${CMAKE_SOURCE_DIR}/inst")
+
+# Add library directories
+link_directories(
+    ${MDIO_INSTALL_DIR}/lib
+    ${MDIO_INSTALL_DIR}/lib/drivers
+)
+
+# CURL is built and installed by the MDIO installer; no need for separate find_package.
+
+# Define MDIO linker flags - corrected format to match SCons configuration
+set(MDIO_LINK_FLAGS
+    "-Wl,-rpath,${MDIO_INSTALL_DIR}/lib,-rpath,${MDIO_INSTALL_DIR}/lib/drivers,--whole-archive,-L${MDIO_INSTALL_DIR}/lib,-L${MDIO_INSTALL_DIR}/lib/drivers,\
+-lnlohmann_json_schema_validator,\
+-ltensorstore_driver_zarr_bzip2_compressor,\
+-ltensorstore_driver_zarr_driver,\
+-ltensorstore_driver_zarr_spec,\
+-ltensorstore_driver_zarr_zlib_compressor,\
+-ltensorstore_driver_zarr_zstd_compressor,\
+-ltensorstore_driver_zarr_blosc_compressor,\
+-ltensorstore_kvstore_gcs_http,\
+-ltensorstore_kvstore_gcs_gcs_resource,\
+-ltensorstore_kvstore_gcs_validate,\
+-ltensorstore_kvstore_gcs_http_gcs_resource,\
+-ltensorstore_kvstore_s3,\
+-ltensorstore_kvstore_s3_aws_credentials_resource,\
+-ltensorstore_kvstore_s3_credentials_default_credential_provider,\
+-ltensorstore_kvstore_s3_credentials_environment_credential_provider,\
+-ltensorstore_kvstore_s3_credentials_file_credential_provider,\
+-ltensorstore_kvstore_s3_credentials_ec2_credential_provider,\
+-ltensorstore_kvstore_s3_s3_metadata,\
+-ltensorstore_kvstore_s3_s3_resource,\
+-ltensorstore_driver_json,\
+-ltensorstore_internal_cache_cache_pool_resource,\
+-ltensorstore_internal_data_copy_concurrency_resource,\
+-ltensorstore_kvstore_file,\
+-ltensorstore_internal_file_io_concurrency_resource,\
+-ltensorstore_internal_cache_kvs_backed_chunk_cache,\
+-labsl,\
+-lblosc,\
+-ltensorstore,\
+-lre2,\
+-lriegeli,\
+-ltinyxml2_tinyxml2,\
+-lcurl,\
+-lopenssl,\
+--no-whole-archive,\
+-lpthread,\
+-lm"
+)
+
+# Debug: Print out the MDIO_LINK_FLAGS
+message(STATUS "MDIO_LINK_FLAGS: ${MDIO_LINK_FLAGS}")
+
+# Create the executable target. (Assumes main.cpp exists in your project.)
+add_executable(read main.cc)
+
+# Append the linker flags to the target's link flags.
+set_target_properties(read PROPERTIES LINK_FLAGS "${MDIO_LINK_FLAGS}")
+
+# Add compile definitions
+target_compile_definitions(read PRIVATE HAVE_MDIO MAX_NUM_SLICES=32)
+
+# Add MDIO and third-party include directories for target 'read'
+# Collect all immediate subdirectories from the MDIO include directory.
+file(GLOB CHILD_DIRS LIST_DIRECTORIES true "${MDIO_INSTALL_DIR}/include/*")
+
+# Also include the top-level include directory so that headers like "mdio/mdio.h" are found.
+list(INSERT CHILD_DIRS 0 "${MDIO_INSTALL_DIR}/include")
+
+# Remove any unwanted directories (for example, the gtest-src directory)
+foreach(dir ${CHILD_DIRS})
+    get_filename_component(basename ${dir} NAME)
+    if(basename MATCHES "gtest-src")
+        list(REMOVE_ITEM CHILD_DIRS "${dir}")
+    endif()
+endforeach()
+
+# Append additional directories that the installer uses but might not be one-level deep.
+list(APPEND CHILD_DIRS 
+    "${MDIO_INSTALL_DIR}/include/nlohmann_json-src/include"
+    "${MDIO_INSTALL_DIR}/include/half-src/include"
+)
+
+target_include_directories(read PRIVATE ${CHILD_DIRS})
+
+# Debug: Print out the include directories for target 'read'
+get_target_property(READ_INCLUDES read INCLUDE_DIRECTORIES)
+message(STATUS "Target 'read' include directories: ${READ_INCLUDES}")

examples/seismic_reader/README.md

@@ -0,0 +1,64 @@
+# Seismic data reader
+
+The purpose of this example is to demonstrate one method of integrating the **MDIO** C++ library into a seismic data reader into an existing project.
+
+Due to the variety of target build systems and differing complexities of integrating the library with existing CMake projects, this example will also expose a way to build MDIO and its dependencies as a set of shared and linkable libraries and integrate it that way.
+
+## Table of contents
+- [Concepts demonstrated](#concepts-demonstrated)
+- [Overview](#overview)
+- [Running](#running)
+- [Glossary](#glossary)
+
+## Concepts demonstrated
+- Index-based slicing
+- Value-based slicing
+- Dimension coordinates
+- Coordinates
+- Caching
+
+Chunk-aligned opreations were briefly discussed but not examined in great detail for this example.
+
+## Overview
+
+This example assumes that you are able to build the main branch of **MDIO**.
+
+This example will perform the following steps:
+
+1. Clone the installer to this directory.
+2. Run the installer, building and installing several archives and shared files.
+3. Link the installed files in the `CMakeLists.txt`.
+4. Demonstrate opening a Dataset with a configurable cache.
+5. Demonstrate acquiring the corner points for the UTM grid on the Open Poseidon dataset from S3.
+    - Calculates the latitude and longitude coordinates for the corner points.
+    - Provides a web link to display the surface area on a map.
+6. Demonstrate acquiring the inline and crossline extents.
+7. Demonstrate an index-based slice for chunk-aligned, tracewise processing.
+    - Calculates basic statistics for a small section of the dataset.
+    - Tracks the highest (peak) and lowest (trough) amplitudes and their actual inline/crossline coordinate pairs.
+8. Demonstrate a value-based slice for more targeted analysis.
+    - Corolates the cdp-x and cdp-y coordinate pair for the peak and trough values.
+    - Provides a web link to display each one's location on a map.
+
+## Running
+
+```bash
+$ ./bootstrap.sh
+$ cd build
+$ cmake ..
+$ make -j
+```
+
+The bootstrap shell script will build and install the mdio-cpp library as a set of shared and static objects.
+
+It will then set up the build directory if it completes without error.
+
+After the program has finished building it can be run with `./read`.
+
+## Glossary
+- Dimension coordinate: A 1-dimensional Variable that describes a dimension of the dataset. 
+- Coordinate: A Variable that helps describe data outside of its natural (logical) domain. May be greater than 1-dimensional.
+- Slicing: The act of subsetting a dataset along one or more dimension coordinates. A subset of a dataset is still considered a dataset.
+- Index-based slicing (*isel*): Subsetting a dataset based on the logical indices of its *dimension coordinate*(s).
+- Value-based slicing (*sel*): Subsetting a dataset based on the values contained by its *dimension coordinate*(s).
+- Chunk-aligned: Slicing the data along its logical chunk boundries for efficient I/O performance.

examples/seismic_reader/bootstrap.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# bootstrap.sh
+#
+# This script bootstraps the MDIO installation.
+# It clones the mdio-cpp-installer repository and installs MDIO
+# into the current directory (in the "inst" folder).
+#
+# Instruction from the installer:
+#   $ ./install.sh [install_directory] [mdio_tag]
+# (the mdio_tag is ignored)
+# The hidden --curl flag is also used.
+
+# Define installation directory as the "inst" folder in the current directory
+INST_DIR="$(pwd)/inst"
+
+echo "Installing MDIO in: $INST_DIR"
+
+# Clone the installer repo if it does not already exist
+if [ ! -d "mdio-cpp-installer" ]; then
+    echo "Cloning mdio-cpp-installer repository..."
+    git clone https://github.com/BrianMichell/mdio-cpp-installer.git || { 
+        echo "Failed to clone repository."; exit 1; 
+    }
+fi
+
+# Change into the installer directory
+cd mdio-cpp-installer || { 
+    echo "Failed to change directory into mdio-cpp-installer."; exit 1; 
+}
+
+# Make sure the installer script is executable
+chmod +x install.sh
+
+# Run the installer: Pass the installation directory, a dummy tag,
+# and the hidden "--curl" flag.
+echo "Running MDIO installer..."
+./install.sh "$INST_DIR" dummy_tag --curl || { 
+    echo "MDIO installation failed."; exit 1; 
+}
+
+echo "MDIO installation completed successfully."
+
+# Return to the original directory
+cd ..
+
+mkdir build