Update Tensorstore driver to support structured dtype and add support for Zarr v2 and v3 backend

CMakeLists.txt

@@ -75,6 +75,8 @@ include(CMakeDependentOption)
 # provides a method to download dependencies:
 include(FetchContent)
 include(CMakeHelpers/MdioHelpers)
+# optional code coverage support:
+include(CodeCoverage)
 
 list(APPEND mdio_DEFAULT_COPTS
     "-Wno-deprecated-declarations"
@@ -108,6 +110,7 @@ if(NOT CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
   set(mdio_INTERNAL_DEPS
     tensorstore::driver_array
     tensorstore::driver_zarr
+    tensorstore::driver_zarr3
     tensorstore::driver_json
     tensorstore::kvstore_file
     tensorstore::stack

cmake/CMakeHelpers/MdioHelpers.cmake

@@ -297,6 +297,12 @@ function(mdio_cc_test)
     PRIVATE ${mdio_CC_TEST_COPTS}
   )
 
+  # Add coverage flags only to mdio test targets (not dependencies)
+  if(MDIO_ENABLE_COVERAGE)
+    target_compile_options(${_NAME} PRIVATE ${MDIO_COVERAGE_COMPILE_FLAGS})
+    target_link_options(${_NAME} PRIVATE ${MDIO_COVERAGE_LINK_FLAGS})
+  endif()
+
   target_link_libraries(${_NAME}
     PUBLIC ${mdio_CC_TEST_DEPS}
     PRIVATE ${mdio_CC_TEST_LINKOPTS}

cmake/CodeCoverage.cmake

@@ -0,0 +1,166 @@
+# Copyright 2024 TGS
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ==============================================================================
+# Code Coverage Support (gcov + lcov)
+# ==============================================================================
+#
+# Coverage instrumentation is applied ONLY to mdio test targets, not to
+# dependencies like Tensorstore. This keeps build times and test execution fast.
+#
+# REQUIRED CMAKE FLAGS:
+#   -DMDIO_ENABLE_COVERAGE=ON    Enable coverage instrumentation
+#   -DCMAKE_BUILD_TYPE=Debug     Recommended for meaningful line coverage
+#
+# REQUIRED SYSTEM TOOLS:
+#   - gcov   (usually bundled with GCC)
+#   - lcov   (install via: apt install lcov / brew install lcov)
+#   - genhtml (included with lcov)
+#
+# USAGE:
+#   1. Configure and build with coverage enabled:
+#        cd build
+#        cmake .. -DMDIO_ENABLE_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug
+#        make
+#
+#   2. Run tests to generate coverage data (coverage accumulates across runs):
+#        ./mdio/mdio_variable_test                              # single test
+#        ./mdio/mdio_variable_test && ./mdio/mdio_dataset_test  # multiple tests
+#        ctest                                                  # all registered tests
+#
+#   3. Generate HTML coverage report:
+#        make coverage-capture
+#
+#   4. View the report:
+#        Open build/coverage_report/index.html in a browser
+#
+# AVAILABLE TARGETS:
+#   make coverage         - Reset counters, capture data, generate report
+#   make coverage-capture - Capture current data and generate report (no reset)
+#   make coverage-reset   - Zero out all coverage counters
+#
+# ==============================================================================
+
+option(MDIO_ENABLE_COVERAGE "Enable code coverage instrumentation (requires GCC or Clang)" OFF)
+
+if(MDIO_ENABLE_COVERAGE)
+  message(STATUS "Code coverage enabled")
+
+  # Check for supported compiler
+  if(NOT CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
+    message(FATAL_ERROR "Code coverage requires GCC or Clang compiler")
+  endif()
+
+  # Coverage compiler/linker flags - exported for use in MdioHelpers.cmake
+  # These are applied only to mdio targets, not to dependencies like Tensorstore
+  set(MDIO_COVERAGE_COMPILE_FLAGS "-fprofile-arcs;-ftest-coverage" CACHE INTERNAL "Coverage compile flags")
+  set(MDIO_COVERAGE_LINK_FLAGS "--coverage" CACHE INTERNAL "Coverage link flags")
+
+  # Find required tools
+  find_program(LCOV_PATH lcov)
+  find_program(GENHTML_PATH genhtml)
+
+  if(NOT LCOV_PATH)
+    message(WARNING "lcov not found - coverage report generation will not be available")
+  endif()
+
+  if(NOT GENHTML_PATH)
+    message(WARNING "genhtml not found - coverage report generation will not be available")
+  endif()
+
+  # Create coverage report target if tools are available
+  if(LCOV_PATH AND GENHTML_PATH)
+    # Custom target to generate coverage report
+    add_custom_target(coverage
+      COMMENT "Generating code coverage report..."
+
+      # Clear previous coverage data
+      COMMAND ${LCOV_PATH} --directory ${CMAKE_BINARY_DIR} --zerocounters
+
+      # Run all tests (ctest must be run separately or you can uncomment below)
+      # COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure
+
+      # Capture coverage data
+      COMMAND ${LCOV_PATH} 
+        --directory ${CMAKE_BINARY_DIR}
+        --capture
+        --output-file ${CMAKE_BINARY_DIR}/coverage.info
+        --ignore-errors mismatch,negative
+
+      # Remove coverage data for external dependencies
+      COMMAND ${LCOV_PATH}
+        --remove ${CMAKE_BINARY_DIR}/coverage.info
+        '/usr/*'
+        '${CMAKE_BINARY_DIR}/_deps/*'
+        '*/googletest/*'
+        '*/gtest/*'
+        '*/gmock/*'
+        '*_test.cc'
+        --output-file ${CMAKE_BINARY_DIR}/coverage.info
+        --ignore-errors unused,negative
+
+      # Generate HTML report
+      COMMAND ${GENHTML_PATH}
+        ${CMAKE_BINARY_DIR}/coverage.info
+        --output-directory ${CMAKE_BINARY_DIR}/coverage_report
+        --title "MDIO Code Coverage"
+        --legend
+        --show-details
+
+      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+    )
+
+    # Target to just capture coverage (without zeroing first)
+    add_custom_target(coverage-capture
+      COMMENT "Capturing coverage data..."
+
+      COMMAND ${LCOV_PATH} 
+        --directory ${CMAKE_BINARY_DIR}
+        --capture
+        --output-file ${CMAKE_BINARY_DIR}/coverage.info
+        --ignore-errors mismatch,negative
+
+      COMMAND ${LCOV_PATH}
+        --remove ${CMAKE_BINARY_DIR}/coverage.info
+        '/usr/*'
+        '${CMAKE_BINARY_DIR}/_deps/*'
+        '*/googletest/*'
+        '*/gtest/*'
+        '*/gmock/*'
+        '*_test.cc'
+        --output-file ${CMAKE_BINARY_DIR}/coverage.info
+        --ignore-errors unused,negative
+
+      COMMAND ${GENHTML_PATH}
+        ${CMAKE_BINARY_DIR}/coverage.info
+        --output-directory ${CMAKE_BINARY_DIR}/coverage_report
+        --title "MDIO Code Coverage"
+        --legend
+        --show-details
+
+      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+    )
+
+    # Target to reset coverage counters
+    add_custom_target(coverage-reset
+      COMMENT "Resetting coverage counters..."
+      COMMAND ${LCOV_PATH} --directory ${CMAKE_BINARY_DIR} --zerocounters
+      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+    )
+
+    message(STATUS "Coverage targets available: 'coverage', 'coverage-capture', 'coverage-reset'")
+    message(STATUS "Coverage report will be generated at: ${CMAKE_BINARY_DIR}/coverage_report/index.html")
+  endif()
+endif()
+

cmake/FindEXT_TENSORSTORE.cmake

@@ -3,11 +3,18 @@ IF ( NOT TARGET tensorstore )
 
  include(FetchContent)
 
- FetchContent_Declare(
-  tensorstore
-  GIT_REPOSITORY
-  https://github.com/brian-michell/tensorstore.git
-  GIT_TAG v0.1.63_latest
+#  FetchContent_Declare(
+#   tensorstore
+#   GIT_REPOSITORY
+#   https://github.com/brian-michell/tensorstore.git
+#   GIT_TAG v0.1.63_latest
+# )
+
+FetchContent_Declare(
+ tensorstore
+ GIT_REPOSITORY
+ https://github.com/BrianMichell/tensorstore.git
+ GIT_TAG v3_structs
 )
 
 FetchContent_MakeAvailable(tensorstore)

mdio/CMakeLists.txt

@@ -101,6 +101,7 @@ mdio_cc_test(
     GTest::gmock_main
     tensorstore::driver_array
     tensorstore::driver_zarr
+    tensorstore::driver_zarr3
     tensorstore::driver_json
     tensorstore::kvstore_file
     tensorstore::stack
@@ -125,6 +126,7 @@ mdio_cc_test(
     GTest::gmock_main
     tensorstore::driver_array
     tensorstore::driver_zarr
+    tensorstore::driver_zarr3
     tensorstore::driver_json
     tensorstore::kvstore_file
     tensorstore::stack
@@ -134,6 +136,8 @@ mdio_cc_test(
     tensorstore::util_status_testutil
     nlohmann_json_schema_validator
     tensorstore::kvstore_s3
+    absl::log_initialize
+    absl::log_globals
 )
 
 mdio_cc_test(
@@ -186,6 +190,7 @@ mdio_cc_test(
   GTest::gmock_main
     tensorstore::driver_array
     tensorstore::driver_zarr
+    tensorstore::driver_zarr3
     tensorstore::driver_json
     tensorstore::kvstore_file
     tensorstore::stack
@@ -209,6 +214,7 @@ mdio_cc_test(
     GTest::gmock_main
     tensorstore::driver_array
     tensorstore::driver_zarr
+    tensorstore::driver_zarr3
     tensorstore::driver_json
     tensorstore::kvstore_file
     tensorstore::stack
@@ -232,6 +238,7 @@ mdio_cc_test(
     GTest::gmock_main
     tensorstore::driver_array
     tensorstore::driver_zarr
+    tensorstore::driver_zarr3
     tensorstore::driver_json
     tensorstore::kvstore_file
     tensorstore::tensorstore
@@ -255,6 +262,7 @@ mdio_cc_test(
     GTest::gmock_main
     tensorstore::driver_array
     tensorstore::driver_zarr
+    tensorstore::driver_zarr3
     tensorstore::driver_json
     tensorstore::kvstore_file
     tensorstore::stack
@@ -277,6 +285,7 @@ mdio_cc_test(
     GTest::gmock_main
     tensorstore::driver_array
     tensorstore::driver_zarr
+    tensorstore::driver_zarr3
     tensorstore::driver_json
     tensorstore::kvstore_file
     tensorstore::stack
@@ -301,6 +310,7 @@ mdio_cc_test(
     GTest::gmock_main
     tensorstore::driver_array
     tensorstore::driver_zarr
+    tensorstore::driver_zarr3
     tensorstore::driver_json
     tensorstore::kvstore_file
     tensorstore::stack
@@ -321,14 +331,31 @@ mdio_cc_test(
     ${mdio_DEFAULT_COPTS} 
   LINKOPTS
     ${mdio_DEFAULT_LINKOPTS}
+  DEPS
+    GTest::gmock_main
+    tensorstore::driver_json
+    tensorstore::tensorstore
+    nlohmann_json_schema_validator
+)
+
+mdio_cc_test(
+  NAME
+    coordinate_selector_test
+  SRCS
+    coordinate_selector_test.cc
+  COPTS
+    ${mdio_DEFAULT_COPTS} 
+  LINKOPTS
+    ${mdio_DEFAULT_LINKOPTS}
   DEPS
     GTest::gmock_main
     tensorstore::driver_array
     tensorstore::driver_zarr
+    tensorstore::driver_zarr3
     tensorstore::driver_json
     tensorstore::kvstore_file
-    tensorstore::stack
     tensorstore::tensorstore
+    tensorstore::stack
     tensorstore::index_space_dim_expression
     tensorstore::index_space_index_transform
     tensorstore::util_status_testutil
@@ -337,9 +364,9 @@ mdio_cc_test(
 
 mdio_cc_test(
   NAME
-    coordinate_selector_test
+    zarr_test
   SRCS
-    coordinate_selector_test.cc
+    zarr/zarr_test.cc
   COPTS
     ${mdio_DEFAULT_COPTS} 
   LINKOPTS
@@ -348,6 +375,7 @@ mdio_cc_test(
     GTest::gmock_main
     tensorstore::driver_array
     tensorstore::driver_zarr
+    tensorstore::driver_zarr3
     tensorstore::driver_json
     tensorstore::kvstore_file
     tensorstore::tensorstore
@@ -357,3 +385,4 @@ mdio_cc_test(
     tensorstore::util_status_testutil
     nlohmann_json_schema_validator
 )
+