Support variable bin sizes

.github/workflows/codecov.yml

@@ -212,8 +212,7 @@ jobs:
           test/scripts/hictk_load_coo.sh build/src/hictk/hictk unsorted
           test/scripts/hictk_load_bg2.sh build/src/hictk/hictk sorted
           test/scripts/hictk_load_bg2.sh build/src/hictk/hictk unsorted
-          test/scripts/hictk_load_4dn.sh build/src/hictk/hictk sorted
-          test/scripts/hictk_load_4dn.sh build/src/hictk/hictk unsorted
+          test/scripts/hictk_load_4dn.sh build/src/hictk/hictk
 
           test/scripts/hictk_merge.sh build/src/hictk/hictk
 

.github/workflows/macos-ci.yml

@@ -430,13 +430,9 @@ jobs:
         run: |
           test/scripts/hictk_load_bg2.sh bin/hictk unsorted
 
-      - name: Test hictk load 4dn sorted
+      - name: Test hictk load 4dn
         run: |
-          test/scripts/hictk_load_4dn.sh bin/hictk sorted
-
-      - name: Test hictk load 4dn unsorted
-        run: |
-          test/scripts/hictk_load_4dn.sh bin/hictk unsorted
+          test/scripts/hictk_load_4dn.sh bin/hictk
 
       - name: Test hictk merge
         run: |

.github/workflows/ubuntu-ci.yml

@@ -483,13 +483,9 @@ jobs:
         run: |
           test/scripts/hictk_load_bg2.sh bin/hictk unsorted
 
-      - name: Test hictk load 4dn sorted
+      - name: Test hictk load 4dn
         run: |
-          test/scripts/hictk_load_4dn.sh bin/hictk sorted
-
-      - name: Test hictk load 4dn unsorted
-        run: |
-          test/scripts/hictk_load_4dn.sh bin/hictk unsorted
+          test/scripts/hictk_load_4dn.sh bin/hictk
 
       - name: Test hictk merge
         run: |

cmake/FetchTestDataset.cmake

@@ -4,8 +4,8 @@
 
 # cmake-format: off
 file(
-  DOWNLOAD https://zenodo.org/record/8392109/files/hictk_test_data.tar.xz?download=1
-  EXPECTED_HASH SHA256=43d05077082603cf03dc5ce2bfc0f81b2422c7249ee987e09512d2e3a56afff1
+  DOWNLOAD https://zenodo.org/records/10289491/files/hictk_test_data.tar.xz?download=1
+  EXPECTED_HASH SHA256=5e69dceb8789d923a38aed7add8fc18abfdfe531aea6effcdb7efe3c9bcf5246
   "${PROJECT_SOURCE_DIR}/test/data/hictk_test_data.tar.xz")
 # cmake-format: on
 

src/hictk/cli/cli_load.cpp

@@ -35,19 +35,25 @@
       ->check(CLI::ExistingFile)
       ->required();
 
-  sc.add_option(
-      "bin-size",
-      c.bin_size,
-      "Bin size (bp).")
-      ->check(CLI::PositiveNumber)
-      ->required();
-
   sc.add_option(
       "output-uri",
       c.uri,
       "Path to output Cooler (URI syntax supported).")
       ->required();
 
+  sc.add_option(
+      "-b,--bin-size",
+      c.bin_size,
+      "Bin size (bp).\n"
+      "Required when --bin-table is not used.")
+      ->check(CLI::PositiveNumber);
+
+  sc.add_option(
+      "-t,--bin-table",
+      c.path_to_bin_table,
+      "Path to a BED3+ file with the bin table.")
+      ->check(CLI::ExistingFile);
+
   sc.add_option(
       "-f,--format",
       c.format,
@@ -67,6 +73,13 @@
       "Assembly name.")
       ->capture_default_str();
 
+  sc.add_flag(
+      "--one-based,!--zero-based",
+      c.one_based,
+      "Interpret genomic coordinates or bins as one/zero based.\n"
+      "By default coordinates are assumed to be one-based for interactions in\n"
+      "4dn and validapairs formats and zero-based otherwise.");
+
   sc.add_flag(
       "--count-as-float",
       c.count_as_float,
@@ -89,24 +102,48 @@
   sc.add_option(
       "--batch-size",
       c.batch_size,
-      "Number of pixels to buffer in memory. Only used when processing unsorted interactions or pairs")
+      "Number of pixels to buffer in memory.\n"
+      "Only used when processing unsorted interactions or pairs.")
       ->capture_default_str();
   // clang-format on
 
+  sc.get_option("--bin-size")->excludes(sc.get_option("--bin-table"));
   _config = std::monostate{};
 }
 
 void Cli::validate_load_subcommand() const {
   assert(_cli.get_subcommand("load")->parsed());
 
+  std::vector<std::string> warnings;
   std::vector<std::string> errors;
   const auto& c = std::get<LoadConfig>(_config);
+  const auto& sc = *_cli.get_subcommand("load");
 
   if (!c.force && std::filesystem::exists(c.uri)) {
     errors.emplace_back(fmt::format(
         FMT_STRING("Refusing to overwrite file {}. Pass --force to overwrite."), c.uri));
   }
 
+  if (c.path_to_bin_table.empty() && c.path_to_chrom_sizes.empty()) {
+    assert(c.bin_size == 0);
+    errors.emplace_back("--bin-size is required when --bin-table is not specified.");
+  }
+
+  if ((c.format == "bg2" || c.format == "coo") && !sc.get_option("--bin-table")->empty()) {
+    errors.emplace_back(
+        "specifying bins through the --bin-table is not supported when ingesting pre-binned "
+        "interactions.");
+  }
+
+  if (c.format == "4dn" && c.format == "validpairs" && c.assume_sorted) {
+    warnings.emplace_back(
+        "--assume-sorted has no effect when ingesting interactions in 4dn or validpairs format.");
+  }
+
+  for (const auto& w : warnings) {
+    SPDLOG_WARN(FMT_STRING("{}"), w);
+  }
+
   if (!errors.empty()) {
     throw std::runtime_error(
         fmt::format(FMT_STRING("the following error(s) where encountered while validating CLI "
@@ -117,6 +154,15 @@
 
 void Cli::transform_args_load_subcommand() {
   auto& c = std::get<LoadConfig>(_config);
+  const auto& sc = *_cli.get_subcommand("load");
+
+  if (sc.get_option("--one-based")->empty()) {
+    if (c.format == "4dn" || c.format == "validpairs") {
+      c.offset = -1;
+    }
+  } else {
+    c.offset = c.one_based ? -1 : 0;
+  }
 
   // in spdlog, high numbers correspond to low log levels
   assert(c.verbosity > 0 && c.verbosity < 5);

src/hictk/cli/cli_zoomify.cpp

@@ -145,6 +145,12 @@
         "--resolutions.");
   }
 
+  if (clr.bin_size() == 0) {  // Variable bin size
+    errors.clear();
+    warnings.clear();
+    errors.emplace_back("zoomifying files with variable bin size is not currently supported.");
+  }
+
   for (const auto& w : warnings) {
     SPDLOG_WARN(FMT_STRING("{}"), w);
   }

src/hictk/convert/cool_to_hic.cpp

@@ -314,6 +314,12 @@
                                          c.path_to_input.string(), c.resolutions.front());
 
       const cooler::File clr(uri);
+
+      if (clr.bin_size() == 0) {
+        throw std::runtime_error(
+            "converting cooler files with variable bin size is not supported.");
+      }
+
       dump_chrom_sizes(clr, chrom_sizes);
       dump_pixels(clr, pixels, c.gzip_compression_lvl, c.threads);
     }

src/hictk/include/hictk/tools/config.hpp

@@ -103,9 +103,13 @@ struct LoadConfig {
   std::string uri{};
 
   std::filesystem::path path_to_chrom_sizes{};
+  std::filesystem::path path_to_bin_table{};
   std::uint32_t bin_size{};
+
   std::string format{};
   std::string assembly{"unknown"};
+  bool one_based{true};
+  std::int64_t offset{0};
   bool count_as_float{false};
   bool assume_sorted{false};
   bool force{false};

src/hictk/load/common.hpp

@@ -31,20 +31,20 @@
 
 template <typename N>
 [[nodiscard]] inline ThinPixel<N> parse_pixel(const BinTable& bins, std::string_view line,
-                                              Format format) {
+                                              Format format, std::int64_t offset) {
   ThinPixel<N> pixel{};
   switch (format) {
     case Format::COO:
-      pixel = ThinPixel<N>::from_coo(bins, line);
+      pixel = ThinPixel<N>::from_coo(bins, line, offset);
       break;
     case Format::BG2:
-      pixel = Pixel<N>::from_bg2(bins, line).to_thin();
+      pixel = Pixel<N>::from_bg2(bins, line, offset).to_thin();
       break;
     case Format::VP:
-      pixel = Pixel<N>::from_validpair(bins, line).to_thin();
+      pixel = Pixel<N>::from_validpair(bins, line, offset).to_thin();
       break;
     case Format::_4DN:
-      pixel = Pixel<N>::from_4dn_pairs(bins, line).to_thin();
+      pixel = Pixel<N>::from_4dn_pairs(bins, line, offset).to_thin();
       break;
   }
   if (pixel.bin1_id > pixel.bin2_id) {