[ntuple][rfile] Add support for RFile in RNTupleWriter

io/io/inc/ROOT/RFile.hxx

@@ -36,6 +36,8 @@ ROOT::RLogChannel &RFileLog();
 /// This method is meant to only be used by the pythonization.
 [[nodiscard]] void *RFile_GetObjectFromKey(RFile &file, const RKeyInfo &key);
 
+TFile *GetRFileTFile(RFile &rfile);
+
 } // namespace Internal
 
 namespace Detail {
@@ -224,6 +226,7 @@ auto myObj = file->Get<TH1D>("h");
 */
 class RFile final {
    friend void *Internal::RFile_GetObjectFromKey(RFile &file, const RKeyInfo &key);
+   friend TFile *Internal::GetRFileTFile(RFile &rfile);
 
    /// Flags used in PutInternal()
    enum PutFlags {

io/io/src/RFile.cxx

@@ -551,3 +551,8 @@ void *ROOT::Experimental::Internal::RFile_GetObjectFromKey(RFile &file, const RK
    void *obj = file.GetUntyped(key.GetPath(), key.GetClassName().c_str());
    return obj;
 }
+
+TFile *ROOT::Experimental::Internal::GetRFileTFile(RFile &file)
+{
+   return file.fFile.get();
+}

io/io/test/CMakeLists.txt

@@ -18,7 +18,7 @@ if(uring AND NOT DEFINED ENV{ROOTTEST_IGNORE_URING})
   ROOT_ADD_GTEST(RIoUring RIoUring.cxx LIBRARIES RIO)
 endif()
 
-ROOT_ADD_GTEST(rfile rfile.cxx LIBRARIES RIO Hist)
+ROOT_ADD_GTEST(rfile rfile.cxx LIBRARIES RIO Hist ROOTNTuple)
 if(pyroot)
   ROOT_ADD_PYUNITTEST(rfile_py rfile.py)
 endif()

io/io/test/rfile.cxx

@@ -11,6 +11,9 @@
 #include <ROOT/RFile.hxx>
 #include <ROOT/TestSupport.hxx>
 #include <ROOT/RLogger.hxx>
+#include <ROOT/RNTuple.hxx>
+#include <ROOT/RNTupleReader.hxx>
+#include <ROOT/RNTupleWriter.hxx>
 
 using ROOT::Experimental::RFile;
 using ROOT::TestSupport::FileRaii;
@@ -685,7 +688,7 @@ TEST(RFile, GetKeyInfo)
 
    EXPECT_EQ(file->GetKeyInfo("foo"), std::nullopt);
 
-   for (const std::string_view path : { "/s", "a/b/c", "b", "/a/d" }) {
+   for (const std::string_view path : {"/s", "a/b/c", "b", "/a/d"}) {
       auto key = file->GetKeyInfo(path);
       ASSERT_NE(key, std::nullopt);
       EXPECT_EQ(key->GetPath(), path[0] == '/' ? path.substr(1) : path);
@@ -694,3 +697,27 @@ TEST(RFile, GetKeyInfo)
       EXPECT_EQ(key->GetCycle(), 1);
    }
 }
+
+TEST(RFile, RNTuple)
+{
+   FileRaii fileGuard("test_rfile_rntuple.root");
+
+   // Writing
+   {
+      auto file = RFile::Recreate(fileGuard.GetPath());
+
+      auto model = ROOT::RNTupleModel::Create();
+      *model->MakeField<float>("x") = 42;
+
+      auto writer = ROOT::Experimental::RNTupleWriter_Append(std::move(model), "data", *file);
+      writer->Fill();
+   }
+
+   // Reading back
+   auto file = RFile::Open(fileGuard.GetPath());
+   auto ntuple = file->Get<ROOT::RNTuple>("data");
+   ASSERT_NE(ntuple, nullptr);
+   auto reader = ROOT::RNTupleReader::Open(*ntuple);
+   EXPECT_EQ(reader->GetNEntries(), 1);
+   EXPECT_FLOAT_EQ(reader->GetView<float>("x")(0), 42);
+}

tree/ntuple/inc/ROOT/RMiniFile.hxx

@@ -34,6 +34,12 @@ namespace ROOT {
 
 class RNTupleWriteOptions;
 
+namespace Experimental {
+
+class RFile;
+
+}
+
 namespace Internal {
 
 class RRawFile;
@@ -128,6 +134,18 @@ private:
       operator bool() const { return fDirectory; }
    };
 
+   struct RFileRFile {
+      ROOT::Experimental::RFile *fFile = nullptr;
+      std::string fDir;
+      /// Low-level writing using a TFile
+      void Write(const void *buffer, size_t nbytes, std::int64_t offset);
+      /// Reserves an RBlob opaque key as data record and returns the offset of the record. If keyBuffer is specified,
+      /// it must be written *before* the returned offset. (Note that the array type is purely documentation, the
+      /// argument is actually just a pointer.)
+      std::uint64_t ReserveBlobKey(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen] = nullptr);
+      operator bool() const { return fFile; }
+   };
+
    struct RFileSimple {
       /// Direct I/O requires that all buffers and write lengths are aligned. It seems 512 byte alignment is the minimum
       /// for Direct I/O to work, but further testing showed that it results in worse performance than 4kB.
@@ -179,9 +197,16 @@ private:
       operator bool() const { return fFile; }
    };
 
+   template <typename T>
+   static std::uint64_t
+   ReserveBlobKey(T &caller, TFile &file, std::size_t nbytes, std::size_t len, unsigned char keyBuffer[kBlobKeyLen]);
+
    /// RFileSimple: for simple use cases, survives without libRIO dependency
    /// RFileProper: for updating existing files and for storing more than just an RNTuple in the file
-   std::variant<RFileSimple, RFileProper> fFile;
+   /// RFileRFile: like RFileProper but using RFile instead of TFile.
+   using FileType_t = std::variant<RFileSimple, RFileProper, RFileRFile>;
+   FileType_t fFile;
+
    /// A simple file can either be written as TFile container or as NTuple bare file
    bool fIsBare = false;
    /// The identifier of the RNTuple; A single writer object can only write a single RNTuple but multiple
@@ -227,6 +252,9 @@ public:
    static std::unique_ptr<RNTupleFileWriter>
    Append(std::string_view ntupleName, TDirectory &fileOrDirectory, std::uint64_t maxKeySize);
 
+   static std::unique_ptr<RNTupleFileWriter> Append(std::string_view ntupleName, ROOT::Experimental::RFile &file,
+                                                    std::string_view dirPath, std::uint64_t maxKeySize);
+
    RNTupleFileWriter(const RNTupleFileWriter &other) = delete;
    RNTupleFileWriter(RNTupleFileWriter &&other) = delete;
    RNTupleFileWriter &operator=(const RNTupleFileWriter &other) = delete;

tree/ntuple/inc/ROOT/RNTupleWriter.hxx

@@ -37,6 +37,22 @@ namespace ROOT {
 
 class RNTupleWriteOptions;
 
+namespace Experimental {
+class RFile;
+
+/// Creates an RNTupleWriter that writes into the given `file`, appending to it. The RNTuple is written under the
+/// path `ntuplePath`.
+/// `ntuplePath` may have the form `"path/to/ntuple"`, in which case the ntuple's name will be `"ntuple"` and it will
+/// be stored under the given `ntuplePath` in the RFile.
+/// Throws an exception if the model is null.
+/// NOTE: this is a temporary, experimental API that will be replaced by an overload of RNTupleWriter::Append in the
+/// future.
+std::unique_ptr<RNTupleWriter>
+RNTupleWriter_Append(std::unique_ptr<ROOT::RNTupleModel> model, std::string_view ntuplePath,
+                     ROOT::Experimental::RFile &file,
+                     const ROOT::RNTupleWriteOptions &options = ROOT::RNTupleWriteOptions());
+} // namespace Experimental
+
 namespace Internal {
 // Non-public factory method for an RNTuple writer that uses an already constructed page sink
 std::unique_ptr<RNTupleWriter>
@@ -102,6 +118,9 @@ class RNTupleWriter {
    friend ROOT::RNTupleModel::RUpdater;
    friend std::unique_ptr<RNTupleWriter>
       Internal::CreateRNTupleWriter(std::unique_ptr<ROOT::RNTupleModel>, std::unique_ptr<Internal::RPageSink>);
+   friend std::unique_ptr<RNTupleWriter>
+   Experimental::RNTupleWriter_Append(std::unique_ptr<ROOT::RNTupleModel> model, std::string_view ntuplePath,
+                                      ROOT::Experimental::RFile &file, const ROOT::RNTupleWriteOptions &options);
 
 private:
    RNTupleFillContext fFillContext;
@@ -152,6 +171,7 @@ public:
    static std::unique_ptr<RNTupleWriter> Append(std::unique_ptr<ROOT::RNTupleModel> model, std::string_view ntupleName,
                                                 TDirectory &fileOrDirectory,
                                                 const ROOT::RNTupleWriteOptions &options = ROOT::RNTupleWriteOptions());
+
    RNTupleWriter(const RNTupleWriter &) = delete;
    RNTupleWriter &operator=(const RNTupleWriter &) = delete;
    RNTupleWriter(RNTupleWriter &&) = delete;

tree/ntuple/inc/ROOT/RPageStorageFile.hxx

@@ -35,6 +35,10 @@ namespace ROOT {
 class RNTuple; // for making RPageSourceFile a friend of RNTuple
 class RNTupleLocator;
 
+namespace Experimental {
+class RFile;
+}
+
 namespace Internal {
 class RRawFile;
 class RPageAllocatorHeap;
@@ -96,6 +100,8 @@ protected:
 public:
    RPageSinkFile(std::string_view ntupleName, std::string_view path, const ROOT::RNTupleWriteOptions &options);
    RPageSinkFile(std::string_view ntupleName, TDirectory &fileOrDirectory, const ROOT::RNTupleWriteOptions &options);
+   RPageSinkFile(std::string_view ntupleName, ROOT::Experimental::RFile &file, std::string_view ntupleDir,
+                 const ROOT::RNTupleWriteOptions &options);
    RPageSinkFile(const RPageSinkFile &) = delete;
    RPageSinkFile &operator=(const RPageSinkFile &) = delete;
    RPageSinkFile(RPageSinkFile &&) = default;

tree/ntuple/src/RMiniFile.cxx

@@ -20,6 +20,7 @@
 #include <ROOT/RNTupleZip.hxx>
 #include <ROOT/RNTupleSerialize.hxx>
 #include <ROOT/RNTupleWriteOptions.hxx>
+#include <ROOT/RFile.hxx>
 
 #include <Byteswap.h>
 #include <TBufferFile.h>
@@ -1155,19 +1156,12 @@ std::uint64_t ROOT::Internal::RNTupleFileWriter::RFileSimple::ReserveBlobKey(std
 
 ////////////////////////////////////////////////////////////////////////////////
 
-void ROOT::Internal::RNTupleFileWriter::RFileProper::Write(const void *buffer, size_t nbytes, std::int64_t offset)
-{
-   fDirectory->GetFile()->Seek(offset);
-   bool rv = fDirectory->GetFile()->WriteBuffer((char *)(buffer), nbytes);
-   if (rv)
-      throw RException(R__FAIL("WriteBuffer failed."));
-}
-
-std::uint64_t ROOT::Internal::RNTupleFileWriter::RFileProper::ReserveBlobKey(size_t nbytes, size_t len,
-                                                                             unsigned char keyBuffer[kBlobKeyLen])
+template <typename T>
+std::uint64_t ROOT::Internal::RNTupleFileWriter::ReserveBlobKey(T &caller, TFile &file, std::size_t nbytes,
+                                                                std::size_t len, unsigned char keyBuffer[kBlobKeyLen])
 {
    std::uint64_t offsetKey;
-   RKeyBlob keyBlob(fDirectory->GetFile());
+   ROOT::Internal::RKeyBlob keyBlob(&file);
    // Since it is unknown beforehand if offsetKey is beyond the 2GB limit or not,
    // RKeyBlob will always reserve space for a big key (version >= 1000)
    keyBlob.Reserve(nbytes, &offsetKey);
@@ -1177,21 +1171,55 @@ std::uint64_t ROOT::Internal::RNTupleFileWriter::RFileProper::ReserveBlobKey(siz
    } else {
       unsigned char localKeyBuffer[kBlobKeyLen];
       PrepareBlobKey(offsetKey, nbytes, len, localKeyBuffer);
-      Write(localKeyBuffer, kBlobKeyLen, offsetKey);
+      caller.Write(localKeyBuffer, kBlobKeyLen, offsetKey);
    }
 
    if (keyBlob.WasAllocatedInAFreeSlot()) {
       // If the key was allocated in a free slot, the last 4 bytes of its buffer contain the new size
       // of the remaining free slot and we need to write it to disk before the key gets destroyed at the end of the
       // function.
-      Write(keyBlob.GetBuffer() + nbytes, sizeof(Int_t), offsetKey + kBlobKeyLen + nbytes);
+      caller.Write(keyBlob.GetBuffer() + nbytes, sizeof(Int_t), offsetKey + kBlobKeyLen + nbytes);
    }
 
    auto offsetData = offsetKey + kBlobKeyLen;
 
    return offsetData;
 }
 
+void ROOT::Internal::RNTupleFileWriter::RFileProper::Write(const void *buffer, size_t nbytes, std::int64_t offset)
+{
+   fDirectory->GetFile()->Seek(offset);
+   bool rv = fDirectory->GetFile()->WriteBuffer((char *)(buffer), nbytes);
+   if (rv)
+      throw RException(R__FAIL("WriteBuffer failed."));
+}
+
+std::uint64_t ROOT::Internal::RNTupleFileWriter::RFileProper::ReserveBlobKey(size_t nbytes, size_t len,
+                                                                             unsigned char keyBuffer[kBlobKeyLen])
+{
+   auto offsetData = RNTupleFileWriter::ReserveBlobKey(*this, *fDirectory->GetFile(), nbytes, len, keyBuffer);
+   return offsetData;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void ROOT::Internal::RNTupleFileWriter::RFileRFile::Write(const void *buffer, size_t nbytes, std::int64_t offset)
+{
+   auto *file = ROOT::Experimental::Internal::GetRFileTFile(*fFile);
+   file->Seek(offset);
+   bool rv = file->WriteBuffer((char *)(buffer), nbytes);
+   if (rv)
+      throw RException(R__FAIL("WriteBuffer failed."));
+}
+
+std::uint64_t ROOT::Internal::RNTupleFileWriter::RFileRFile::ReserveBlobKey(size_t nbytes, size_t len,
+                                                                            unsigned char keyBuffer[kBlobKeyLen])
+{
+   auto *file = ROOT::Experimental::Internal::GetRFileTFile(*fFile);
+   auto offsetData = RNTupleFileWriter::ReserveBlobKey(*this, *file, nbytes, len, keyBuffer);
+   return offsetData;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 ROOT::Internal::RNTupleFileWriter::RNTupleFileWriter(std::string_view name, std::uint64_t maxKeySize)
@@ -1277,6 +1305,18 @@ ROOT::Internal::RNTupleFileWriter::Append(std::string_view ntupleName, TDirector
    return writer;
 }
 
+std::unique_ptr<ROOT::Internal::RNTupleFileWriter>
+ROOT::Internal::RNTupleFileWriter::Append(std::string_view ntupleName, ROOT::Experimental::RFile &file,
+                                          std::string_view ntupleDir, std::uint64_t maxKeySize)
+{
+   auto writer = std::unique_ptr<RNTupleFileWriter>(new RNTupleFileWriter(ntupleName, maxKeySize));
+   auto &rfile = writer->fFile.emplace<RFileRFile>();
+   rfile.fFile = &file;
+   R__ASSERT(ntupleDir.empty() || ntupleDir[ntupleDir.size() - 1] == '/');
+   rfile.fDir = ntupleDir;
+   return writer;
+}
+
 void ROOT::Internal::RNTupleFileWriter::Seek(std::uint64_t offset)
 {
    RFileSimple *fileSimple = std::get_if<RFileSimple>(&fFile);
@@ -1295,18 +1335,26 @@ void ROOT::Internal::RNTupleFileWriter::UpdateStreamerInfos(const RNTupleSeriali
 
 void ROOT::Internal::RNTupleFileWriter::Commit(int compression)
 {
-   if (auto fileProper = std::get_if<RFileProper>(&fFile)) {
-      // Easy case, the ROOT file header and the RNTuple streaming is taken care of by TFile
-      fileProper->fDirectory->WriteObject(&fNTupleAnchor, fNTupleName.c_str());
-
+   const auto WriteStreamerInfoToFile = [&](TFile *file) {
       // Make sure the streamer info records used in the RNTuple are written to the file
       TBufferFile buf(TBuffer::kWrite);
-      buf.SetParent(fileProper->fDirectory->GetFile());
+      buf.SetParent(file);
       for (auto [_, info] : fStreamerInfoMap)
          buf.TagStreamerInfo(info);
+   };
 
+   if (auto fileProper = std::get_if<RFileProper>(&fFile)) {
+      // Easy case, the ROOT file header and the RNTuple streaming is taken care of by TFile
+      fileProper->fDirectory->WriteObject(&fNTupleAnchor, fNTupleName.c_str());
+      WriteStreamerInfoToFile(fileProper->fDirectory->GetFile());
       fileProper->fDirectory->GetFile()->Write();
       return;
+   } else if (auto fileRFile = std::get_if<RFileRFile>(&fFile)) {
+      // Same as the case above but handled via RFile
+      fileRFile->fFile->Put(fileRFile->fDir + fNTupleName, fNTupleAnchor);
+      WriteStreamerInfoToFile(ROOT::Experimental::Internal::GetRFileTFile(*fileRFile->fFile));
+      fileRFile->fFile->Flush();
+      return;
    }
 
    // Writing by C file stream: prepare the container format header and stream the RNTuple anchor object
@@ -1417,9 +1465,11 @@ ROOT::Internal::RNTupleFileWriter::ReserveBlob(size_t nbytes, size_t len, unsign
       } else {
          offset = fileSimple->ReserveBlobKey(nbytes, len, keyBuffer);
       }
+   } else if (auto *fileProper = std::get_if<RFileProper>(&fFile)) {
+      offset = fileProper->ReserveBlobKey(nbytes, len, keyBuffer);
    } else {
-      auto &fileProper = std::get<RFileProper>(fFile);
-      offset = fileProper.ReserveBlobKey(nbytes, len, keyBuffer);
+      auto &fileRFile = std::get<RFileRFile>(fFile);
+      offset = fileRFile.ReserveBlobKey(nbytes, len, keyBuffer);
    }
    return offset;
 }
@@ -1428,9 +1478,11 @@ void ROOT::Internal::RNTupleFileWriter::WriteIntoReservedBlob(const void *buffer
 {
    if (auto *fileSimple = std::get_if<RFileSimple>(&fFile)) {
       fileSimple->Write(buffer, nbytes, offset);
+   } else if (auto *fileProper = std::get_if<RFileProper>(&fFile)) {
+      fileProper->Write(buffer, nbytes, offset);
    } else {
-      auto &fileProper = std::get<RFileProper>(fFile);
-      fileProper.Write(buffer, nbytes, offset);
+      auto &fileRFile = std::get<RFileRFile>(fFile);
+      fileRFile.Write(buffer, nbytes, offset);
    }
 }
 

tree/ntuple/src/RNTupleWriter.cxx

@@ -23,6 +23,7 @@
 #include <ROOT/RPageSinkBuf.hxx>
 #include <ROOT/RPageStorage.hxx>
 #include <ROOT/RPageStorageFile.hxx>
+#include <ROOT/RFile.hxx>
 
 #include <TFile.h>
 #include <TROOT.h>
@@ -150,3 +151,12 @@ ROOT::Internal::CreateRNTupleWriter(std::unique_ptr<ROOT::RNTupleModel> model,
 {
    return std::unique_ptr<ROOT::RNTupleWriter>(new ROOT::RNTupleWriter(std::move(model), std::move(sink)));
 }
+
+std::unique_ptr<ROOT::RNTupleWriter>
+ROOT::Experimental::RNTupleWriter_Append(std::unique_ptr<ROOT::RNTupleModel> model, std::string_view ntupleName,
+                                         ROOT::Experimental::RFile &file, const ROOT::RNTupleWriteOptions &options)
+{
+   auto [ntupleDir, ntupleBasename] = ROOT::Experimental::Detail::DecomposePath(ntupleName);
+   auto sink = std::make_unique<ROOT::Internal::RPageSinkFile>(ntupleBasename, file, ntupleDir, options);
+   return ROOT::RNTupleWriter::Create(std::move(model), std::move(sink), options);
+}