GH-49071: [Ruby] Add support for writing list and large list arrays (#49072)

### Rationale for this change

They use different offset size.

### What changes are included in this PR?

* Add `ArrowFormat::ListType#to_flatbuffers`
* Add `ArrowFormat::LargeListType#to_flatbuffers`
* Add `ArrowFormat::VariableSizeListArray#child`
* Add `ArrowFormat::VariableSizeListArray#each_buffer`
* `garrow_array_get_null_bitmap()` returns `NULL` when null bitmap doesn't exist
* Add `garrow_list_array_get_value_offsets_buffer()`
* Add `garrow_large_list_array_get_value_offsets_buffer()`

### Are these changes tested?

Yes.

### Are there any user-facing changes?

Yes.
* GitHub Issue: #49071

Authored-by: Sutou Kouhei <kou@clear-code.com>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>

Author: kou

c_glib/arrow-glib/basic-array.cpp

@@ -1114,7 +1114,11 @@ garrow_array_get_null_bitmap(GArrowArray *array)
 
   auto arrow_array = garrow_array_get_raw(array);
   auto arrow_null_bitmap = arrow_array->null_bitmap();
-  return garrow_buffer_new_raw(&arrow_null_bitmap);
+  if (arrow_null_bitmap) {
+    return garrow_buffer_new_raw(&arrow_null_bitmap);
+  } else {
+    return nullptr;
+  }
 }
 
 /**

c_glib/arrow-glib/composite-array.cpp

@@ -188,6 +188,22 @@ garrow_base_list_array_get_value_offsets(GArrowArray *array, gint64 *n_offsets)
   return arrow_list_array->raw_value_offsets();
 };
 
+template <typename LIST_ARRAY_CLASS>
+GArrowBuffer *
+garrow_base_list_array_get_value_offsets_buffer(GArrowArray *array)
+{
+  GArrowBuffer *buffer = nullptr;
+  g_object_get(array, "buffer1", &buffer, nullptr);
+  if (buffer) {
+    return buffer;
+  }
+
+  auto arrow_array = garrow_array_get_raw(array);
+  auto arrow_list_array = std::static_pointer_cast<LIST_ARRAY_CLASS>(arrow_array);
+  auto arrow_buffer = arrow_list_array->value_offsets();
+  return garrow_buffer_new_raw(&arrow_buffer);
+};
+
 G_BEGIN_DECLS
 
 static void
@@ -385,6 +401,21 @@ garrow_list_array_get_value_offsets(GArrowListArray *array, gint64 *n_offsets)
                                                                     n_offsets);
 }
 
+/**
+ * garrow_list_array_get_value_offsets_buffer:
+ * @array: A #GArrowListArray.
+ *
+ * Returns: (transfer full) (nullable): The value offsets buffer.
+ *
+ * Since: 24.0.0
+ */
+GArrowBuffer *
+garrow_list_array_get_value_offsets_buffer(GArrowListArray *array)
+{
+  return garrow_base_list_array_get_value_offsets_buffer<arrow::ListArray>(
+    GARROW_ARRAY(array));
+}
+
 typedef struct GArrowLargeListArrayPrivate_
 {
   GArrowArray *raw_values;
@@ -602,6 +633,21 @@ garrow_large_list_array_get_value_offsets(GArrowLargeListArray *array, gint64 *n
   return reinterpret_cast<const gint64 *>(value_offsets);
 }
 
+/**
+ * garrow_large_list_array_get_value_offsets_buffer:
+ * @array: A #GArrowLargeListArray.
+ *
+ * Returns: (transfer full) (nullable): The value offsets buffer.
+ *
+ * Since: 24.0.0
+ */
+GArrowBuffer *
+garrow_large_list_array_get_value_offsets_buffer(GArrowLargeListArray *array)
+{
+  return garrow_base_list_array_get_value_offsets_buffer<arrow::LargeListArray>(
+    GARROW_ARRAY(array));
+}
+
 typedef struct GArrowFixedSizeListArrayPrivate_
 {
   GArrowArray *raw_values;

c_glib/arrow-glib/composite-array.h

@@ -68,6 +68,10 @@ GARROW_AVAILABLE_IN_2_0
 const gint32 *
 garrow_list_array_get_value_offsets(GArrowListArray *array, gint64 *n_offsets);
 
+GARROW_AVAILABLE_IN_24_0
+GArrowBuffer *
+garrow_list_array_get_value_offsets_buffer(GArrowListArray *array);
+
 #define GARROW_TYPE_LARGE_LIST_ARRAY (garrow_large_list_array_get_type())
 GARROW_AVAILABLE_IN_0_16
 G_DECLARE_DERIVABLE_TYPE(
@@ -110,6 +114,10 @@ GARROW_AVAILABLE_IN_2_0
 const gint64 *
 garrow_large_list_array_get_value_offsets(GArrowLargeListArray *array, gint64 *n_offsets);
 
+GARROW_AVAILABLE_IN_24_0
+GArrowBuffer *
+garrow_large_list_array_get_value_offsets_buffer(GArrowLargeListArray *array);
+
 #define GARROW_TYPE_FIXED_SIZE_LIST_ARRAY (garrow_fixed_size_list_array_get_type())
 GARROW_AVAILABLE_IN_23_0
 G_DECLARE_DERIVABLE_TYPE(GArrowFixedSizeListArray,

ruby/red-arrow-format/lib/arrow-format/array.rb

@@ -370,12 +370,20 @@ class Decimal256Array < DecimalArray
   end
 
   class VariableSizeListArray < Array
+    attr_reader :child
     def initialize(type, size, validity_buffer, offsets_buffer, child)
       super(type, size, validity_buffer)
       @offsets_buffer = offsets_buffer
       @child = child
     end
 
+    def each_buffer(&block)
+      return to_enum(__method__) unless block_given?
+
+      yield(@validity_buffer)
+      yield(@offsets_buffer)
+    end
+
     def to_a
       child_values = @child.to_a
       values = @offsets_buffer.

ruby/red-arrow-format/lib/arrow-format/field.rb

@@ -49,7 +49,9 @@ def to_flatbuffers
       else
         fb_field.type = @type.to_flatbuffers
       end
-      if @type.respond_to?(:children)
+      if @type.respond_to?(:child)
+        fb_field.children = [@type.child.to_flatbuffers]
+      elsif @type.respond_to?(:children)
         fb_field.children = @type.children.collect(&:to_flatbuffers)
       end
       # fb_field.custom_metadata = @custom_metadata

ruby/red-arrow-format/lib/arrow-format/record-batch.rb

@@ -70,7 +70,9 @@ def all_columns_enumerator
       Enumerator.new do |yielder|
         traverse = lambda do |array|
           yielder << array
-          if array.respond_to?(:children)
+          if array.respond_to?(:child)
+            traverse.call(array.child)
+          elsif array.respond_to?(:children)
             array.children.each do |child_array|
               traverse.call(child_array)
             end

ruby/red-arrow-format/lib/arrow-format/type.rb

@@ -707,7 +707,6 @@ def initialize(child)
       super()
       @child = child
     end
-
   end
 
   class ListType < VariableSizeListType
@@ -718,6 +717,10 @@ def name
     def build_array(size, validity_buffer, offsets_buffer, child)
       ListArray.new(self, size, validity_buffer, offsets_buffer, child)
     end
+
+    def to_flatbuffers
+      FB::List::Data.new
+    end
   end
 
   class LargeListType < VariableSizeListType
@@ -728,6 +731,10 @@ def name
     def build_array(size, validity_buffer, offsets_buffer, child)
       LargeListArray.new(self, size, validity_buffer, offsets_buffer, child)
     end
+
+    def to_flatbuffers
+      FB::LargeList::Data.new
+    end
   end
 
   class StructType < Type

ruby/red-arrow-format/test/test-writer.rb

@@ -83,11 +83,22 @@ def convert_type(red_arrow_type)
                                       red_arrow_type.scale)
     when Arrow::FixedSizeBinaryDataType
       ArrowFormat::FixedSizeBinaryType.new(red_arrow_type.byte_width)
+    when Arrow::ListDataType
+      ArrowFormat::ListType.new(convert_field(red_arrow_type.field))
+    when Arrow::LargeListDataType
+      ArrowFormat::LargeListType.new(convert_field(red_arrow_type.field))
     else
       raise "Unsupported type: #{red_arrow_type.inspect}"
     end
   end
 
+  def convert_field(red_arrow_field)
+    ArrowFormat::Field.new(red_arrow_field.name,
+                           convert_type(red_arrow_field.data_type),
+                           red_arrow_field.nullable?,
+                           nil)
+  end
+
   def convert_buffer(buffer)
     return nil if buffer.nil?
     IO::Buffer.for(buffer.data.to_s)
@@ -111,6 +122,11 @@ def convert_array(red_arrow_array)
       type.build_array(red_arrow_array.size,
                        convert_buffer(red_arrow_array.null_bitmap),
                        convert_buffer(red_arrow_array.data_buffer))
+    when ArrowFormat::VariableSizeListType
+      type.build_array(red_arrow_array.size,
+                       convert_buffer(red_arrow_array.null_bitmap),
+                       convert_buffer(red_arrow_array.value_offsets_buffer),
+                       convert_array(red_arrow_array.values_raw))
     else
       raise "Unsupported array #{red_arrow_array.inspect}"
     end
@@ -706,6 +722,32 @@ def test_write
                          @values)
           end
         end
+
+        sub_test_case("List") do
+          def build_array
+            data_type = Arrow::ListDataType.new(name: "count", type: :int8)
+            Arrow::ListArray.new(data_type, [[-128, 127], nil, [-1, 0, 1]])
+          end
+
+          def test_write
+            assert_equal([[-128, 127], nil, [-1, 0, 1]],
+                         @values)
+          end
+        end
+
+        sub_test_case("LargeList") do
+          def build_array
+            data_type = Arrow::LargeListDataType.new(name: "count",
+                                                     type: :int8)
+            Arrow::LargeListArray.new(data_type,
+                                      [[-128, 127], nil, [-1, 0, 1]])
+          end
+
+          def test_write
+            assert_equal([[-128, 127], nil, [-1, 0, 1]],
+                         @values)
+          end
+        end
       end
     end
   end