From a498d31db9fdcb4c8ceaccb5d5ebdfbabf14fe2c Mon Sep 17 00:00:00 2001
From: Emil Ejbyfeldt <emil.ejbyfeldt@choreograph.com>
Date: Wed, 18 Feb 2026 16:18:49 +0100
Subject: [PATCH] SPARK-55589: TransformingEncoder support from Option to
 nullable

---
 .../catalyst/DeserializerBuildHelper.scala    |  3 ++-
 .../encoders/ExpressionEncoderSuite.scala     | 22 +++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala
index 080794643fa0e..4c29f3227b060 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala
@@ -494,7 +494,8 @@ object DeserializerBuildHelper {
         Literal.create(provider(), ObjectType(classOf[Codec[_, _]])),
         "decode",
         dataTypeForClass(tag.runtimeClass),
-        createDeserializer(encoder, path, walkedTypePath, isTopLevel) :: Nil)
+        createDeserializer(encoder, path, walkedTypePath, isTopLevel) :: Nil,
+        propagateNull = !encoder.nullable)
   }
 
   private def deserializeArray(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
index 287b99d10d659..8d2bbb2577a6f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
@@ -766,6 +766,28 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
     testDataTransformingEnc(enc, data)
   }
 
+  test("SPARK-55589: TransformingEncoder from Option to nullable and back") {
+    type Nested = Option[Option[Int]]
+    type Inner = Tuple1[Option[Int]]
+    val data: Seq[Nested] = Seq(None, Some(None), Some(Some(1)))
+    val codec = new Codec[Nested, Inner] with Serializable {
+      override def encode(in: Nested): Inner = in match {
+        case Some(v) => Tuple1(v)
+        case None => null
+      }
+      override def decode(out: Inner): Nested = Option.when(out != null)(out._1)
+    }
+    val inner = ProductEncoder[Inner](
+      classTag,
+      Seq(EncoderField("_1", OptionEncoder(PrimitiveIntEncoder), nullable = true, Metadata.empty)),
+      None
+    )
+    val enc = TransformingEncoder[Nested, Tuple1[Option[Int]]](
+      classTag, inner, () => codec, nullable = true
+    )
+    testDataTransformingEnc(enc, data)
+  }
+
   test("SPARK-52601 TransformingEncoder from primitive to timestamp") {
     val enc: AgnosticEncoder[Long] =
       TransformingEncoder[Long, java.sql.Timestamp](