Merge pull request #64 from ocean/fix/in-clause-json-encoding

Fix issue #63: IN clause datatype mismatch with JSON-encoded lists

Author: ocean

lib/ecto_libsql/query.ex

@@ -90,24 +90,12 @@ defmodule EctoLibSql.Query do
       end
     end
 
-    # List/Array encoding: lists are encoded to JSON arrays
-    # Lists must contain only JSON-serializable values (strings, numbers, booleans,
-    # nil, lists, and maps). This enables array parameter support in raw SQL queries.
-    defp encode_param(value) when is_list(value) do
-      case Jason.encode(value) do
-        {:ok, json} ->
-          json
-
-        {:error, %Jason.EncodeError{message: msg}} ->
-          raise ArgumentError,
-            message:
-              "Cannot encode list parameter to JSON. List contains non-JSON-serializable value. " <>
-                "Lists can only contain strings, numbers, booleans, nil, lists, and maps. " <>
-                "Reason: #{msg}. List: #{inspect(value)}"
-      end
-    end
-
     # Pass through all other values unchanged
+    # Note: Lists are not automatically JSON-encoded here.
+    # - For Ecto queries with IN clauses: Ecto's query builder expands lists into individual parameters
+    # - For array fields in schemas: Ecto dumpers handle JSON encoding via array_encode/1
+    # - For raw SQL with arrays: Users should pre-encode lists using Jason.encode!
+    # This design allows IN clauses to work correctly while still supporting array fields.
     defp encode_param(value), do: value
 
     # Pass through results from Native.ex unchanged.

test/issue_63_in_clause_test.exs

@@ -0,0 +1,125 @@
+defmodule EctoLibSql.Issue63InClauseTest do
+  @moduledoc """
+  Test case for issue #63: Datatype mismatch due to JSON encoding of lists in IN statements.
+
+  The issue occurs when lists are used as parameters in IN clauses.
+  Instead of expanding the list into individual parameters, the entire list
+  was being JSON-encoded as a single string parameter, causing SQLite to raise
+  a "datatype mismatch" error.
+  """
+
+  use EctoLibSql.Integration.Case, async: false
+
+  alias EctoLibSql.Integration.TestRepo
+  alias EctoLibSql.Schemas.Product
+
+  import Ecto.Query
+
+  @test_db "z_ecto_libsql_test-issue_63.db"
+
+  setup_all do
+    Application.put_env(:ecto_libsql, EctoLibSql.Integration.TestRepo,
+      adapter: Ecto.Adapters.LibSql,
+      database: @test_db
+    )
+
+    {:ok, _} = EctoLibSql.Integration.TestRepo.start_link()
+
+    # Create test table with state column
+    Ecto.Adapters.SQL.query!(TestRepo, """
+    CREATE TABLE IF NOT EXISTS test_items (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      state TEXT,
+      name TEXT,
+      inserted_at TEXT,
+      updated_at TEXT
+    )
+    """)
+
+    on_exit(fn ->
+      EctoLibSql.TestHelpers.cleanup_db_files(@test_db)
+    end)
+
+    :ok
+  end
+
+  setup do
+    # Clear table before each test
+    Ecto.Adapters.SQL.query!(TestRepo, "DELETE FROM test_items", [])
+    :ok
+  end
+
+  test "IN clause with list parameter should not JSON-encode the list" do
+    # Insert test data with various states
+    Ecto.Adapters.SQL.query!(TestRepo, """
+    INSERT INTO test_items (state, name, inserted_at, updated_at)
+    VALUES ('scheduled', 'item1', datetime('now'), datetime('now')),
+           ('retryable', 'item2', datetime('now'), datetime('now')),
+           ('completed', 'item3', datetime('now'), datetime('now')),
+           ('failed', 'item4', datetime('now'), datetime('now'))
+    """)
+
+    # This query should work without datatype mismatch error
+    # Using a list parameter in an IN clause
+    states = ["scheduled", "retryable"]
+
+    query =
+      from(t in "test_items",
+        where: t.state in ^states,
+        select: t.name
+      )
+
+    # Execute the query - this should not raise "datatype mismatch" error
+    result = TestRepo.all(query)
+
+    # Should return the two items with scheduled or retryable state
+    assert length(result) == 2
+    assert "item1" in result
+    assert "item2" in result
+  end
+
+  test "IN clause with multiple parameter lists should work correctly" do
+    # Insert test data
+    Ecto.Adapters.SQL.query!(TestRepo, """
+    INSERT INTO test_items (state, name, inserted_at, updated_at)
+    VALUES ('active', 'item1', datetime('now'), datetime('now')),
+           ('inactive', 'item2', datetime('now'), datetime('now')),
+           ('pending', 'item3', datetime('now'), datetime('now'))
+    """)
+
+    # Query with multiple filters including IN clause
+    states = ["active", "pending"]
+
+    query =
+      from(t in "test_items",
+        where: t.state in ^states,
+        select: t.name
+      )
+
+    result = TestRepo.all(query)
+
+    assert length(result) == 2
+    assert "item1" in result
+    assert "item3" in result
+  end
+
+  test "IN clause with empty list parameter" do
+    # Insert test data
+    Ecto.Adapters.SQL.query!(TestRepo, """
+    INSERT INTO test_items (state, name, inserted_at, updated_at)
+    VALUES ('test', 'item1', datetime('now'), datetime('now'))
+    """)
+
+    # Query with empty list should return no results
+    query =
+      from(t in "test_items",
+        where: t.state in ^[],
+        select: t.name
+      )
+
+    result = TestRepo.all(query)
+
+    # Empty IN clause should match nothing
+    assert result == []
+  end
+end

test/type_loader_dumper_test.exs

@@ -650,19 +650,20 @@ defmodule EctoLibSql.TypeLoaderDumperTest do
   describe "array types" do
     test "array fields load and dump as JSON arrays" do
       array = ["a", "b", "c"]
+      json_array = Jason.encode!(array)
 
       {:ok, _} =
         Ecto.Adapters.SQL.query(
           TestRepo,
           "INSERT INTO all_types (array_field) VALUES (?)",
-          [array]
+          [json_array]
         )
 
       {:ok, result} = Ecto.Adapters.SQL.query(TestRepo, "SELECT array_field FROM all_types")
 
       # Should be stored as JSON array string
-      assert [[json_string]] = result.rows
-      assert {:ok, decoded} = Jason.decode(json_string)
+      assert [[^json_array]] = result.rows
+      assert {:ok, decoded} = Jason.decode(json_array)
       assert decoded == ["a", "b", "c"]
     end
 
@@ -682,16 +683,18 @@ defmodule EctoLibSql.TypeLoaderDumperTest do
     end
 
     test "handles empty arrays" do
+      empty_json = Jason.encode!([])
+
       {:ok, _} =
         Ecto.Adapters.SQL.query(
           TestRepo,
           "INSERT INTO all_types (array_field) VALUES (?)",
-          [[]]
+          [empty_json]
         )
 
       {:ok, result} = Ecto.Adapters.SQL.query(TestRepo, "SELECT array_field FROM all_types")
 
-      assert [["[]"]] = result.rows
+      assert [[^empty_json]] = result.rows
     end
 
     test "empty string defaults to empty array" do
@@ -762,6 +765,7 @@ defmodule EctoLibSql.TypeLoaderDumperTest do
       }
 
       # Insert via raw SQL
+      # Note: arrays and maps must be pre-encoded to JSON when using raw SQL
       {:ok, _} =
         Ecto.Adapters.SQL.query(
           TestRepo,
@@ -789,9 +793,9 @@ defmodule EctoLibSql.TypeLoaderDumperTest do
             attrs.naive_datetime_usec_field,
             attrs.utc_datetime_field,
             attrs.utc_datetime_usec_field,
-            attrs.map_field,
-            attrs.json_field,
-            attrs.array_field
+            Jason.encode!(attrs.map_field),
+            Jason.encode!(attrs.json_field),
+            Jason.encode!(attrs.array_field)
           ]
         )