Skip to content

Commit cb3f237

Browse files
committed
Add opt-in INSERT IGNORE support for MySQL via insert_mode option
When using on_conflict: :nothing, the MySQL adapter uses the ON DUPLICATE KEY UPDATE x = x workaround which always reports 1 affected row regardless of actual insert. This adds insert_mode: :ignore_errors option for insert_all that uses INSERT IGNORE instead, providing accurate row counts (0 when ignored, 1 when inserted). Usage: Repo.insert_all(Post, posts, on_conflict: :nothing, insert_mode: :ignore_errors) The Connection behavior is updated to accept opts in insert/8, allowing adapter-specific options to flow through.
1 parent 7478895 commit cb3f237

File tree

8 files changed

+135
-56
lines changed

8 files changed

+135
-56
lines changed

integration_test/myxql/upsert_all_test.exs

Lines changed: 44 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -13,60 +13,84 @@ defmodule Ecto.Integration.UpsertAllTest do
1313

1414
test "on conflict ignore" do
1515
post = [title: "first", uuid: "6fa459ea-ee8a-3ca4-894e-db77e160355e"]
16-
# First insert succeeds - 1 row inserted
16+
# Default :nothing behavior uses ON DUPLICATE KEY UPDATE x = x workaround
17+
# which always reports rows as affected
18+
assert TestRepo.insert_all(Post, [post], on_conflict: :nothing) == {1, nil}
1719
assert TestRepo.insert_all(Post, [post], on_conflict: :nothing) == {1, nil}
20+
end
21+
22+
test "insert_mode: :ignore_errors" do
23+
post = [title: "first", uuid: "6fa459ea-ee8a-3ca4-894e-db77e160355e"]
24+
# First insert succeeds - 1 row inserted
25+
assert TestRepo.insert_all(Post, [post],
26+
on_conflict: :nothing,
27+
insert_mode: :ignore_errors
28+
) == {1, nil}
29+
1830
# Second insert is ignored due to duplicate - 0 rows inserted (INSERT IGNORE behavior)
19-
assert TestRepo.insert_all(Post, [post], on_conflict: :nothing) == {0, nil}
31+
assert TestRepo.insert_all(Post, [post],
32+
on_conflict: :nothing,
33+
insert_mode: :ignore_errors
34+
) == {0, nil}
2035
end
2136

22-
test "on conflict ignore with mixed records (some conflicts, some new)" do
37+
test "insert_mode: :ignore_errors with mixed records (some conflicts, some new)" do
2338
# Insert an existing post
2439
existing_uuid = "6fa459ea-ee8a-3ca4-894e-db77e160355e"
2540
existing_post = [title: "existing", uuid: existing_uuid]
26-
assert TestRepo.insert_all(Post, [existing_post], on_conflict: :nothing) == {1, nil}
41+
42+
assert TestRepo.insert_all(Post, [existing_post],
43+
on_conflict: :nothing,
44+
insert_mode: :ignore_errors
45+
) == {1, nil}
2746

2847
# Now insert a batch with one duplicate and two new records
2948
new_uuid1 = "7fa459ea-ee8a-3ca4-894e-db77e160355f"
3049
new_uuid2 = "8fa459ea-ee8a-3ca4-894e-db77e160355a"
3150

3251
posts = [
33-
[title: "new post 1", uuid: new_uuid1], # new - should be inserted
34-
[title: "duplicate", uuid: existing_uuid], # duplicate - should be ignored
35-
[title: "new post 2", uuid: new_uuid2] # new - should be inserted
52+
[title: "new post 1", uuid: new_uuid1],
53+
[title: "duplicate", uuid: existing_uuid],
54+
[title: "new post 2", uuid: new_uuid2]
3655
]
3756

3857
# With INSERT IGNORE, only 2 rows should be inserted (the non-duplicates)
39-
assert TestRepo.insert_all(Post, posts, on_conflict: :nothing) == {2, nil}
58+
assert TestRepo.insert_all(Post, posts,
59+
on_conflict: :nothing,
60+
insert_mode: :ignore_errors
61+
) == {2, nil}
4062

4163
# Verify the data - should have 3 posts total (1 existing + 2 new)
4264
assert length(TestRepo.all(Post)) == 3
4365

4466
# Verify the existing post was not modified
4567
[original] = TestRepo.all(from p in Post, where: p.uuid == ^existing_uuid)
46-
assert original.title == "existing" # title unchanged
68+
assert original.title == "existing"
4769

4870
# Verify new posts were inserted
4971
assert TestRepo.exists?(from p in Post, where: p.uuid == ^new_uuid1)
5072
assert TestRepo.exists?(from p in Post, where: p.uuid == ^new_uuid2)
5173
end
5274

53-
test "on conflict ignore with all duplicates" do
75+
test "insert_mode: :ignore_errors with all duplicates" do
5476
# Insert initial posts
5577
uuid1 = "1fa459ea-ee8a-3ca4-894e-db77e160355e"
5678
uuid2 = "2fa459ea-ee8a-3ca4-894e-db77e160355e"
57-
initial_posts = [
58-
[title: "first", uuid: uuid1],
59-
[title: "second", uuid: uuid2]
60-
]
61-
assert TestRepo.insert_all(Post, initial_posts, on_conflict: :nothing) == {2, nil}
79+
initial_posts = [[title: "first", uuid: uuid1], [title: "second", uuid: uuid2]]
80+
81+
assert TestRepo.insert_all(Post, initial_posts,
82+
on_conflict: :nothing,
83+
insert_mode: :ignore_errors
84+
) == {2, nil}
6285

6386
# Try to insert all duplicates
64-
duplicate_posts = [
65-
[title: "dup1", uuid: uuid1],
66-
[title: "dup2", uuid: uuid2]
67-
]
87+
duplicate_posts = [[title: "dup1", uuid: uuid1], [title: "dup2", uuid: uuid2]]
88+
6889
# All are duplicates, so 0 rows inserted
69-
assert TestRepo.insert_all(Post, duplicate_posts, on_conflict: :nothing) == {0, nil}
90+
assert TestRepo.insert_all(Post, duplicate_posts,
91+
on_conflict: :nothing,
92+
insert_mode: :ignore_errors
93+
) == {0, nil}
7094

7195
# Verify count unchanged
7296
assert length(TestRepo.all(Post)) == 2

lib/ecto/adapters/myxql.ex

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,25 @@ defmodule Ecto.Adapters.MyXQL do
103103
automatically commits after some commands like CREATE TABLE.
104104
Therefore MySQL migrations does not run inside transactions.
105105
106+
### Upserts
107+
108+
When using `on_conflict: :nothing`, the adapter uses the
109+
`ON DUPLICATE KEY UPDATE x = x` workaround to simulate "do nothing"
110+
behavior. This always reports 1 affected row regardless of whether
111+
the row was actually inserted or ignored.
112+
113+
If you need accurate row counts (0 when ignored, 1 when inserted),
114+
you can opt into MySQL's `INSERT IGNORE` by specifying:
115+
116+
Repo.insert_all(Post, posts,
117+
on_conflict: :nothing,
118+
insert_mode: :ignore_errors)
119+
120+
Note that `INSERT IGNORE` has broader semantics in MySQL - it also
121+
ignores certain type conversion errors, not just duplicate key conflicts.
122+
The `insert_mode: :ignore_errors` option only affects the behavior of
123+
`on_conflict: :nothing`.
124+
106125
## Old MySQL versions
107126
108127
### JSON support
@@ -319,7 +338,10 @@ defmodule Ecto.Adapters.MyXQL do
319338

320339
key = primary_key!(schema_meta, returning)
321340
{fields, values} = :lists.unzip(params)
322-
sql = @conn.insert(prefix, source, fields, [fields], on_conflict, [], [])
341+
342+
# Extract insert_mode and pass it to the connection's insert function
343+
insert_opts = if opts[:insert_mode], do: [insert_mode: opts[:insert_mode]], else: []
344+
sql = @conn.insert(prefix, source, fields, [fields], on_conflict, [], [], insert_opts)
323345

324346
opts =
325347
if is_nil(Keyword.get(opts, :cache_statement)) do
@@ -330,16 +352,14 @@ defmodule Ecto.Adapters.MyXQL do
330352

331353
case Ecto.Adapters.SQL.query(adapter_meta, sql, values ++ query_params, opts) do
332354
{:ok, %{num_rows: 0}} ->
333-
# With INSERT IGNORE (on_conflict: :nothing), 0 rows means the row was
334-
# ignored due to a conflict, which is expected behavior
335-
case on_conflict do
336-
{:nothing, _, _} ->
337-
{:ok, []}
338-
339-
_ ->
340-
raise "insert operation failed to insert any row in the database. " <>
341-
"This may happen if you have trigger or other database conditions rejecting operations. " <>
342-
"The emitted SQL was: #{sql}"
355+
# With INSERT IGNORE (insert_mode: :ignore_errors), 0 rows means the row
356+
# was ignored due to a conflict, which is expected behavior
357+
if opts[:insert_mode] == :ignore_errors do
358+
{:ok, []}
359+
else
360+
raise "insert operation failed to insert any row in the database. " <>
361+
"This may happen if you have trigger or other database conditions rejecting operations. " <>
362+
"The emitted SQL was: #{sql}"
343363
end
344364

345365
# We were used to check if num_rows was 1 or 2 (in case of upserts)

lib/ecto/adapters/myxql/connection.ex

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -179,45 +179,61 @@ if Code.ensure_loaded?(MyXQL) do
179179
end
180180

181181
@impl true
182-
def insert(prefix, table, header, rows, on_conflict, [], []) do
182+
def insert(prefix, table, header, rows, on_conflict, returning, placeholders, opts \\ [])
183+
184+
def insert(prefix, table, header, rows, on_conflict, [], [], opts) do
183185
fields = quote_names(header)
184-
insert_keyword = insert_keyword(on_conflict)
186+
insert_keyword = insert_keyword(on_conflict, opts)
185187

186188
[
187189
insert_keyword,
188190
quote_table(prefix, table),
189191
" (",
190192
fields,
191193
") ",
192-
insert_all(rows) | on_conflict(on_conflict, header)
194+
insert_all(rows) | on_conflict(on_conflict, header, opts)
193195
]
194196
end
195197

196-
defp insert_keyword({:nothing, _, []}), do: "INSERT IGNORE INTO "
197-
defp insert_keyword(_), do: "INSERT INTO "
198-
199-
def insert(_prefix, _table, _header, _rows, _on_conflict, _returning, []) do
198+
def insert(_prefix, _table, _header, _rows, _on_conflict, _returning, [], _opts) do
200199
error!(nil, ":returning is not supported in insert/insert_all by MySQL")
201200
end
202201

203-
def insert(_prefix, _table, _header, _rows, _on_conflict, _returning, _placeholders) do
202+
def insert(_prefix, _table, _header, _rows, _on_conflict, _returning, _placeholders, _opts) do
204203
error!(nil, ":placeholders is not supported by MySQL")
205204
end
206205

207-
defp on_conflict({_, _, [_ | _]}, _header) do
206+
# INSERT IGNORE when insert_mode: :ignore_errors is passed
207+
defp insert_keyword({:nothing, _, _}, opts) do
208+
if Keyword.get(opts, :insert_mode) == :ignore_errors do
209+
"INSERT IGNORE INTO "
210+
else
211+
"INSERT INTO "
212+
end
213+
end
214+
215+
defp insert_keyword(_, _opts), do: "INSERT INTO "
216+
217+
defp on_conflict({_, _, [_ | _]}, _header, _opts) do
208218
error!(nil, ":conflict_target is not supported in insert/insert_all by MySQL")
209219
end
210220

211-
defp on_conflict({:raise, _, []}, _header) do
221+
defp on_conflict({:raise, _, []}, _header, _opts) do
212222
[]
213223
end
214224

215-
defp on_conflict({:nothing, _, []}, _header) do
216-
# Handled by INSERT IGNORE
217-
[]
225+
# With insert_mode: :ignore_errors, INSERT IGNORE handles conflicts - no ON DUPLICATE KEY needed
226+
defp on_conflict({:nothing, _, []}, [field | _], opts) do
227+
if Keyword.get(opts, :insert_mode) == :ignore_errors do
228+
[]
229+
else
230+
# Default :nothing - uses workaround to simulate "do nothing" behavior
231+
quoted = quote_name(field)
232+
[" ON DUPLICATE KEY UPDATE ", quoted, " = " | quoted]
233+
end
218234
end
219235

220-
defp on_conflict({fields, _, []}, _header) when is_list(fields) do
236+
defp on_conflict({fields, _, []}, _header, _opts) when is_list(fields) do
221237
[
222238
" ON DUPLICATE KEY UPDATE "
223239
| Enum.map_intersperse(fields, ?,, fn field ->
@@ -227,11 +243,11 @@ if Code.ensure_loaded?(MyXQL) do
227243
]
228244
end
229245

230-
defp on_conflict({%{wheres: []} = query, _, []}, _header) do
246+
defp on_conflict({%{wheres: []} = query, _, []}, _header, _opts) do
231247
[" ON DUPLICATE KEY " | update_all(query, "UPDATE ")]
232248
end
233249

234-
defp on_conflict({_query, _, []}, _header) do
250+
defp on_conflict({_query, _, []}, _header, _opts) do
235251
error!(
236252
nil,
237253
"Using a query with :where in combination with the :on_conflict option is not supported by MySQL"

lib/ecto/adapters/postgres/connection.ex

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ if Code.ensure_loaded?(Postgrex) do
229229
end
230230

231231
@impl true
232-
def insert(prefix, table, header, rows, on_conflict, returning, placeholders) do
232+
def insert(prefix, table, header, rows, on_conflict, returning, placeholders, _opts \\ []) do
233233
counter_offset = length(placeholders) + 1
234234

235235
values =

lib/ecto/adapters/sql.ex

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -970,7 +970,7 @@ defmodule Ecto.Adapters.SQL do
970970
rows -> unzip_inserts(header, rows)
971971
end
972972

973-
sql = conn.insert(prefix, source, header, rows, on_conflict, returning, placeholders)
973+
sql = conn.insert(prefix, source, header, rows, on_conflict, returning, placeholders, opts)
974974

975975
opts =
976976
if is_nil(Keyword.get(opts, :cache_statement)) do

lib/ecto/adapters/sql/connection.ex

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,8 @@ defmodule Ecto.Adapters.SQL.Connection do
9494
rows :: [[atom | nil]],
9595
on_conflict :: Ecto.Adapter.Schema.on_conflict(),
9696
returning :: [atom],
97-
placeholders :: [term]
97+
placeholders :: [term],
98+
opts :: Keyword.t()
9899
) :: iodata
99100

100101
@doc """

lib/ecto/adapters/tds/connection.ex

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,7 @@ if Code.ensure_loaded?(Tds) do
220220
end
221221

222222
@impl true
223-
def insert(prefix, table, header, rows, on_conflict, returning, placeholders) do
223+
def insert(prefix, table, header, rows, on_conflict, returning, placeholders, _opts \\ []) do
224224
counter_offset = length(placeholders) + 1
225225
[] = on_conflict(on_conflict, header)
226226
returning = returning(returning, "INSERTED")

test/ecto/adapters/myxql_test.exs

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,8 @@ defmodule Ecto.Adapters.MyXQLTest do
5656
defp delete_all(query), do: query |> SQL.delete_all() |> IO.iodata_to_binary()
5757
defp execute_ddl(query), do: query |> SQL.execute_ddl() |> Enum.map(&IO.iodata_to_binary/1)
5858

59-
defp insert(prefx, table, header, rows, on_conflict, returning) do
60-
IO.iodata_to_binary(SQL.insert(prefx, table, header, rows, on_conflict, returning, []))
59+
defp insert(prefx, table, header, rows, on_conflict, returning, opts \\ []) do
60+
IO.iodata_to_binary(SQL.insert(prefx, table, header, rows, on_conflict, returning, [], opts))
6161
end
6262

6363
defp update(prefx, table, fields, filter, returning) do
@@ -1465,10 +1465,12 @@ defmodule Ecto.Adapters.MyXQLTest do
14651465
end
14661466
end
14671467

1468-
test "insert with on conflict" do
1469-
# Using INSERT IGNORE for :nothing on_conflict
1468+
test "insert with on duplicate key" do
1469+
# Default :nothing uses ON DUPLICATE KEY UPDATE workaround
14701470
query = insert(nil, "schema", [:x, :y], [[:x, :y]], {:nothing, [], []}, [])
1471-
assert query == ~s{INSERT IGNORE INTO `schema` (`x`,`y`) VALUES (?,?)}
1471+
1472+
assert query ==
1473+
~s{INSERT INTO `schema` (`x`,`y`) VALUES (?,?) ON DUPLICATE KEY UPDATE `x` = `x`}
14721474

14731475
update = from("schema", update: [set: [z: "foo"]]) |> plan(:update_all)
14741476
query = insert(nil, "schema", [:x, :y], [[:x, :y]], {update, [], []}, [])
@@ -1498,6 +1500,22 @@ defmodule Ecto.Adapters.MyXQLTest do
14981500
end
14991501
end
15001502

1503+
test "insert with insert_mode: :ignore_errors" do
1504+
# INSERT IGNORE via insert_mode: :ignore_errors option
1505+
query =
1506+
insert(
1507+
nil,
1508+
"schema",
1509+
[:x, :y],
1510+
[[:x, :y]],
1511+
{:nothing, [], []},
1512+
[],
1513+
insert_mode: :ignore_errors
1514+
)
1515+
1516+
assert query == ~s{INSERT IGNORE INTO `schema` (`x`,`y`) VALUES (?,?)}
1517+
end
1518+
15011519
test "insert with query" do
15021520
select_query = from("schema", select: [:id]) |> plan(:all)
15031521

0 commit comments

Comments
 (0)