From 14e8e6f631b6c00970c1a9fc3574e9f0f0b286e4 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 18 Aug 2025 09:50:51 +0300 Subject: [PATCH 1/6] Add getSequences method --- src/Database/Adapter.php | 7 ++++ src/Database/Adapter/SQL.php | 71 ++++++++++++++++++++++-------------- src/Database/Database.php | 4 ++ 3 files changed, 54 insertions(+), 28 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index c4579715f..c20593ba1 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -733,6 +733,13 @@ abstract public function createOrUpdateDocuments( array $changes ): array; + /** + * @param string $collection + * @param array $documents + * @return array + */ + abstract public function getSequences(string $collection, array $documents): array; + /** * Delete Document * diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 7b45fc102..ad8429637 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -710,49 +710,64 @@ public function deleteDocuments(string $collection, array $sequences, array $per } /** - * Get internal IDs for the given documents + * Assign internal IDs for the given documents * * @param string $collection - * @param array $documentIds - * @param array $documentTenants - * @return array + * @param array $documents + * @return array * @throws DatabaseException */ - protected function getSequences(string $collection, array $documentIds, array $documentTenants = []): array + public function getSequences(string $collection, array $documents): array { - $sequences = []; + $documentIds = []; + $keys = []; + $binds = []; - /** - * UID, _tenant bottleneck is ~ 5000 rows since we use _uid IN query - */ - foreach (\array_chunk($documentIds, 1000) as $documentIdsChunk) { - $sql = " - SELECT _uid, _id - FROM {$this->getSQLTable($collection)} - WHERE {$this->quote('_uid')} IN (" . implode(',', array_map(fn ($index) => ":_key_{$index}", array_keys($documentIdsChunk))) . ") - {$this->getTenantQuery($collection, tenantCount: \count($documentIdsChunk))} - "; + foreach ($documents as $i => $document) { + if (empty($document->getSequence())) { + $documentIds[] = $document->getId(); - $stmt = $this->getPDO()->prepare($sql); + $key = ":uid_{$i}"; - foreach ($documentIdsChunk as $index => $id) { - $stmt->bindValue(":_key_{$index}", $id); - } + $binds[$key] = $document->getId(); + $keys[] = $key; - if ($this->sharedTables) { - foreach ($documentIdsChunk as $index => $id) { - $stmt->bindValue(":_tenant_{$index}", \array_shift($documentTenants)); + if ($this->sharedTables) { + $binds[':_tenant_'.$i] = $document->getTenant(); } } + } - $stmt->execute(); - $results = $stmt->fetchAll(\PDO::FETCH_KEY_PAIR); // Fetch as [documentId => sequence] - $stmt->closeCursor(); + if (empty($documentIds)) { + return $documents; + } + + $placeholders = implode(',', array_values($keys)); + + $sql = " + SELECT _uid, _id + FROM {$this->getSQLTable($collection)} + WHERE {$this->quote('_uid')} IN ({$placeholders}) + {$this->getTenantQuery($collection, tenantCount: \count($documentIds))} + "; + + $stmt = $this->getPDO()->prepare($sql); - $sequences = [...$sequences, ...$results]; + foreach ($binds as $key => $value) { + $stmt->bindValue($key, $value); } - return $sequences; + $stmt->execute(); + $sequences = $stmt->fetchAll(\PDO::FETCH_KEY_PAIR); // Fetch as [documentId => sequence] + $stmt->closeCursor(); + + foreach ($documents as $document) { + if (isset($sequences[$document->getId()])) { + $document['$sequence'] = $sequences[$document->getId()]; + } + } + + return $documents; } /** diff --git a/src/Database/Database.php b/src/Database/Database.php index 6c44be72c..a72f565bd 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -3782,6 +3782,8 @@ public function createDocuments( return $this->adapter->createDocuments($collection->getId(), $chunk); }); + $batch = $this->adapter->getSequences($collection->getId(), $batch); + foreach ($batch as $document) { if ($this->resolveRelationships) { $document = $this->silent(fn () => $this->populateDocumentRelationships($collection, $document)); @@ -5097,6 +5099,8 @@ public function createOrUpdateDocumentsWithIncrease( $chunk ))); + $batch = $this->adapter->getSequences($collection->getId(), $batch); + foreach ($chunk as $change) { if ($change->getOld()->isEmpty()) { $created++; From 4fa95953af61787031023b04d9758db92ab044a8 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 18 Aug 2025 10:46:18 +0300 Subject: [PATCH 2/6] Add getSequences method --- phpunit.xml | 2 +- src/Database/Adapter/MariaDB.php | 15 --------------- src/Database/Adapter/SQL.php | 17 ----------------- 3 files changed, 1 insertion(+), 33 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index 2a0531cfd..34365d48d 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false" + stopOnFailure="true" > diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 708926548..23872617f 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1171,7 +1171,6 @@ public function createOrUpdateDocuments( $bindIndex = 0; $batchKeys = []; $bindValues = []; - $documentIds = []; $documentTenants = []; foreach ($changes as $change) { @@ -1184,8 +1183,6 @@ public function createOrUpdateDocuments( if (!empty($document->getSequence())) { $attributes['_id'] = $document->getSequence(); - } else { - $documentIds[] = $document->getId(); } if ($this->sharedTables) { @@ -1349,18 +1346,6 @@ public function createOrUpdateDocuments( } $stmtAddPermissions->execute(); } - - $sequences = $this->getSequences( - $collection, - $documentIds, - $documentTenants - ); - - foreach ($changes as $change) { - if (isset($sequences[$change->getNew()->getId()])) { - $change->getNew()->setAttribute('$sequence', $sequences[$change->getNew()->getId()]); - } - } } catch (PDOException $e) { throw $this->processException($e); } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index ad8429637..ee5debd59 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -1823,8 +1823,6 @@ public function createDocuments(string $collection, array $documents): array $batchKeys = []; $bindValues = []; $permissions = []; - $documentIds = []; - $documentTenants = []; foreach ($documents as $index => $document) { $attributes = $document->getAttributes(); @@ -1835,13 +1833,10 @@ public function createDocuments(string $collection, array $documents): array if (!empty($document->getSequence())) { $attributes['_id'] = $document->getSequence(); - } else { - $documentIds[] = $document->getId(); } if ($this->sharedTables) { $attributes['_tenant'] = $document->getTenant(); - $documentTenants[] = $document->getTenant(); } $bindKeys = []; @@ -1903,18 +1898,6 @@ public function createDocuments(string $collection, array $documents): array $this->execute($stmtPermissions); } - - $sequences = $this->getSequences( - $collection, - $documentIds, - $documentTenants - ); - - foreach ($documents as $document) { - if (isset($sequences[$document->getId()])) { - $document['$sequence'] = $sequences[$document->getId()]; - } - } } catch (PDOException $e) { throw $this->processException($e); } From f6830cbbc45230aac72d66f29649c792d3ac1e0e Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 18 Aug 2025 10:52:32 +0300 Subject: [PATCH 3/6] formatting --- src/Database/Adapter/Pool.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index 0a4e59018..e64db87ec 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -494,4 +494,9 @@ protected function execute(mixed $stmt): bool { return $this->delegate(__FUNCTION__, \func_get_args()); } + + public function getSequences(string $collection, array $documents): array + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } } From 7dffac5815c6c0cfdb01a17aa073a7943187c796 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 18 Aug 2025 11:03:39 +0300 Subject: [PATCH 4/6] revert stopOnFailure --- phpunit.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phpunit.xml b/phpunit.xml index 34365d48d..2a0531cfd 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="true" + stopOnFailure="false" > From 564a77215afe879b6569ecf67d3c139dab33e4f1 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 18 Aug 2025 11:10:27 +0300 Subject: [PATCH 5/6] Add tests --- tests/e2e/Adapter/Scopes/DocumentTests.php | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 445d7b21a..77f7ce457 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -252,6 +252,7 @@ public function testCreateDocuments(): void foreach ($results as $document) { $this->assertNotEmpty(true, $document->getId()); + $this->assertNotEmpty(true, $document->getSequence()); $this->assertIsString($document->getAttribute('string')); $this->assertEquals('textđź“ť', $document->getAttribute('string')); // Also makes sure an emoji is working $this->assertIsInt($document->getAttribute('integer')); From a302b44a5795d7b4095ce9e50f85fdbffdbd635c Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 18 Aug 2025 13:30:44 +0300 Subject: [PATCH 6/6] test getSequence --- tests/e2e/Adapter/Scopes/DocumentTests.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 77f7ce457..023c5b71a 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -251,8 +251,8 @@ public function testCreateDocuments(): void $this->assertEquals($count, \count($results)); foreach ($results as $document) { - $this->assertNotEmpty(true, $document->getId()); - $this->assertNotEmpty(true, $document->getSequence()); + $this->assertEquals(false, empty($document->getId())); + $this->assertEquals(false, empty($document->getSequence())); $this->assertIsString($document->getAttribute('string')); $this->assertEquals('textđź“ť', $document->getAttribute('string')); // Also makes sure an emoji is working $this->assertIsInt($document->getAttribute('integer'));