From 5d21db222626aa65a14dfa3ee42994f5e99ee324 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 27 Aug 2025 14:45:29 +0530 Subject: [PATCH 01/69] added attribute validation in the index --- src/Database/Database.php | 39 ++++++++++ tests/e2e/Adapter/Scopes/SpatialTests.php | 94 ++++++++++++++++++++++- 2 files changed, 132 insertions(+), 1 deletion(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index eb1f9c0d3..25ffaf81a 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1852,6 +1852,12 @@ private function validateAttribute( if (!$this->adapter->getSupportForSpatialAttributes()) { throw new DatabaseException('Spatial attributes are not supported'); } + if (!empty($size)) { + throw new DatabaseException('Size must be empty'); + } + if (!empty($array)) { + throw new DatabaseException('Array must be empty'); + } break; default: throw new DatabaseException('Unknown attribute type: ' . $type . '. Must be one of ' . self::VAR_STRING . ', ' . self::VAR_INTEGER . ', ' . self::VAR_FLOAT . ', ' . self::VAR_BOOLEAN . ', ' . self::VAR_DATETIME . ', ' . self::VAR_RELATIONSHIP . ', ' . self::VAR_POINT . ', ' . self::VAR_LINESTRING . ', ' . self::VAR_POLYGON); @@ -2173,6 +2179,20 @@ public function updateAttribute(string $collection, string $id, ?string $type = throw new DatabaseException('Size must be empty'); } break; + + case self::VAR_POINT: + case self::VAR_LINESTRING: + case self::VAR_POLYGON: + if (!$this->adapter->getSupportForSpatialAttributes()) { + throw new DatabaseException('Spatial attributes are not supported'); + } + if (!empty($size)) { + throw new DatabaseException('Size must be empty'); + } + if (!empty($array)) { + throw new DatabaseException('Array must be empty'); + } + break; default: throw new DatabaseException('Unknown attribute type: ' . $type . '. Must be one of ' . self::VAR_STRING . ', ' . self::VAR_INTEGER . ', ' . self::VAR_FLOAT . ', ' . self::VAR_BOOLEAN . ', ' . self::VAR_DATETIME . ', ' . self::VAR_RELATIONSHIP); } @@ -2221,6 +2241,25 @@ public function updateAttribute(string $collection, string $id, ?string $type = throw new LimitException('Row width limit reached. Cannot update attribute.'); } + // checking required of attributes in case of spatial types + if (in_array($type, Database::SPATIAL_TYPES) && !$this->adapter->getSupportForSpatialIndexNull()) { + $attributeMap = []; + foreach ($attributes as $attribute) { + $key = \strtolower($attribute->getAttribute('key', $attribute->getAttribute('$id'))); + $attributeMap[$key] = $attribute; + } + $indexes = $collectionDoc->getAttribute('indexes', []); + foreach ($indexes as $index) { + $attributes = $index->getAttribute('attributes', []); + foreach ($attributes as $attributeName) { + $attribute = $attributeMap[\strtolower($attributeName)]; + if (!$required) { + throw new IndexException('Spatial indexes do not allow null values. Mark the attribute "' . $attributeName . '" as required or create the index on a column with no null values.'); + } + } + } + } + if ($altering) { $indexes = $collectionDoc->getAttribute('indexes'); diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 8f6c2898b..97c0fb0a7 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -4,6 +4,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Exception; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; @@ -340,7 +341,12 @@ public function testSpatialAttributes(): void // Create spatial indexes $this->assertEquals(true, $database->createIndex($collectionName, 'idx_point', Database::INDEX_SPATIAL, ['pointAttr'])); - $this->assertEquals(true, $database->createIndex($collectionName, 'idx_line', Database::INDEX_SPATIAL, ['lineAttr'])); + if ($database->getAdapter()->getSupportForSpatialIndexNull()) { + $this->assertEquals(true, $database->createIndex($collectionName, 'idx_line', Database::INDEX_SPATIAL, ['lineAttr'])); + } else { + // Attribute was created as required above; directly create index once + $this->assertEquals(true, $database->createIndex($collectionName, 'idx_line', Database::INDEX_SPATIAL, ['lineAttr'])); + } $this->assertEquals(true, $database->createIndex($collectionName, 'idx_poly', Database::INDEX_SPATIAL, ['polyAttr'])); $collection = $database->getCollection($collectionName); @@ -1773,4 +1779,90 @@ public function testSptialAggregation(): void $database->deleteCollection($collectionName); } } + + public function testUpdateSpatialAttributes(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + if (!$database->getAdapter()->getSupportForSpatialAttributes()) { + $this->markTestSkipped('Adapter does not support spatial attributes'); + } + + $collectionName = 'spatial_update_attrs_'; + try { + $database->createCollection($collectionName); + + // 0) Disallow creation of spatial attributes with size or array + try { + $database->createAttribute($collectionName, 'geom_bad_size', Database::VAR_POINT, 10, true); + $this->fail('Expected DatabaseException when creating spatial attribute with non-zero size'); + } catch (\Throwable $e) { + $this->assertInstanceOf(Exception::class, $e); + } + + try { + $database->createAttribute($collectionName, 'geom_bad_array', Database::VAR_POINT, 0, true, array: true); + $this->fail('Expected DatabaseException when creating spatial attribute with array=true'); + } catch (\Throwable $e) { + $this->assertInstanceOf(Exception::class, $e); + } + + // Create a single spatial attribute (required=true) + $this->assertEquals(true, $database->createAttribute($collectionName, 'geom', Database::VAR_POINT, 0, true)); + $this->assertEquals(true, $database->createIndex($collectionName, 'idx_geom', Database::INDEX_SPATIAL, ['geom'])); + + // 1) Disallow size and array updates on spatial attributes: expect DatabaseException + try { + $database->updateAttribute($collectionName, 'geom', size: 10); + $this->fail('Expected DatabaseException when updating size on spatial attribute'); + } catch (\Throwable $e) { + $this->assertInstanceOf(Exception::class, $e); + } + + try { + $database->updateAttribute($collectionName, 'geom', array: true); + $this->fail('Expected DatabaseException when updating array on spatial attribute'); + } catch (\Throwable $e) { + $this->assertInstanceOf(Exception::class, $e); + } + + // 2) required=true -> create index -> update required=false + $nullSupported = $database->getAdapter()->getSupportForSpatialIndexNull(); + if ($nullSupported) { + // Should succeed on adapters that allow nullable spatial indexes + $database->updateAttribute($collectionName, 'geom', required: false); + $meta = $database->getCollection($collectionName); + $this->assertEquals(false, $meta->getAttribute('attributes')[0]['required']); + } else { + // Should error (index constraint) when making required=false while spatial index exists + $threw = false; + try { + $database->updateAttribute($collectionName, 'geom', required: false); + } catch (\Throwable $e) { + $threw = true; + } + $this->assertTrue($threw, 'Expected error when setting required=false with existing spatial index and adapter not supporting nullable indexes'); + // Ensure attribute remains required + $meta = $database->getCollection($collectionName); + $this->assertEquals(true, $meta->getAttribute('attributes')[0]['required']); + } + + // 3) Spatial index order support: providing orders should fail if not supported + $orderSupported = $database->getAdapter()->getSupportForSpatialIndexOrder(); + if ($orderSupported) { + $this->assertTrue($database->createIndex($collectionName, 'idx_geom_desc', Database::INDEX_SPATIAL, ['geom'], [], [Database::ORDER_DESC])); + // cleanup + $this->assertTrue($database->deleteIndex($collectionName, 'idx_geom_desc')); + } else { + try { + $database->createIndex($collectionName, 'idx_geom_desc', Database::INDEX_SPATIAL, ['geom'], [], ['DESC']); + $this->fail('Expected error when providing orders for spatial index on adapter without order support'); + } catch (\Throwable $e) { + $this->assertTrue(true); + } + } + } finally { + $database->deleteCollection($collectionName); + } + } } From f2f5ee380f2c3b4d6141f39d85d604b491ab8ddf Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 27 Aug 2025 16:17:22 +0530 Subject: [PATCH 02/69] added default handling for the spatial types --- src/Database/Database.php | 7 +- tests/e2e/Adapter/Scopes/SpatialTests.php | 104 ++++++++++++++++++++++ 2 files changed, 109 insertions(+), 2 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 25ffaf81a..b33edc5f9 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1909,8 +1909,11 @@ protected function validateDefaultTypes(string $type, mixed $default): void } if ($defaultType === 'array') { - foreach ($default as $value) { - $this->validateDefaultTypes($type, $value); + // spatial types require the array itself + if (!in_array($type, Database::SPATIAL_TYPES)) { + foreach ($default as $value) { + $this->validateDefaultTypes($type, $value); + } } return; } diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 97c0fb0a7..63f1b3c49 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -1865,4 +1865,108 @@ public function testUpdateSpatialAttributes(): void $database->deleteCollection($collectionName); } } + + public function testSpatialAttributeDefaults(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + if (!$database->getAdapter()->getSupportForSpatialAttributes()) { + $this->markTestSkipped('Adapter does not support spatial attributes'); + } + + $collectionName = 'spatial_defaults_'; + try { + $database->createCollection($collectionName); + + // Create spatial attributes with defaults and no indexes to avoid nullability/index constraints + $this->assertEquals(true, $database->createAttribute($collectionName, 'pt', Database::VAR_POINT, 0, false, [1.0, 2.0])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'ln', Database::VAR_LINESTRING, 0, false, [[0.0, 0.0], [1.0, 1.0]])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'pg', Database::VAR_POLYGON, 0, false, [[[0.0, 0.0], [0.0, 2.0], [2.0, 2.0], [0.0, 0.0]]])); + + // Create non-spatial attributes (mix of defaults and no defaults) + $this->assertEquals(true, $database->createAttribute($collectionName, 'title', Database::VAR_STRING, 255, false, 'Untitled')); + $this->assertEquals(true, $database->createAttribute($collectionName, 'count', Database::VAR_INTEGER, 0, false, 0)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'rating', Database::VAR_FLOAT, 0, false)); // no default + $this->assertEquals(true, $database->createAttribute($collectionName, 'active', Database::VAR_BOOLEAN, 0, false, true)); + + // Create document without providing spatial values, expect defaults applied + $doc = $database->createDocument($collectionName, new Document([ + '$id' => ID::custom('d1'), + '$permissions' => [Permission::read(Role::any())] + ])); + $this->assertInstanceOf(Document::class, $doc); + $this->assertEquals([1.0, 2.0], $doc->getAttribute('pt')); + $this->assertEquals([[0.0, 0.0], [1.0, 1.0]], $doc->getAttribute('ln')); + $this->assertEquals([[[0.0, 0.0], [0.0, 2.0], [2.0, 2.0], [0.0, 0.0]]], $doc->getAttribute('pg')); + // Non-spatial defaults + $this->assertEquals('Untitled', $doc->getAttribute('title')); + $this->assertEquals(0, $doc->getAttribute('count')); + $this->assertNull($doc->getAttribute('rating')); + $this->assertTrue($doc->getAttribute('active')); + + // Create document overriding defaults + $doc2 = $database->createDocument($collectionName, new Document([ + '$id' => ID::custom('d2'), + '$permissions' => [Permission::read(Role::any())], + 'pt' => [9.0, 9.0], + 'ln' => [[2.0, 2.0], [3.0, 3.0]], + 'pg' => [[[1.0, 1.0], [1.0, 3.0], [3.0, 3.0], [1.0, 1.0]]], + 'title' => 'Custom', + 'count' => 5, + 'rating' => 4.5, + 'active' => false + ])); + $this->assertInstanceOf(Document::class, $doc2); + $this->assertEquals([9.0, 9.0], $doc2->getAttribute('pt')); + $this->assertEquals([[2.0, 2.0], [3.0, 3.0]], $doc2->getAttribute('ln')); + $this->assertEquals([[[1.0, 1.0], [1.0, 3.0], [3.0, 3.0], [1.0, 1.0]]], $doc2->getAttribute('pg')); + $this->assertEquals('Custom', $doc2->getAttribute('title')); + $this->assertEquals(5, $doc2->getAttribute('count')); + $this->assertEquals(4.5, $doc2->getAttribute('rating')); + $this->assertFalse($doc2->getAttribute('active')); + + // Update defaults and ensure they are applied for new documents + $database->updateAttributeDefault($collectionName, 'pt', [5.0, 6.0]); + $database->updateAttributeDefault($collectionName, 'ln', [[10.0, 10.0], [20.0, 20.0]]); + $database->updateAttributeDefault($collectionName, 'pg', [[[5.0, 5.0], [5.0, 7.0], [7.0, 7.0], [5.0, 5.0]]]); + $database->updateAttributeDefault($collectionName, 'title', 'Updated'); + $database->updateAttributeDefault($collectionName, 'count', 10); + $database->updateAttributeDefault($collectionName, 'active', false); + + $doc3 = $database->createDocument($collectionName, new Document([ + '$id' => ID::custom('d3'), + '$permissions' => [Permission::read(Role::any())] + ])); + $this->assertInstanceOf(Document::class, $doc3); + $this->assertEquals([5.0, 6.0], $doc3->getAttribute('pt')); + $this->assertEquals([[10.0, 10.0], [20.0, 20.0]], $doc3->getAttribute('ln')); + $this->assertEquals([[[5.0, 5.0], [5.0, 7.0], [7.0, 7.0], [5.0, 5.0]]], $doc3->getAttribute('pg')); + $this->assertEquals('Updated', $doc3->getAttribute('title')); + $this->assertEquals(10, $doc3->getAttribute('count')); + $this->assertNull($doc3->getAttribute('rating')); + $this->assertFalse($doc3->getAttribute('active')); + + // Invalid defaults should raise errors + try { + $database->updateAttributeDefault($collectionName, 'pt', [[1.0, 2.0]]); // wrong dimensionality + $this->fail('Expected exception for invalid point default shape'); + } catch (\Throwable $e) { + $this->assertTrue(true); + } + try { + $database->updateAttributeDefault($collectionName, 'ln', [1.0, 2.0]); // wrong dimensionality + $this->fail('Expected exception for invalid linestring default shape'); + } catch (\Throwable $e) { + $this->assertTrue(true); + } + try { + $database->updateAttributeDefault($collectionName, 'pg', [[1.0, 2.0]]); // wrong dimensionality + $this->fail('Expected exception for invalid polygon default shape'); + } catch (\Throwable $e) { + $this->assertTrue(true); + } + } finally { + $database->deleteCollection($collectionName); + } + } } From df7902c19cf4a5b271e2c546d4efea52dbe379d4 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 27 Aug 2025 16:31:39 +0530 Subject: [PATCH 03/69] updated index lookup --- src/Database/Database.php | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index b33edc5f9..7240a0a0c 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -2244,19 +2244,29 @@ public function updateAttribute(string $collection, string $id, ?string $type = throw new LimitException('Row width limit reached. Cannot update attribute.'); } - // checking required of attributes in case of spatial types - if (in_array($type, Database::SPATIAL_TYPES) && !$this->adapter->getSupportForSpatialIndexNull()) { + if (in_array($type, self::SPATIAL_TYPES, true) && !$this->adapter->getSupportForSpatialIndexNull()) { $attributeMap = []; - foreach ($attributes as $attribute) { - $key = \strtolower($attribute->getAttribute('key', $attribute->getAttribute('$id'))); - $attributeMap[$key] = $attribute; + foreach ($attributes as $attrDoc) { + $key = \strtolower($attrDoc->getAttribute('key', $attrDoc->getAttribute('$id'))); + $attributeMap[$key] = $attrDoc; } + $indexes = $collectionDoc->getAttribute('indexes', []); foreach ($indexes as $index) { - $attributes = $index->getAttribute('attributes', []); - foreach ($attributes as $attributeName) { - $attribute = $attributeMap[\strtolower($attributeName)]; - if (!$required) { + if ($index->getAttribute('type') !== self::INDEX_SPATIAL) { + continue; + } + $indexAttributes = $index->getAttribute('attributes', []); + foreach ($indexAttributes as $attributeName) { + $lookup = \strtolower($attributeName); + if (!isset($attributeMap[$lookup])) { + continue; + } + $attrDoc = $attributeMap[$lookup]; + $attrType = $attrDoc->getAttribute('type'); + $attrRequired = (bool)$attrDoc->getAttribute('required', false); + + if (in_array($attrType, self::SPATIAL_TYPES, true) && !$attrRequired) { throw new IndexException('Spatial indexes do not allow null values. Mark the attribute "' . $attributeName . '" as required or create the index on a column with no null values.'); } } From af73122718c3f8902ed5569d727df268394959f1 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k <83803257+ArnabChatterjee20k@users.noreply.github.com> Date: Wed, 27 Aug 2025 16:44:56 +0530 Subject: [PATCH 04/69] Update src/Database/Database.php Co-authored-by: Jake Barnby --- src/Database/Database.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 7240a0a0c..0fbdd426e 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1853,7 +1853,7 @@ private function validateAttribute( throw new DatabaseException('Spatial attributes are not supported'); } if (!empty($size)) { - throw new DatabaseException('Size must be empty'); + throw new DatabaseException('Size must be empty for spatial attributes'); } if (!empty($array)) { throw new DatabaseException('Array must be empty'); From 01aac3deca8d75bbbe118f6dfdb17a33c2dbe795 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k <83803257+ArnabChatterjee20k@users.noreply.github.com> Date: Wed, 27 Aug 2025 16:45:31 +0530 Subject: [PATCH 05/69] Update src/Database/Database.php Co-authored-by: Jake Barnby --- src/Database/Database.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 0fbdd426e..bd291866a 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1856,7 +1856,7 @@ private function validateAttribute( throw new DatabaseException('Size must be empty for spatial attributes'); } if (!empty($array)) { - throw new DatabaseException('Array must be empty'); + throw new DatabaseException('Spatial attributes cannot be arrays'); } break; default: From c98b33b69aa4920794a9bdf7de09df12231e37c5 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k <83803257+ArnabChatterjee20k@users.noreply.github.com> Date: Wed, 27 Aug 2025 16:47:04 +0530 Subject: [PATCH 06/69] Update src/Database/Database.php Co-authored-by: Jake Barnby --- src/Database/Database.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index bd291866a..19eed670e 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -2190,10 +2190,10 @@ public function updateAttribute(string $collection, string $id, ?string $type = throw new DatabaseException('Spatial attributes are not supported'); } if (!empty($size)) { - throw new DatabaseException('Size must be empty'); + throw new DatabaseException('Size must be empty for spatial attributes'); } if (!empty($array)) { - throw new DatabaseException('Array must be empty'); + throw new DatabaseException('Spatial attributes cannot be arrays'); } break; default: From 45a1370a884745816b1390c939dc4db50dc4beca Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 27 Aug 2025 23:31:58 +1200 Subject: [PATCH 07/69] Fix nested selections getting empty array when all are removed --- src/Database/Database.php | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 7cec2a28b..78ad960bb 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -7024,7 +7024,12 @@ private function processRelationshipQueries( // 'foo.bar.baz' becomes 'bar.baz' $nestingPath = \implode('.', $nesting); - $nestedSelections[$selectedKey][] = Query::select([$nestingPath]); + // If nestingPath is empty, it means we want all fields (*) for this relationship + if (empty($nestingPath)) { + $nestedSelections[$selectedKey][] = Query::select(['*']); + } else { + $nestedSelections[$selectedKey][] = Query::select([$nestingPath]); + } $type = $relationship->getAttribute('options')['relationType']; $side = $relationship->getAttribute('options')['side']; @@ -7053,7 +7058,13 @@ private function processRelationshipQueries( } } - $query->setValues(\array_values($values)); + $finalValues = \array_values($values); + if ($query->getMethod() === Query::TYPE_SELECT) { + if (empty($finalValues)) { + $finalValues = ['*']; + } + } + $query->setValues($finalValues); } return $nestedSelections; @@ -7086,7 +7097,7 @@ protected function encodeSpatialData(mixed $value, string $type): string case self::VAR_POLYGON: // Check if this is a single ring (flat array of points) or multiple rings $isSingleRing = count($value) > 0 && is_array($value[0]) && - count($value[0]) === 2 && is_numeric($value[0][0]) && is_numeric($value[0][1]); + count($value[0]) === 2 && is_numeric($value[0][0]) && is_numeric($value[0][1]); if ($isSingleRing) { // Convert single ring format [[x1,y1], [x2,y2], ...] to multi-ring format From 48e299be159657c262c0fcfd959c223dc331d3e5 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 31 Aug 2025 15:26:26 +0300 Subject: [PATCH 08/69] with transactions --- src/Database/Adapter.php | 31 ++++++++++++++++++++++++++----- src/Database/Adapter/SQL.php | 20 +++++++------------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 54bf57e63..1f2e8e306 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -4,7 +4,12 @@ use Exception; use Utopia\Database\Exception as DatabaseException; +use Utopia\Database\Exception\Authorization as AuthorizationException; +use Utopia\Database\Exception\Conflict as ConflictException; use Utopia\Database\Exception\Duplicate as DuplicateException; +use Utopia\Database\Exception\Limit as LimitException; +use Utopia\Database\Exception\Relationship as RelationshipException; +use Utopia\Database\Exception\Restricted as RestrictedException; use Utopia\Database\Exception\Timeout as TimeoutException; use Utopia\Database\Exception\Transaction as TransactionException; @@ -371,7 +376,10 @@ public function inTransaction(): bool */ public function withTransaction(callable $callback): mixed { - for ($attempts = 0; $attempts < 3; $attempts++) { + $sleep = 50_000; // 50 milliseconds + $retries = 2; + + for ($attempts = 0; $attempts <= $retries; $attempts++) { try { $this->startTransaction(); $result = $callback(); @@ -380,9 +388,22 @@ public function withTransaction(callable $callback): mixed } catch (\Throwable $action) { try { $this->rollbackTransaction(); + + if ( + $action instanceof DuplicateException || + $action instanceof RestrictedException || + $action instanceof AuthorizationException || + $action instanceof RelationshipException || + $action instanceof ConflictException || + $action instanceof LimitException + ) { + $this->inTransaction = 0; + throw $action; + } + } catch (\Throwable $rollback) { - if ($attempts < 2) { - \usleep(5000); // 5ms + if ($attempts < $retries) { + \usleep($sleep * ($attempts + 1)); continue; } @@ -390,8 +411,8 @@ public function withTransaction(callable $callback): mixed throw $rollback; } - if ($attempts < 2) { - \usleep(5000); // 5ms + if ($attempts < $retries) { + \usleep($sleep * ($attempts + 1)); continue; } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index c81f3d6fb..0c626239f 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -66,19 +66,17 @@ public function startTransaction(): bool $this->getPDO()->prepare('ROLLBACK')->execute(); } - $result = $this->getPDO()->beginTransaction(); + $this->getPDO()->beginTransaction(); + } else { - $result = $this->getPDO()->exec('SAVEPOINT transaction' . $this->inTransaction); + $this->getPDO()->exec('SAVEPOINT transaction' . $this->inTransaction); } } catch (PDOException $e) { throw new TransactionException('Failed to start transaction: ' . $e->getMessage(), $e->getCode(), $e); } - if (!$result) { - throw new TransactionException('Failed to start transaction'); - } - $this->inTransaction++; + return true; } @@ -124,21 +122,17 @@ public function rollbackTransaction(): bool try { if ($this->inTransaction > 1) { - $result = $this->getPDO()->exec('ROLLBACK TO transaction' . ($this->inTransaction - 1)); + $this->getPDO()->exec('ROLLBACK TO transaction' . ($this->inTransaction - 1)); $this->inTransaction--; } else { - $result = $this->getPDO()->rollBack(); + $this->getPDO()->rollBack(); $this->inTransaction = 0; } } catch (PDOException $e) { throw new DatabaseException('Failed to rollback transaction: ' . $e->getMessage(), $e->getCode(), $e); } - if (!$result) { - throw new TransactionException('Failed to rollback transaction'); - } - - return $result; + return true; } /** From 18a2875f09a1f887bbe5b88529791f669c589f2c Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 31 Aug 2025 15:30:14 +0300 Subject: [PATCH 09/69] with transactions --- src/Database/Adapter/SQL.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 0c626239f..a62385275 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -13,7 +13,6 @@ use Utopia\Database\Exception\NotFound as NotFoundException; use Utopia\Database\Exception\Transaction as TransactionException; use Utopia\Database\Query; -use Utopia\Database\Validator\Spatial; abstract class SQL extends Adapter { From 73ffad0f79c4d4779fb59738b5708ab2987ba53f Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 2 Sep 2025 08:30:30 +0300 Subject: [PATCH 10/69] Run tests --- src/Database/Adapter/MariaDB.php | 340 ----------------------------- src/Database/Adapter/Postgres.php | 337 ----------------------------- src/Database/Adapter/SQL.php | 342 ++++++++++++++++++++++++++++++ 3 files changed, 342 insertions(+), 677 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 1badc3966..b9cbad96f 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1354,346 +1354,6 @@ public function deleteDocument(string $collection, string $id): bool return $deleted; } - /** - * Find Documents - * - * @param Document $collection - * @param array $queries - * @param int|null $limit - * @param int|null $offset - * @param array $orderAttributes - * @param array $orderTypes - * @param array $cursor - * @param string $cursorDirection - * @param string $forPermission - * @return array - * @throws DatabaseException - * @throws TimeoutException - * @throws Exception - */ - public function find(Document $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, string $forPermission = Database::PERMISSION_READ): array - { - $spatialAttributes = $this->getSpatialAttributes($collection); - $attributes = $collection->getAttribute('attributes', []); - - $collection = $collection->getId(); - $name = $this->filter($collection); - $roles = Authorization::getRoles(); - $where = []; - $orders = []; - $alias = Query::DEFAULT_ALIAS; - $binds = []; - - $queries = array_map(fn ($query) => clone $query, $queries); - - $cursorWhere = []; - - foreach ($orderAttributes as $i => $originalAttribute) { - $attribute = $this->getInternalKeyForAttribute($originalAttribute); - $attribute = $this->filter($attribute); - - $orderType = $this->filter($orderTypes[$i] ?? Database::ORDER_ASC); - $direction = $orderType; - - if ($cursorDirection === Database::CURSOR_BEFORE) { - $direction = ($direction === Database::ORDER_ASC) - ? Database::ORDER_DESC - : Database::ORDER_ASC; - } - - $orders[] = "{$this->quote($attribute)} {$direction}"; - - // Build pagination WHERE clause only if we have a cursor - if (!empty($cursor)) { - // Special case: No tie breaks. only 1 attribute and it's a unique primary key - if (count($orderAttributes) === 1 && $i === 0 && $originalAttribute === '$sequence') { - $operator = ($direction === Database::ORDER_DESC) - ? Query::TYPE_LESSER - : Query::TYPE_GREATER; - - $bindName = ":cursor_pk"; - $binds[$bindName] = $cursor[$originalAttribute]; - - $cursorWhere[] = "{$this->quote($alias)}.{$this->quote($attribute)} {$this->getSQLOperator($operator)} {$bindName}"; - break; - } - - $conditions = []; - - // Add equality conditions for previous attributes - for ($j = 0; $j < $i; $j++) { - $prevOriginal = $orderAttributes[$j]; - $prevAttr = $this->filter($this->getInternalKeyForAttribute($prevOriginal)); - - $bindName = ":cursor_{$j}"; - $binds[$bindName] = $cursor[$prevOriginal]; - - $conditions[] = "{$this->quote($alias)}.{$this->quote($prevAttr)} = {$bindName}"; - } - - // Add comparison for current attribute - $operator = ($direction === Database::ORDER_DESC) - ? Query::TYPE_LESSER - : Query::TYPE_GREATER; - - $bindName = ":cursor_{$i}"; - $binds[$bindName] = $cursor[$originalAttribute]; - - $conditions[] = "{$this->quote($alias)}.{$this->quote($attribute)} {$this->getSQLOperator($operator)} {$bindName}"; - - $cursorWhere[] = '(' . implode(' AND ', $conditions) . ')'; - } - } - - if (!empty($cursorWhere)) { - $where[] = '(' . implode(' OR ', $cursorWhere) . ')'; - } - - $conditions = $this->getSQLConditions($queries, $binds, attributes:$attributes); - if (!empty($conditions)) { - $where[] = $conditions; - } - - if (Authorization::$status) { - $where[] = $this->getSQLPermissionsCondition($name, $roles, $alias, $forPermission); - } - - if ($this->sharedTables) { - $binds[':_tenant'] = $this->tenant; - $where[] = "{$this->getTenantQuery($collection, $alias, condition: '')}"; - } - - $sqlWhere = !empty($where) ? 'WHERE ' . implode(' AND ', $where) : ''; - $sqlOrder = 'ORDER BY ' . implode(', ', $orders); - - $sqlLimit = ''; - if (! \is_null($limit)) { - $binds[':limit'] = $limit; - $sqlLimit = 'LIMIT :limit'; - } - - if (! \is_null($offset)) { - $binds[':offset'] = $offset; - $sqlLimit .= ' OFFSET :offset'; - } - - $selections = $this->getAttributeSelections($queries); - - - $sql = " - SELECT {$this->getAttributeProjection($selections, $alias, $spatialAttributes)} - FROM {$this->getSQLTable($name)} AS {$this->quote($alias)} - {$sqlWhere} - {$sqlOrder} - {$sqlLimit}; - "; - - $sql = $this->trigger(Database::EVENT_DOCUMENT_FIND, $sql); - - try { - $stmt = $this->getPDO()->prepare($sql); - - foreach ($binds as $key => $value) { - if (gettype($value) === 'double') { - $stmt->bindValue($key, $this->getFloatPrecision($value), PDO::PARAM_STR); - } else { - $stmt->bindValue($key, $value, $this->getPDOType($value)); - } - } - - $stmt->execute(); - } catch (PDOException $e) { - throw $this->processException($e); - } - - $results = $stmt->fetchAll(); - $stmt->closeCursor(); - - foreach ($results as $index => $document) { - if (\array_key_exists('_uid', $document)) { - $results[$index]['$id'] = $document['_uid']; - unset($results[$index]['_uid']); - } - if (\array_key_exists('_id', $document)) { - $results[$index]['$sequence'] = $document['_id']; - unset($results[$index]['_id']); - } - if (\array_key_exists('_tenant', $document)) { - $results[$index]['$tenant'] = $document['_tenant']; - unset($results[$index]['_tenant']); - } - if (\array_key_exists('_createdAt', $document)) { - $results[$index]['$createdAt'] = $document['_createdAt']; - unset($results[$index]['_createdAt']); - } - if (\array_key_exists('_updatedAt', $document)) { - $results[$index]['$updatedAt'] = $document['_updatedAt']; - unset($results[$index]['_updatedAt']); - } - if (\array_key_exists('_permissions', $document)) { - $results[$index]['$permissions'] = \json_decode($document['_permissions'] ?? '[]', true); - unset($results[$index]['_permissions']); - } - - $results[$index] = new Document($results[$index]); - } - - if ($cursorDirection === Database::CURSOR_BEFORE) { - $results = \array_reverse($results); - } - - return $results; - } - - /** - * Count Documents - * - * @param Document $collection - * @param array $queries - * @param int|null $max - * @return int - * @throws Exception - * @throws PDOException - */ - public function count(Document $collection, array $queries = [], ?int $max = null): int - { - $attributes = $collection->getAttribute("attributes", []); - $collection = $collection->getId(); - $name = $this->filter($collection); - $roles = Authorization::getRoles(); - $binds = []; - $where = []; - $alias = Query::DEFAULT_ALIAS; - - $limit = ''; - if (! \is_null($max)) { - $binds[':limit'] = $max; - $limit = 'LIMIT :limit'; - } - - $queries = array_map(fn ($query) => clone $query, $queries); - - $conditions = $this->getSQLConditions($queries, $binds, attributes:$attributes); - if (!empty($conditions)) { - $where[] = $conditions; - } - - if (Authorization::$status) { - $where[] = $this->getSQLPermissionsCondition($name, $roles, $alias); - } - - if ($this->sharedTables) { - $binds[':_tenant'] = $this->tenant; - $where[] = "{$this->getTenantQuery($collection, $alias, condition: '')}"; - } - - $sqlWhere = !empty($where) - ? 'WHERE ' . \implode(' AND ', $where) - : ''; - - $sql = " - SELECT COUNT(1) as sum FROM ( - SELECT 1 - FROM {$this->getSQLTable($name)} AS {$this->quote($alias)} - {$sqlWhere} - {$limit} - ) table_count - "; - - $sql = $this->trigger(Database::EVENT_DOCUMENT_COUNT, $sql); - - $stmt = $this->getPDO()->prepare($sql); - - foreach ($binds as $key => $value) { - $stmt->bindValue($key, $value, $this->getPDOType($value)); - } - - $stmt->execute(); - - $result = $stmt->fetchAll(); - $stmt->closeCursor(); - if (!empty($result)) { - $result = $result[0]; - } - - return $result['sum'] ?? 0; - } - - /** - * Sum an Attribute - * - * @param Document $collection - * @param string $attribute - * @param array $queries - * @param int|null $max - * @return int|float - * @throws Exception - * @throws PDOException - */ - public function sum(Document $collection, string $attribute, array $queries = [], ?int $max = null): int|float - { - $collectionAttributes = $collection->getAttribute("attributes", []); - $collection = $collection->getId(); - $name = $this->filter($collection); - $roles = Authorization::getRoles(); - $where = []; - $alias = Query::DEFAULT_ALIAS; - $binds = []; - - $limit = ''; - if (! \is_null($max)) { - $binds[':limit'] = $max; - $limit = 'LIMIT :limit'; - } - - $queries = array_map(fn ($query) => clone $query, $queries); - - $conditions = $this->getSQLConditions($queries, $binds, attributes:$collectionAttributes); - if (!empty($conditions)) { - $where[] = $conditions; - } - - if (Authorization::$status) { - $where[] = $this->getSQLPermissionsCondition($name, $roles, $alias); - } - - if ($this->sharedTables) { - $binds[':_tenant'] = $this->tenant; - $where[] = "{$this->getTenantQuery($collection, $alias, condition: '')}"; - } - - $sqlWhere = !empty($where) - ? 'WHERE ' . \implode(' AND ', $where) - : ''; - - $sql = " - SELECT SUM({$this->quote($attribute)}) as sum FROM ( - SELECT {$this->quote($attribute)} - FROM {$this->getSQLTable($name)} AS {$this->quote($alias)} - {$sqlWhere} - {$limit} - ) table_count - "; - - $sql = $this->trigger(Database::EVENT_DOCUMENT_SUM, $sql); - - $stmt = $this->getPDO()->prepare($sql); - - foreach ($binds as $key => $value) { - $stmt->bindValue($key, $value, $this->getPDOType($value)); - } - - $stmt->execute(); - - $result = $stmt->fetchAll(); - $stmt->closeCursor(); - if (!empty($result)) { - $result = $result[0]; - } - - return $result['sum'] ?? 0; - } - /** * Handle spatial queries * diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 13583bb1b..4b11cc61d 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1444,343 +1444,6 @@ public function deleteDocument(string $collection, string $id): bool return $deleted; } - /** - * Find Documents - * - * @param Document $collection - * @param array $queries - * @param int|null $limit - * @param int|null $offset - * @param array $orderAttributes - * @param array $orderTypes - * @param array $cursor - * @param string $cursorDirection - * @param string $forPermission - * @return array - * @throws DatabaseException - * @throws TimeoutException - * @throws Exception - */ - public function find(Document $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, string $forPermission = Database::PERMISSION_READ): array - { - $spatialAttributes = $this->getSpatialAttributes($collection); - $attributes = $collection->getAttribute('attributes', []); - $collection = $collection->getId(); - $name = $this->filter($collection); - $roles = Authorization::getRoles(); - $where = []; - $orders = []; - $alias = Query::DEFAULT_ALIAS; - $binds = []; - - $queries = array_map(fn ($query) => clone $query, $queries); - - $cursorWhere = []; - - foreach ($orderAttributes as $i => $originalAttribute) { - $attribute = $this->getInternalKeyForAttribute($originalAttribute); - $attribute = $this->filter($attribute); - - $orderType = $this->filter($orderTypes[$i] ?? Database::ORDER_ASC); - $direction = $orderType; - - if ($cursorDirection === Database::CURSOR_BEFORE) { - $direction = ($direction === Database::ORDER_ASC) - ? Database::ORDER_DESC - : Database::ORDER_ASC; - } - - $orders[] = "{$this->quote($attribute)} {$direction}"; - - // Build pagination WHERE clause only if we have a cursor - if (!empty($cursor)) { - // Special case: only 1 attribute and it's a unique primary key - if (count($orderAttributes) === 1 && $i === 0 && $originalAttribute === '$sequence') { - $operator = ($direction === Database::ORDER_DESC) - ? Query::TYPE_LESSER - : Query::TYPE_GREATER; - - $bindName = ":cursor_pk"; - $binds[$bindName] = $cursor[$originalAttribute]; - - $cursorWhere[] = "{$this->quote($alias)}.{$this->quote($attribute)} {$this->getSQLOperator($operator)} {$bindName}"; - break; - } - - $conditions = []; - - // Add equality conditions for previous attributes - for ($j = 0; $j < $i; $j++) { - $prevOriginal = $orderAttributes[$j]; - $prevAttr = $this->filter($this->getInternalKeyForAttribute($prevOriginal)); - - $bindName = ":cursor_{$j}"; - $binds[$bindName] = $cursor[$prevOriginal]; - - $conditions[] = "{$this->quote($alias)}.{$this->quote($prevAttr)} = {$bindName}"; - } - - // Add comparison for current attribute - $operator = ($direction === Database::ORDER_DESC) - ? Query::TYPE_LESSER - : Query::TYPE_GREATER; - - $bindName = ":cursor_{$i}"; - $binds[$bindName] = $cursor[$originalAttribute]; - - $conditions[] = "{$this->quote($alias)}.{$this->quote($attribute)} {$this->getSQLOperator($operator)} {$bindName}"; - - $cursorWhere[] = '(' . implode(' AND ', $conditions) . ')'; - } - } - - if (!empty($cursorWhere)) { - $where[] = '(' . implode(' OR ', $cursorWhere) . ')'; - } - - $conditions = $this->getSQLConditions($queries, $binds, attributes:$attributes); - if (!empty($conditions)) { - $where[] = $conditions; - } - - if (Authorization::$status) { - $where[] = $this->getSQLPermissionsCondition($name, $roles, $alias, $forPermission); - } - - if ($this->sharedTables) { - $binds[':_tenant'] = $this->tenant; - $where[] = "{$this->getTenantQuery($collection, $alias, condition: '')}"; - } - - $sqlWhere = !empty($where) ? 'WHERE ' . implode(' AND ', $where) : ''; - $sqlOrder = 'ORDER BY ' . implode(', ', $orders); - - $sqlLimit = ''; - if (! \is_null($limit)) { - $binds[':limit'] = $limit; - $sqlLimit = 'LIMIT :limit'; - } - - if (! \is_null($offset)) { - $binds[':offset'] = $offset; - $sqlLimit .= ' OFFSET :offset'; - } - - $selections = $this->getAttributeSelections($queries); - - $sql = " - SELECT {$this->getAttributeProjection($selections, $alias, $spatialAttributes)} - FROM {$this->getSQLTable($name)} AS {$this->quote($alias)} - {$sqlWhere} - {$sqlOrder} - {$sqlLimit}; - "; - - $sql = $this->trigger(Database::EVENT_DOCUMENT_FIND, $sql); - - try { - $stmt = $this->getPDO()->prepare($sql); - - foreach ($binds as $key => $value) { - if (gettype($value) === 'double') { - $stmt->bindValue($key, $this->getFloatPrecision($value), PDO::PARAM_STR); - } else { - $stmt->bindValue($key, $value, $this->getPDOType($value)); - } - } - $this->execute($stmt); - } catch (PDOException $e) { - throw $this->processException($e); - } - - $results = $stmt->fetchAll(); - $stmt->closeCursor(); - - foreach ($results as $index => $document) { - if (\array_key_exists('_uid', $document)) { - $results[$index]['$id'] = $document['_uid']; - unset($results[$index]['_uid']); - } - if (\array_key_exists('_id', $document)) { - $results[$index]['$sequence'] = $document['_id']; - unset($results[$index]['_id']); - } - if (\array_key_exists('_tenant', $document)) { - $results[$index]['$tenant'] = $document['_tenant']; - unset($results[$index]['_tenant']); - } - if (\array_key_exists('_createdAt', $document)) { - $results[$index]['$createdAt'] = $document['_createdAt']; - unset($results[$index]['_createdAt']); - } - if (\array_key_exists('_updatedAt', $document)) { - $results[$index]['$updatedAt'] = $document['_updatedAt']; - unset($results[$index]['_updatedAt']); - } - if (\array_key_exists('_permissions', $document)) { - $results[$index]['$permissions'] = \json_decode($document['_permissions'] ?? '[]', true); - unset($results[$index]['_permissions']); - } - - $results[$index] = new Document($results[$index]); - } - - if ($cursorDirection === Database::CURSOR_BEFORE) { - $results = \array_reverse($results); - } - - return $results; - } - - /** - * Count Documents - * @param Document $collection - * @param array $queries - * @param int|null $max - * @return int - * @throws Exception - * @throws PDOException - */ - public function count(Document $collection, array $queries = [], ?int $max = null): int - { - $collectionAttributes = $collection->getAttribute("attributes", []); - $collection = $collection->getId(); - $name = $this->filter($collection); - $roles = Authorization::getRoles(); - $binds = []; - $where = []; - $alias = Query::DEFAULT_ALIAS; - - $limit = ''; - if (! \is_null($max)) { - $binds[':limit'] = $max; - $limit = 'LIMIT :limit'; - } - - $queries = array_map(fn ($query) => clone $query, $queries); - - $conditions = $this->getSQLConditions($queries, $binds, attributes:$collectionAttributes); - if (!empty($conditions)) { - $where[] = $conditions; - } - - if (Authorization::$status) { - $where[] = $this->getSQLPermissionsCondition($name, $roles, $alias); - } - - if ($this->sharedTables) { - $binds[':_tenant'] = $this->tenant; - $where[] = "{$this->getTenantQuery($collection, $alias, condition: '')}"; - } - - $sqlWhere = !empty($where) - ? 'WHERE ' . \implode(' AND ', $where) - : ''; - - $sql = " - SELECT COUNT(1) as sum FROM ( - SELECT 1 - FROM {$this->getSQLTable($name)} AS {$this->quote($alias)} - {$sqlWhere} - {$limit} - ) table_count - "; - - - $sql = $this->trigger(Database::EVENT_DOCUMENT_COUNT, $sql); - - $stmt = $this->getPDO()->prepare($sql); - - foreach ($binds as $key => $value) { - $stmt->bindValue($key, $value, $this->getPDOType($value)); - } - - $this->execute($stmt); - - $result = $stmt->fetchAll(); - $stmt->closeCursor(); - if (!empty($result)) { - $result = $result[0]; - } - - return $result['sum'] ?? 0; - } - - /** - * Sum an Attribute - * - * @param Document $collection - * @param string $attribute - * @param array $queries - * @param int|null $max - * @return int|float - * @throws Exception - * @throws PDOException - */ - public function sum(Document $collection, string $attribute, array $queries = [], ?int $max = null): int|float - { - $collectionAttributes = $collection->getAttribute("attributes", []); - $collection = $collection->getId(); - $name = $this->filter($collection); - $roles = Authorization::getRoles(); - $where = []; - $alias = Query::DEFAULT_ALIAS; - $binds = []; - - $limit = ''; - if (! \is_null($max)) { - $binds[':limit'] = $max; - $limit = 'LIMIT :limit'; - } - - $queries = array_map(fn ($query) => clone $query, $queries); - - $conditions = $this->getSQLConditions($queries, $binds, attributes:$collectionAttributes); - if (!empty($conditions)) { - $where[] = $conditions; - } - - if (Authorization::$status) { - $where[] = $this->getSQLPermissionsCondition($name, $roles, $alias); - } - - if ($this->sharedTables) { - $binds[':_tenant'] = $this->tenant; - $where[] = "{$this->getTenantQuery($collection, $alias, condition: '')}"; - } - - $sqlWhere = !empty($where) - ? 'WHERE ' . \implode(' AND ', $where) - : ''; - - $sql = " - SELECT SUM({$this->quote($attribute)}) as sum FROM ( - SELECT {$this->quote($attribute)} - FROM {$this->getSQLTable($name)} AS {$this->quote($alias)} - {$sqlWhere} - {$limit} - ) table_count - "; - - $sql = $this->trigger(Database::EVENT_DOCUMENT_SUM, $sql); - - $stmt = $this->getPDO()->prepare($sql); - - foreach ($binds as $key => $value) { - $stmt->bindValue($key, $value, $this->getPDOType($value)); - } - - $this->execute($stmt); - - $result = $stmt->fetchAll(); - $stmt->closeCursor(); - if (!empty($result)) { - $result = $result[0]; - } - - return $result['sum'] ?? 0; - } - /** * @return string */ diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index a62385275..252dd5fe1 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -11,8 +11,10 @@ use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\NotFound as NotFoundException; +use Utopia\Database\Exception\Timeout as TimeoutException; use Utopia\Database\Exception\Transaction as TransactionException; use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; abstract class SQL extends Adapter { @@ -2314,4 +2316,344 @@ protected function getAttributeType(string $attributeName, array $attributes): ? } return null; } + + /** + * Find Documents + * + * @param Document $collection + * @param array $queries + * @param int|null $limit + * @param int|null $offset + * @param array $orderAttributes + * @param array $orderTypes + * @param array $cursor + * @param string $cursorDirection + * @param string $forPermission + * @return array + * @throws DatabaseException + * @throws TimeoutException + * @throws Exception + */ + public function find(Document $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, string $forPermission = Database::PERMISSION_READ): array + { + $spatialAttributes = $this->getSpatialAttributes($collection); + $attributes = $collection->getAttribute('attributes', []); + + $collection = $collection->getId(); + $name = $this->filter($collection); + $roles = Authorization::getRoles(); + $where = []; + $orders = []; + $alias = Query::DEFAULT_ALIAS; + $binds = []; + + $queries = array_map(fn ($query) => clone $query, $queries); + + $cursorWhere = []; + + foreach ($orderAttributes as $i => $originalAttribute) { + $attribute = $this->getInternalKeyForAttribute($originalAttribute); + $attribute = $this->filter($attribute); + + $orderType = $this->filter($orderTypes[$i] ?? Database::ORDER_ASC); + $direction = $orderType; + + if ($cursorDirection === Database::CURSOR_BEFORE) { + $direction = ($direction === Database::ORDER_ASC) + ? Database::ORDER_DESC + : Database::ORDER_ASC; + } + + $orders[] = "{$this->quote($attribute)} {$direction}"; + + // Build pagination WHERE clause only if we have a cursor + if (!empty($cursor)) { + // Special case: No tie breaks. only 1 attribute and it's a unique primary key + if (count($orderAttributes) === 1 && $i === 0 && $originalAttribute === '$sequence') { + $operator = ($direction === Database::ORDER_DESC) + ? Query::TYPE_LESSER + : Query::TYPE_GREATER; + + $bindName = ":cursor_pk"; + $binds[$bindName] = $cursor[$originalAttribute]; + + $cursorWhere[] = "{$this->quote($alias)}.{$this->quote($attribute)} {$this->getSQLOperator($operator)} {$bindName}"; + break; + } + + $conditions = []; + + // Add equality conditions for previous attributes + for ($j = 0; $j < $i; $j++) { + $prevOriginal = $orderAttributes[$j]; + $prevAttr = $this->filter($this->getInternalKeyForAttribute($prevOriginal)); + + $bindName = ":cursor_{$j}"; + $binds[$bindName] = $cursor[$prevOriginal]; + + $conditions[] = "{$this->quote($alias)}.{$this->quote($prevAttr)} = {$bindName}"; + } + + // Add comparison for current attribute + $operator = ($direction === Database::ORDER_DESC) + ? Query::TYPE_LESSER + : Query::TYPE_GREATER; + + $bindName = ":cursor_{$i}"; + $binds[$bindName] = $cursor[$originalAttribute]; + + $conditions[] = "{$this->quote($alias)}.{$this->quote($attribute)} {$this->getSQLOperator($operator)} {$bindName}"; + + $cursorWhere[] = '(' . implode(' AND ', $conditions) . ')'; + } + } + + if (!empty($cursorWhere)) { + $where[] = '(' . implode(' OR ', $cursorWhere) . ')'; + } + + $conditions = $this->getSQLConditions($queries, $binds, attributes:$attributes); + if (!empty($conditions)) { + $where[] = $conditions; + } + + if (Authorization::$status) { + $where[] = $this->getSQLPermissionsCondition($name, $roles, $alias, $forPermission); + } + + if ($this->sharedTables) { + $binds[':_tenant'] = $this->tenant; + $where[] = "{$this->getTenantQuery($collection, $alias, condition: '')}"; + } + + $sqlWhere = !empty($where) ? 'WHERE ' . implode(' AND ', $where) : ''; + $sqlOrder = 'ORDER BY ' . implode(', ', $orders); + + $sqlLimit = ''; + if (! \is_null($limit)) { + $binds[':limit'] = $limit; + $sqlLimit = 'LIMIT :limit'; + } + + if (! \is_null($offset)) { + $binds[':offset'] = $offset; + $sqlLimit .= ' OFFSET :offset'; + } + + $selections = $this->getAttributeSelections($queries); + + + $sql = " + SELECT {$this->getAttributeProjection($selections, $alias, $spatialAttributes)} + FROM {$this->getSQLTable($name)} AS {$this->quote($alias)} + {$sqlWhere} + {$sqlOrder} + {$sqlLimit}; + "; + + $sql = $this->trigger(Database::EVENT_DOCUMENT_FIND, $sql); + + try { + $stmt = $this->getPDO()->prepare($sql); + + foreach ($binds as $key => $value) { + if (gettype($value) === 'double') { + $stmt->bindValue($key, $this->getFloatPrecision($value), PDO::PARAM_STR); + } else { + $stmt->bindValue($key, $value, $this->getPDOType($value)); + } + } + + $stmt->execute(); + } catch (PDOException $e) { + throw $this->processException($e); + } + + $results = $stmt->fetchAll(); + $stmt->closeCursor(); + + foreach ($results as $index => $document) { + if (\array_key_exists('_uid', $document)) { + $results[$index]['$id'] = $document['_uid']; + unset($results[$index]['_uid']); + } + if (\array_key_exists('_id', $document)) { + $results[$index]['$sequence'] = $document['_id']; + unset($results[$index]['_id']); + } + if (\array_key_exists('_tenant', $document)) { + $results[$index]['$tenant'] = $document['_tenant']; + unset($results[$index]['_tenant']); + } + if (\array_key_exists('_createdAt', $document)) { + $results[$index]['$createdAt'] = $document['_createdAt']; + unset($results[$index]['_createdAt']); + } + if (\array_key_exists('_updatedAt', $document)) { + $results[$index]['$updatedAt'] = $document['_updatedAt']; + unset($results[$index]['_updatedAt']); + } + if (\array_key_exists('_permissions', $document)) { + $results[$index]['$permissions'] = \json_decode($document['_permissions'] ?? '[]', true); + unset($results[$index]['_permissions']); + } + + $results[$index] = new Document($results[$index]); + } + + if ($cursorDirection === Database::CURSOR_BEFORE) { + $results = \array_reverse($results); + } + + return $results; + } + + /** + * Count Documents + * + * @param Document $collection + * @param array $queries + * @param int|null $max + * @return int + * @throws Exception + * @throws PDOException + */ + public function count(Document $collection, array $queries = [], ?int $max = null): int + { + $attributes = $collection->getAttribute("attributes", []); + $collection = $collection->getId(); + $name = $this->filter($collection); + $roles = Authorization::getRoles(); + $binds = []; + $where = []; + $alias = Query::DEFAULT_ALIAS; + + $limit = ''; + if (! \is_null($max)) { + $binds[':limit'] = $max; + $limit = 'LIMIT :limit'; + } + + $queries = array_map(fn ($query) => clone $query, $queries); + + $conditions = $this->getSQLConditions($queries, $binds, attributes:$attributes); + if (!empty($conditions)) { + $where[] = $conditions; + } + + if (Authorization::$status) { + $where[] = $this->getSQLPermissionsCondition($name, $roles, $alias); + } + + if ($this->sharedTables) { + $binds[':_tenant'] = $this->tenant; + $where[] = "{$this->getTenantQuery($collection, $alias, condition: '')}"; + } + + $sqlWhere = !empty($where) + ? 'WHERE ' . \implode(' AND ', $where) + : ''; + + $sql = " + SELECT COUNT(1) as sum FROM ( + SELECT 1 + FROM {$this->getSQLTable($name)} AS {$this->quote($alias)} + {$sqlWhere} + {$limit} + ) table_count + "; + + $sql = $this->trigger(Database::EVENT_DOCUMENT_COUNT, $sql); + + $stmt = $this->getPDO()->prepare($sql); + + foreach ($binds as $key => $value) { + $stmt->bindValue($key, $value, $this->getPDOType($value)); + } + + $stmt->execute(); + + $result = $stmt->fetchAll(); + $stmt->closeCursor(); + if (!empty($result)) { + $result = $result[0]; + } + + return $result['sum'] ?? 0; + } + + /** + * Sum an Attribute + * + * @param Document $collection + * @param string $attribute + * @param array $queries + * @param int|null $max + * @return int|float + * @throws Exception + * @throws PDOException + */ + public function sum(Document $collection, string $attribute, array $queries = [], ?int $max = null): int|float + { + $collectionAttributes = $collection->getAttribute("attributes", []); + $collection = $collection->getId(); + $name = $this->filter($collection); + $roles = Authorization::getRoles(); + $where = []; + $alias = Query::DEFAULT_ALIAS; + $binds = []; + + $limit = ''; + if (! \is_null($max)) { + $binds[':limit'] = $max; + $limit = 'LIMIT :limit'; + } + + $queries = array_map(fn ($query) => clone $query, $queries); + + $conditions = $this->getSQLConditions($queries, $binds, attributes:$collectionAttributes); + if (!empty($conditions)) { + $where[] = $conditions; + } + + if (Authorization::$status) { + $where[] = $this->getSQLPermissionsCondition($name, $roles, $alias); + } + + if ($this->sharedTables) { + $binds[':_tenant'] = $this->tenant; + $where[] = "{$this->getTenantQuery($collection, $alias, condition: '')}"; + } + + $sqlWhere = !empty($where) + ? 'WHERE ' . \implode(' AND ', $where) + : ''; + + $sql = " + SELECT SUM({$this->quote($attribute)}) as sum FROM ( + SELECT {$this->quote($attribute)} + FROM {$this->getSQLTable($name)} AS {$this->quote($alias)} + {$sqlWhere} + {$limit} + ) table_count + "; + + $sql = $this->trigger(Database::EVENT_DOCUMENT_SUM, $sql); + + $stmt = $this->getPDO()->prepare($sql); + + foreach ($binds as $key => $value) { + $stmt->bindValue($key, $value, $this->getPDOType($value)); + } + + $stmt->execute(); + + $result = $stmt->fetchAll(); + $stmt->closeCursor(); + if (!empty($result)) { + $result = $result[0]; + } + + return $result['sum'] ?? 0; + } } From 5f2f46ae6d10b3b2d2205c308e5e15a6d88a6622 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 2 Sep 2025 08:37:10 +0300 Subject: [PATCH 11/69] PDO namespace --- src/Database/Adapter/MariaDB.php | 8 +++----- src/Database/Adapter/SQL.php | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index b9cbad96f..0e1cb09b4 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -3,7 +3,6 @@ namespace Utopia\Database\Adapter; use Exception; -use PDO; use PDOException; use Utopia\Database\Database; use Utopia\Database\Document; @@ -14,7 +13,6 @@ use Utopia\Database\Exception\Truncate as TruncateException; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; -use Utopia\Database\Validator\Authorization; class MariaDB extends SQL { @@ -1634,9 +1632,9 @@ protected function getSQLType(string $type, int $size, bool $signed = true, bool protected function getPDOType(mixed $value): int { return match (gettype($value)) { - 'string','double' => PDO::PARAM_STR, - 'integer', 'boolean' => PDO::PARAM_INT, - 'NULL' => PDO::PARAM_NULL, + 'string','double' => \PDO::PARAM_STR, + 'integer', 'boolean' => \PDO::PARAM_INT, + 'NULL' => \PDO::PARAM_NULL, default => throw new DatabaseException('Unknown PDO Type for ' . \gettype($value)), }; } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 252dd5fe1..4fb3d5795 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2458,7 +2458,7 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 foreach ($binds as $key => $value) { if (gettype($value) === 'double') { - $stmt->bindValue($key, $this->getFloatPrecision($value), PDO::PARAM_STR); + $stmt->bindValue($key, $this->getFloatPrecision($value), \PDO::PARAM_STR); } else { $stmt->bindValue($key, $value, $this->getPDOType($value)); } From 41440aee7935c218084c11fa83c5fa5b04b66466 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 2 Sep 2025 08:38:14 +0300 Subject: [PATCH 12/69] lint --- src/Database/Adapter/Postgres.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 4b11cc61d..ab44bead1 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -15,7 +15,6 @@ use Utopia\Database\Exception\Truncate as TruncateException; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; -use Utopia\Database\Validator\Authorization; class Postgres extends SQL { From e4f66931ccd8ebdef84ac3d34aaef0cbd28adcd8 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 2 Sep 2025 08:41:51 +0300 Subject: [PATCH 13/69] Use abstract $this->execute($stmt); --- src/Database/Adapter/SQL.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 4fb3d5795..91b53a9e6 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2464,7 +2464,7 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 } } - $stmt->execute(); + $this->execute($stmt); } catch (PDOException $e) { throw $this->processException($e); } @@ -2571,7 +2571,7 @@ public function count(Document $collection, array $queries = [], ?int $max = nul $stmt->bindValue($key, $value, $this->getPDOType($value)); } - $stmt->execute(); + $this->execute($stmt); $result = $stmt->fetchAll(); $stmt->closeCursor(); @@ -2646,7 +2646,7 @@ public function sum(Document $collection, string $attribute, array $queries = [] $stmt->bindValue($key, $value, $this->getPDOType($value)); } - $stmt->execute(); + $this->execute($stmt); $result = $stmt->fetchAll(); $stmt->closeCursor(); From c2fe9b0b049369a748f7dc299c6b0671ab03376a Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 2 Sep 2025 09:00:44 +0300 Subject: [PATCH 14/69] Filter attribute --- src/Database/Adapter/SQL.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 91b53a9e6..cc241c445 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2598,6 +2598,7 @@ public function sum(Document $collection, string $attribute, array $queries = [] $collectionAttributes = $collection->getAttribute("attributes", []); $collection = $collection->getId(); $name = $this->filter($collection); + $attribute = $this->filter($attribute); $roles = Authorization::getRoles(); $where = []; $alias = Query::DEFAULT_ALIAS; From 3a4edc029a807f430ae5a93485e130dd5f028311 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 2 Sep 2025 18:49:47 +0530 Subject: [PATCH 15/69] fixed structure exception not getting raised for spatial types --- src/Database/Database.php | 4 +- src/Database/Validator/Spatial.php | 129 ++++++++-------------- tests/e2e/Adapter/Scopes/SpatialTests.php | 99 +++++++++++++++++ 3 files changed, 145 insertions(+), 87 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index aa87daa0f..8f1d4ba5d 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -7133,7 +7133,9 @@ private function processRelationshipQueries( protected function encodeSpatialData(mixed $value, string $type): string { $validator = new Spatial($type); - $validator->isValid($value); + if (!$validator->isValid($value)) { + throw new StructureException($validator->getDescription()); + } switch ($type) { case self::VAR_POINT: diff --git a/src/Database/Validator/Spatial.php b/src/Database/Validator/Spatial.php index 30026dfe2..06a62e7ae 100644 --- a/src/Database/Validator/Spatial.php +++ b/src/Database/Validator/Spatial.php @@ -3,62 +3,31 @@ namespace Utopia\Database\Validator; use Utopia\Database\Database; -use Utopia\Database\Exception; use Utopia\Validator; class Spatial extends Validator { private string $spatialType; + protected string $message = ''; public function __construct(string $spatialType) { $this->spatialType = $spatialType; } - /** - * Validate spatial data according to its type - * - * @param mixed $value - * @param string $type - * @return bool - * @throws Exception - */ - public static function validate(mixed $value, string $type): bool - { - if (!is_array($value)) { - throw new Exception('Spatial data must be provided as an array'); - } - - switch ($type) { - case Database::VAR_POINT: - return self::validatePoint($value); - - case Database::VAR_LINESTRING: - return self::validateLineString($value); - - case Database::VAR_POLYGON: - return self::validatePolygon($value); - - default: - throw new Exception('Unknown spatial type: ' . $type); - } - } - /** * Validate POINT data - * - * @param array $value - * @return bool - * @throws Exception */ - protected static function validatePoint(array $value): bool + protected function validatePoint(array $value): bool { if (count($value) !== 2) { - throw new Exception('Point must be an array of two numeric values [x, y]'); + $this->message = 'Point must be an array of two numeric values [x, y]'; + return false; } if (!is_numeric($value[0]) || !is_numeric($value[1])) { - throw new Exception('Point coordinates must be numeric values'); + $this->message = 'Point coordinates must be numeric values'; + return false; } return true; @@ -66,24 +35,23 @@ protected static function validatePoint(array $value): bool /** * Validate LINESTRING data - * - * @param array $value - * @return bool - * @throws Exception */ - protected static function validateLineString(array $value): bool + protected function validateLineString(array $value): bool { if (count($value) < 2) { - throw new Exception('LineString must contain at least one point'); + $this->message = 'LineString must contain at least two points'; + return false; } foreach ($value as $point) { if (!is_array($point) || count($point) !== 2) { - throw new Exception('Each point in LineString must be an array of two values [x, y]'); + $this->message = 'Each point in LineString must be an array of two values [x, y]'; + return false; } if (!is_numeric($point[0]) || !is_numeric($point[1])) { - throw new Exception('Each point in LineString must have numeric coordinates'); + $this->message = 'Each point in LineString must have numeric coordinates'; + return false; } } @@ -92,36 +60,39 @@ protected static function validateLineString(array $value): bool /** * Validate POLYGON data - * - * @param array $value - * @return bool - * @throws Exception */ - protected static function validatePolygon(array $value): bool + protected function validatePolygon(array $value): bool { if (empty($value)) { - throw new Exception('Polygon must contain at least one ring'); + $this->message = 'Polygon must contain at least one ring'; + return false; } // Detect single-ring polygon: [[x, y], [x, y], ...] $isSingleRing = isset($value[0]) && is_array($value[0]) && - count($value[0]) === 2 && is_numeric($value[0][0]) && is_numeric($value[0][1]); + count($value[0]) === 2 && + is_numeric($value[0][0]) && + is_numeric($value[0][1]); if ($isSingleRing) { - $value = [$value]; // Wrap single ring into multi-ring format + $value = [$value]; // wrap single ring } foreach ($value as $ring) { if (!is_array($ring) || empty($ring)) { - throw new Exception('Each ring in Polygon must be an array of points'); + $this->message = 'Each ring in Polygon must be an array of points'; + return false; } foreach ($ring as $point) { if (!is_array($point) || count($point) !== 2) { - throw new Exception('Each point in Polygon ring must be an array of two values [x, y]'); + $this->message = 'Each point in Polygon ring must be an array of two values [x, y]'; + return false; } + if (!is_numeric($point[0]) || !is_numeric($point[1])) { - throw new Exception('Each point in Polygon ring must have numeric coordinates'); + $this->message = 'Each point in Polygon ring must have numeric coordinates'; + return false; } } } @@ -131,9 +102,6 @@ protected static function validatePolygon(array $value): bool /** * Check if a value is valid WKT string - * - * @param string $value - * @return bool */ public static function isWKTString(string $value): bool { @@ -141,41 +109,23 @@ public static function isWKTString(string $value): bool return (bool) preg_match('/^(POINT|LINESTRING|POLYGON)\s*\(/i', $value); } - /** - * Get validator description - * - * @return string - */ public function getDescription(): string { - return 'Value must be a valid ' . $this->spatialType . ' format (array or WKT string)'; + return 'Value must be a valid ' . $this->spatialType . ": {$this->message}"; } - /** - * Is array - * - * @return bool - */ public function isArray(): bool { return false; } - /** - * Get Type - * - * @return string - */ public function getType(): string { return 'spatial'; } /** - * Is valid - * - * @param mixed $value - * @return bool + * Main validation entrypoint */ public function isValid($value): bool { @@ -184,20 +134,27 @@ public function isValid($value): bool } if (is_string($value)) { - // Check if it's a valid WKT string return self::isWKTString($value); } if (is_array($value)) { - // Validate the array format according to the specific spatial type - try { - self::validate($value, $this->spatialType); - return true; - } catch (\Exception $e) { - return false; + switch ($this->spatialType) { + case Database::VAR_POINT: + return $this->validatePoint($value); + + case Database::VAR_LINESTRING: + return $this->validateLineString($value); + + case Database::VAR_POLYGON: + return $this->validatePolygon($value); + + default: + $this->message = 'Unknown spatial type: ' . $this->spatialType; + return false; } } + $this->message = 'Spatial value must be array or WKT string'; return false; } } diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 63f1b3c49..4af2afa62 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -5,6 +5,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception; +use Utopia\Database\Exception\Structure as StructureException; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; @@ -1552,6 +1553,19 @@ public function testSpatialBulkOperation(): void $updateResults[] = $doc; }); + // should fail due to invalid structure + try { + $database->updateDocuments($collectionName, new Document([ + 'name' => 'Updated Location', + 'location' => [15.0, 25.0], + 'area' => [15.0, 25.0] // invalid polygon + ])); + $this->fail("fail to throw structure exception for the invalid spatial type"); + } catch (\Throwable $th) { + $this->assertInstanceOf(StructureException::class, $th); + + } + $this->assertGreaterThan(0, $updateCount); // Verify updated documents @@ -1969,4 +1983,89 @@ public function testSpatialAttributeDefaults(): void $database->deleteCollection($collectionName); } } + + public function testInvalidSpatialTypes(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + if (!$database->getAdapter()->getSupportForSpatialAttributes()) { + $this->markTestSkipped('Adapter does not support spatial attributes'); + } + + $collectionName = 'test_invalid_spatial_types'; + + $attributes = [ + new Document([ + '$id' => ID::custom('pointAttr'), + 'type' => Database::VAR_POINT, + 'size' => 0, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => [], + ]), + new Document([ + '$id' => ID::custom('lineAttr'), + 'type' => Database::VAR_LINESTRING, + 'size' => 0, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => [], + ]), + new Document([ + '$id' => ID::custom('polyAttr'), + 'type' => Database::VAR_POLYGON, + 'size' => 0, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => [], + ]) + ]; + + $database->createCollection($collectionName, $attributes); + + // ❌ Invalid Point (must be [x, y]) + try { + $database->createDocument($collectionName, new Document([ + 'pointAttr' => [10.0], // only 1 coordinate + ])); + $this->fail("Expected StructureException for invalid point"); + } catch (\Throwable $th) { + $this->assertInstanceOf(StructureException::class, $th); + } + + // ❌ Invalid LineString (must be [[x,y],[x,y],...], at least 2 points) + try { + $database->createDocument($collectionName, new Document([ + 'lineAttr' => [[10.0, 20.0]], // only one point + ])); + $this->fail("Expected StructureException for invalid line"); + } catch (\Throwable $th) { + $this->assertInstanceOf(StructureException::class, $th); + } + + try { + $database->createDocument($collectionName, new Document([ + 'lineAttr' => [10.0, 20.0], // not an array of arrays + ])); + $this->fail("Expected StructureException for invalid line structure"); + } catch (\Throwable $th) { + $this->assertInstanceOf(StructureException::class, $th); + } + + try { + $database->createDocument($collectionName, new Document([ + 'polyAttr' => [10.0, 20.0] // not an array of arrays + ])); + $this->fail("Expected StructureException for invalid polygon structure"); + } catch (\Throwable $th) { + $this->assertInstanceOf(StructureException::class, $th); + } + + // Cleanup + $database->deleteCollection($collectionName); + } + } From 6746472eac9698137686b28bed3d0c28e1f8d568 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 2 Sep 2025 19:00:42 +0530 Subject: [PATCH 16/69] updated phpdocs --- src/Database/Validator/Spatial.php | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Database/Validator/Spatial.php b/src/Database/Validator/Spatial.php index 06a62e7ae..9843fe38c 100644 --- a/src/Database/Validator/Spatial.php +++ b/src/Database/Validator/Spatial.php @@ -17,6 +17,9 @@ public function __construct(string $spatialType) /** * Validate POINT data + * + * @param array $value + * @return bool */ protected function validatePoint(array $value): bool { @@ -25,16 +28,14 @@ protected function validatePoint(array $value): bool return false; } - if (!is_numeric($value[0]) || !is_numeric($value[1])) { - $this->message = 'Point coordinates must be numeric values'; - return false; - } - return true; } /** * Validate LINESTRING data + * + * @param array $value + * @return bool */ protected function validateLineString(array $value): bool { @@ -60,6 +61,9 @@ protected function validateLineString(array $value): bool /** * Validate POLYGON data + * + * @param array $value + * @return bool */ protected function validatePolygon(array $value): bool { From 21770b5c14a59a5eef3e5472a67c96b0e4fcf331 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 2 Sep 2025 20:41:09 +0530 Subject: [PATCH 17/69] Implement distance spatial queries and validation for spatial attributes --- src/Database/Adapter/MariaDB.php | 61 ++++++++++----- src/Database/Adapter/MySQL.php | 31 ++++++++ src/Database/Adapter/Postgres.php | 86 +++++++++++++-------- src/Database/Database.php | 3 + src/Database/Query.php | 20 +++-- src/Database/Validator/Query/Filter.php | 2 +- src/Database/Validator/Spatial.php | 28 +++++-- tests/e2e/Adapter/Scopes/SpatialTests.php | 94 ++++++++++++++++++++++- tests/unit/Validator/SpatialTest.php | 84 ++++++++++++++++++++ 9 files changed, 339 insertions(+), 70 deletions(-) create mode 100644 tests/unit/Validator/SpatialTest.php diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 0e1cb09b4..c7a989f2e 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1352,6 +1352,47 @@ public function deleteDocument(string $collection, string $id): bool return $deleted; } + /** + * Handle distance spatial queries + * + * @param Query $query + * @param array $binds + * @param string $attribute + * @param string $alias + * @param string $placeholder + * @return string + */ + protected function handleDistanceSpatialQueries(Query $query, array &$binds, string $attribute, string $alias, string $placeholder): string + { + $distanceParams = $query->getValues()[0]; + $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($distanceParams[0]); + $binds[":{$placeholder}_1"] = $distanceParams[1]; + + $meters = isset($distanceParams[2]) && $distanceParams[2] === true; + + switch ($query->getMethod()) { + case Query::TYPE_DISTANCE_EQUAL: + $operator = '='; + break; + case Query::TYPE_DISTANCE_NOT_EQUAL: + $operator = '!='; + break; + case Query::TYPE_DISTANCE_GREATER_THAN: + $operator = '>'; + break; + case Query::TYPE_DISTANCE_LESS_THAN: + $operator = '<'; + break; + default: + throw new DatabaseException('Unknown spatial query method: ' . $query->getMethod()); + } + + if ($meters) { + return "ST_DISTANCE_SPHERE({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0), 6371000) {$operator} :{$placeholder}_1"; + } + return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0)) {$operator} :{$placeholder}_1"; + } + /** * Handle spatial queries * @@ -1374,28 +1415,10 @@ protected function handleSpatialQueries(Query $query, array &$binds, string $att return "NOT ST_Crosses({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; case Query::TYPE_DISTANCE_EQUAL: - $distanceParams = $query->getValues()[0]; - $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($distanceParams[0]); - $binds[":{$placeholder}_1"] = $distanceParams[1]; - return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0)) = :{$placeholder}_1"; - case Query::TYPE_DISTANCE_NOT_EQUAL: - $distanceParams = $query->getValues()[0]; - $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($distanceParams[0]); - $binds[":{$placeholder}_1"] = $distanceParams[1]; - return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0)) != :{$placeholder}_1"; - case Query::TYPE_DISTANCE_GREATER_THAN: - $distanceParams = $query->getValues()[0]; - $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($distanceParams[0]); - $binds[":{$placeholder}_1"] = $distanceParams[1]; - return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0)) > :{$placeholder}_1"; - case Query::TYPE_DISTANCE_LESS_THAN: - $distanceParams = $query->getValues()[0]; - $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($distanceParams[0]); - $binds[":{$placeholder}_1"] = $distanceParams[1]; - return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0)) < :{$placeholder}_1"; + return $this->handleDistanceSpatialQueries($query, $binds, $attribute, $alias, $placeholder); case Query::TYPE_INTERSECTS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index be0cd79d3..84b3cb9e9 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -7,6 +7,7 @@ use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Dependency as DependencyException; use Utopia\Database\Exception\Timeout as TimeoutException; +use Utopia\Database\Query; class MySQL extends MariaDB { @@ -78,6 +79,36 @@ public function getSizeOfCollectionOnDisk(string $collection): int return $size; } + protected function handleDistanceSpatialQueries(Query $query, array &$binds, string $attribute, string $alias, string $placeholder): string + { + $distanceParams = $query->getValues()[0]; + $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($distanceParams[0]); + $binds[":{$placeholder}_1"] = $distanceParams[1]; + + $useMeters = isset($distanceParams[2]) && $distanceParams[2] === true; + + switch ($query->getMethod()) { + case Query::TYPE_DISTANCE_EQUAL: + $operator = '='; + break; + case Query::TYPE_DISTANCE_NOT_EQUAL: + $operator = '!='; + break; + case Query::TYPE_DISTANCE_GREATER_THAN: + $operator = '>'; + break; + case Query::TYPE_DISTANCE_LESS_THAN: + $operator = '<'; + break; + default: + throw new DatabaseException('Unknown spatial query method: ' . $query->getMethod()); + } + + $unit = $useMeters ? ", 'meter'" : ''; + + return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0){$unit}) {$operator} :{$placeholder}_1"; + } + public function getSupportForIndexArray(): bool { /** diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index ab44bead1..be462161b 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1452,6 +1452,43 @@ public function getConnectionId(): string return $stmt->fetchColumn(); } + protected function handleDistanceSpatialQueries(Query $query, array &$binds, string $attribute, string $alias, string $placeholder): string + { + $distanceParams = $query->getValues()[0]; + $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($distanceParams[0]); + $binds[":{$placeholder}_1"] = $distanceParams[1]; + + $meters = isset($distanceParams[2]) && $distanceParams[2] === true; + + switch ($query->getMethod()) { + case Query::TYPE_DISTANCE_EQUAL: + $operator = '='; + break; + case Query::TYPE_DISTANCE_NOT_EQUAL: + $operator = '!='; + break; + case Query::TYPE_DISTANCE_GREATER_THAN: + $operator = '>'; + break; + case Query::TYPE_DISTANCE_LESS_THAN: + $operator = '<'; + break; + default: + throw new DatabaseException('Unknown spatial query method: ' . $query->getMethod()); + } + + if ($meters) { + // Transform both attribute and input geometry to 3857 (meters) for distance calculation + $attr = "ST_Transform({$alias}.{$attribute}, 3857)"; + $geom = "ST_Transform(ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "), 3857)"; + return "ST_Distance({$attr}, {$geom}) {$operator} :{$placeholder}_1"; + } + + // Without meters, use the original SRID (e.g., 4326) + return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . ")) {$operator} :{$placeholder}_1"; + } + + /** * Handle spatial queries * @@ -1474,60 +1511,41 @@ protected function handleSpatialQueries(Query $query, array &$binds, string $att return "NOT ST_Crosses({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; case Query::TYPE_DISTANCE_EQUAL: - $distanceParams = $query->getValues()[0]; - $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($distanceParams[0]); - $binds[":{$placeholder}_1"] = $distanceParams[1]; - return "ST_DWithin({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0), :{$placeholder}_1)"; - case Query::TYPE_DISTANCE_NOT_EQUAL: - $distanceParams = $query->getValues()[0]; - $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($distanceParams[0]); - $binds[":{$placeholder}_1"] = $distanceParams[1]; - return "NOT ST_DWithin({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0), :{$placeholder}_1)"; - case Query::TYPE_DISTANCE_GREATER_THAN: - $distanceParams = $query->getValues()[0]; - $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($distanceParams[0]); - $binds[":{$placeholder}_1"] = $distanceParams[1]; - return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0)) > :{$placeholder}_1"; - case Query::TYPE_DISTANCE_LESS_THAN: - $distanceParams = $query->getValues()[0]; - $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($distanceParams[0]); - $binds[":{$placeholder}_1"] = $distanceParams[1]; - return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0)) < :{$placeholder}_1"; - + return $this->handleDistanceSpatialQueries($query, $binds, $attribute, $alias, $placeholder); case Query::TYPE_EQUAL: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; case Query::TYPE_NOT_EQUAL: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "NOT ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; case Query::TYPE_INTERSECTS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; case Query::TYPE_NOT_INTERSECTS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "NOT ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; case Query::TYPE_OVERLAPS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; case Query::TYPE_NOT_OVERLAPS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "NOT ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; case Query::TYPE_TOUCHES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; case Query::TYPE_NOT_TOUCHES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "NOT ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; case Query::TYPE_CONTAINS: case Query::TYPE_NOT_CONTAINS: @@ -1536,8 +1554,8 @@ protected function handleSpatialQueries(Query $query, array &$binds, string $att $isNot = $query->getMethod() === Query::TYPE_NOT_CONTAINS; $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); return $isNot - ? "NOT ST_Covers({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))" - : "ST_Covers({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + ? "NOT ST_Covers({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))" + : "ST_Covers({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; default: throw new DatabaseException('Unknown spatial query method: ' . $query->getMethod()); @@ -1716,15 +1734,15 @@ protected function getSQLType(string $type, int $size, bool $signed = true, bool case Database::VAR_DATETIME: return 'TIMESTAMP(3)'; - + // in all other DB engines, 4326 is the default SRID case Database::VAR_POINT: - return 'GEOMETRY(POINT)'; + return 'GEOMETRY(POINT,' . Database::SRID . ')'; case Database::VAR_LINESTRING: - return 'GEOMETRY(LINESTRING)'; + return 'GEOMETRY(LINESTRING,' . Database::SRID . ')'; case Database::VAR_POLYGON: - return 'GEOMETRY(POLYGON)'; + return 'GEOMETRY(POLYGON,' . Database::SRID . ')'; default: throw new DatabaseException('Unknown Type: ' . $type . '. Must be one of ' . Database::VAR_STRING . ', ' . Database::VAR_INTEGER . ', ' . Database::VAR_FLOAT . ', ' . Database::VAR_BOOLEAN . ', ' . Database::VAR_DATETIME . ', ' . Database::VAR_RELATIONSHIP . ', ' . Database::VAR_POINT . ', ' . Database::VAR_LINESTRING . ', ' . Database::VAR_POLYGON); diff --git a/src/Database/Database.php b/src/Database/Database.php index 8f1d4ba5d..516949fe5 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -49,6 +49,9 @@ class Database public const BIG_INT_MAX = PHP_INT_MAX; public const DOUBLE_MAX = PHP_FLOAT_MAX; + // Global SRID for geographic coordinates (WGS84) + public const SRID = 4326; + // Relationship Types public const VAR_RELATIONSHIP = 'relationship'; diff --git a/src/Database/Query.php b/src/Database/Query.php index 24f40eece..adc2b524f 100644 --- a/src/Database/Query.php +++ b/src/Database/Query.php @@ -903,11 +903,12 @@ public function setOnArray(bool $bool): void * @param string $attribute * @param array $values * @param int|float $distance + * @param bool $meters * @return Query */ - public static function distanceEqual(string $attribute, array $values, int|float $distance): self + public static function distanceEqual(string $attribute, array $values, int|float $distance, bool $meters = false): self { - return new self(self::TYPE_DISTANCE_EQUAL, $attribute, [[$values,$distance]]); + return new self(self::TYPE_DISTANCE_EQUAL, $attribute, [[$values,$distance,$meters]]); } /** @@ -916,11 +917,12 @@ public static function distanceEqual(string $attribute, array $values, int|float * @param string $attribute * @param array $values * @param int|float $distance + * @param bool $meters * @return Query */ - public static function distanceNotEqual(string $attribute, array $values, int|float $distance): self + public static function distanceNotEqual(string $attribute, array $values, int|float $distance, $meters = false): self { - return new self(self::TYPE_DISTANCE_NOT_EQUAL, $attribute, [[$values,$distance]]); + return new self(self::TYPE_DISTANCE_NOT_EQUAL, $attribute, [[$values,$distance,$meters]]); } /** @@ -929,11 +931,12 @@ public static function distanceNotEqual(string $attribute, array $values, int|fl * @param string $attribute * @param array $values * @param int|float $distance + * @param bool $meters * @return Query */ - public static function distanceGreaterThan(string $attribute, array $values, int|float $distance): self + public static function distanceGreaterThan(string $attribute, array $values, int|float $distance, $meters = false): self { - return new self(self::TYPE_DISTANCE_GREATER_THAN, $attribute, [[$values,$distance]]); + return new self(self::TYPE_DISTANCE_GREATER_THAN, $attribute, [[$values,$distance, $meters]]); } /** @@ -942,11 +945,12 @@ public static function distanceGreaterThan(string $attribute, array $values, int * @param string $attribute * @param array $values * @param int|float $distance + * @param bool $meters * @return Query */ - public static function distanceLessThan(string $attribute, array $values, int|float $distance): self + public static function distanceLessThan(string $attribute, array $values, int|float $distance, bool $meters = false): self { - return new self(self::TYPE_DISTANCE_LESS_THAN, $attribute, [[$values,$distance]]); + return new self(self::TYPE_DISTANCE_LESS_THAN, $attribute, [[$values,$distance,$meters]]); } /** diff --git a/src/Database/Validator/Query/Filter.php b/src/Database/Validator/Query/Filter.php index 9f331d871..9c60f551c 100644 --- a/src/Database/Validator/Query/Filter.php +++ b/src/Database/Validator/Query/Filter.php @@ -263,7 +263,7 @@ public function isValid($value): bool case Query::TYPE_DISTANCE_NOT_EQUAL: case Query::TYPE_DISTANCE_GREATER_THAN: case Query::TYPE_DISTANCE_LESS_THAN: - if (count($value->getValues()) !== 1 || !is_array($value->getValues()[0]) || count($value->getValues()[0]) !== 2) { + if (count($value->getValues()) !== 1 || !is_array($value->getValues()[0]) || count($value->getValues()[0]) !== 3) { $this->message = 'Distance query requires [[geometry, distance]] parameters'; return false; } diff --git a/src/Database/Validator/Spatial.php b/src/Database/Validator/Spatial.php index 9843fe38c..2470562cb 100644 --- a/src/Database/Validator/Spatial.php +++ b/src/Database/Validator/Spatial.php @@ -18,7 +18,7 @@ public function __construct(string $spatialType) /** * Validate POINT data * - * @param array $value + * @param array $value * @return bool */ protected function validatePoint(array $value): bool @@ -28,6 +28,11 @@ protected function validatePoint(array $value): bool return false; } + if (!is_numeric($value[0]) || !is_numeric($value[1])) { + $this->message = 'Point coordinates must be numeric values'; + return false; + } + return true; } @@ -82,23 +87,34 @@ protected function validatePolygon(array $value): bool $value = [$value]; // wrap single ring } - foreach ($value as $ring) { + foreach ($value as $ringIndex => $ring) { if (!is_array($ring) || empty($ring)) { - $this->message = 'Each ring in Polygon must be an array of points'; + $this->message = "Ring #{$ringIndex} must be an array of points"; + return false; + } + + if (count($ring) < 4) { + $this->message = "Ring #{$ringIndex} must contain at least 4 points to form a closed polygon"; return false; } - foreach ($ring as $point) { + foreach ($ring as $pointIndex => $point) { if (!is_array($point) || count($point) !== 2) { - $this->message = 'Each point in Polygon ring must be an array of two values [x, y]'; + $this->message = "Point #{$pointIndex} in ring #{$ringIndex} must be an array of two values [x, y]"; return false; } if (!is_numeric($point[0]) || !is_numeric($point[1])) { - $this->message = 'Each point in Polygon ring must have numeric coordinates'; + $this->message = "Coordinates of point #{$pointIndex} in ring #{$ringIndex} must be numeric"; return false; } } + + // Check that the ring is closed (first point == last point) + if ($ring[0] !== $ring[count($ring) - 1]) { + $this->message = "Ring #{$ringIndex} must be closed (first point must equal last point)"; + return false; + } } return true; diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 4af2afa62..b1d3d9866 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -2026,7 +2026,7 @@ public function testInvalidSpatialTypes(): void $database->createCollection($collectionName, $attributes); - // ❌ Invalid Point (must be [x, y]) + // Invalid Point (must be [x, y]) try { $database->createDocument($collectionName, new Document([ 'pointAttr' => [10.0], // only 1 coordinate @@ -2036,7 +2036,7 @@ public function testInvalidSpatialTypes(): void $this->assertInstanceOf(StructureException::class, $th); } - // ❌ Invalid LineString (must be [[x,y],[x,y],...], at least 2 points) + // Invalid LineString (must be [[x,y],[x,y],...], at least 2 points) try { $database->createDocument($collectionName, new Document([ 'lineAttr' => [[10.0, 20.0]], // only one point @@ -2064,8 +2064,98 @@ public function testInvalidSpatialTypes(): void $this->assertInstanceOf(StructureException::class, $th); } + $invalidPolygons = [ + [[0,0],[1,1],[0,1]], + [[0,0],['a',1],[1,1],[0,0]], + [[0,0],[1,0],[1,1],[0,1]], + [], + [[0,0,5],[1,0,5],[1,1,5],[0,0,5]], + [ + [[0,0],[2,0],[2,2],[0,0]], // valid + [[0,0,1],[1,0,1],[1,1,1],[0,0,1]] // invalid 3D + ] + ]; + foreach ($invalidPolygons as $invalidPolygon) { + try { + $database->createDocument($collectionName, new Document([ + 'polyAttr' => $invalidPolygon + ])); + $this->fail("Expected StructureException for invalid polygon structure"); + } catch (\Throwable $th) { + $this->assertInstanceOf(StructureException::class, $th); + } + } // Cleanup $database->deleteCollection($collectionName); } + public function testSpatialDistanceInMeter(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + if (!$database->getAdapter()->getSupportForSpatialAttributes()) { + $this->markTestSkipped('Adapter does not support spatial attributes'); + } + + $collectionName = 'spatial_distance_meters_'; + try { + $database->createCollection($collectionName); + $this->assertEquals(true, $database->createAttribute($collectionName, 'loc', Database::VAR_POINT, 0, true)); + $this->assertEquals(true, $database->createIndex($collectionName, 'idx_loc', Database::INDEX_SPATIAL, ['loc'])); + + // Two points roughly ~1000 meters apart by latitude delta (~0.009 deg ≈ 1km) + $p0 = $database->createDocument($collectionName, new Document([ + '$id' => 'p0', + 'loc' => [0.0000, 0.0000], + '$permissions' => [Permission::read(Role::any()), Permission::update(Role::any())] + ])); + $p1 = $database->createDocument($collectionName, new Document([ + '$id' => 'p1', + 'loc' => [0.0090, 0.0000], + '$permissions' => [Permission::read(Role::any()), Permission::update(Role::any())] + ])); + + $this->assertInstanceOf(Document::class, $p0); + $this->assertInstanceOf(Document::class, $p1); + + // distanceLessThan with meters=true: within 1500m should include both + $within1_5km = $database->find($collectionName, [ + Query::distanceLessThan('loc', [0.0000, 0.0000], 1500, true) + ], Database::PERMISSION_READ); + $this->assertNotEmpty($within1_5km); + $this->assertCount(2, $within1_5km); + + // Within 500m should include only p0 (exact point) + $within500m = $database->find($collectionName, [ + Query::distanceLessThan('loc', [0.0000, 0.0000], 500, true) + ], Database::PERMISSION_READ); + $this->assertNotEmpty($within500m); + $this->assertCount(1, $within500m); + $this->assertEquals('p0', $within500m[0]->getId()); + + // distanceGreaterThan 500m should include only p1 + $greater500m = $database->find($collectionName, [ + Query::distanceGreaterThan('loc', [0.0000, 0.0000], 500, true) + ], Database::PERMISSION_READ); + $this->assertNotEmpty($greater500m); + $this->assertCount(1, $greater500m); + $this->assertEquals('p1', $greater500m[0]->getId()); + + // distanceEqual with 0m should return exact match p0 + $equalZero = $database->find($collectionName, [ + Query::distanceEqual('loc', [0.0000, 0.0000], 0, true) + ], Database::PERMISSION_READ); + $this->assertNotEmpty($equalZero); + $this->assertEquals('p0', $equalZero[0]->getId()); + + // distanceNotEqual with 0m should return p1 + $notEqualZero = $database->find($collectionName, [ + Query::distanceNotEqual('loc', [0.0000, 0.0000], 0, true) + ], Database::PERMISSION_READ); + $this->assertNotEmpty($notEqualZero); + $this->assertEquals('p1', $notEqualZero[0]->getId()); + } finally { + $database->deleteCollection($collectionName); + } + } } diff --git a/tests/unit/Validator/SpatialTest.php b/tests/unit/Validator/SpatialTest.php new file mode 100644 index 000000000..8fe83a870 --- /dev/null +++ b/tests/unit/Validator/SpatialTest.php @@ -0,0 +1,84 @@ +assertTrue($validator->isValid([10, 20])); + $this->assertTrue($validator->isValid([0, 0])); + $this->assertTrue($validator->isValid([-180.0, 90.0])); + + // Invalid cases + $this->assertFalse($validator->isValid([10])); // Only one coordinate + $this->assertFalse($validator->isValid([10, 'a'])); // Non-numeric + $this->assertFalse($validator->isValid([[10, 20]])); // Nested array + } + + public function testValidLineString(): void + { + $validator = new Spatial(Database::VAR_LINESTRING); + + $this->assertTrue($validator->isValid([[0, 0], [1, 1]])); + + $this->assertTrue($validator->isValid([[10, 10], [20, 20], [30, 30]])); + + // Invalid cases + $this->assertFalse($validator->isValid([[10, 10]])); // Only one point + $this->assertFalse($validator->isValid([[10, 10], [20]])); // Malformed point + $this->assertFalse($validator->isValid([[10, 10], ['x', 'y']])); // Non-numeric + } + + public function testValidPolygon(): void + { + $validator = new Spatial(Database::VAR_POLYGON); + + // Single ring polygon (closed) + $this->assertTrue($validator->isValid([ + [0, 0], + [0, 1], + [1, 1], + [1, 0], + [0, 0] + ])); + + // Multi-ring polygon + $this->assertTrue($validator->isValid([ + [ // Outer ring + [0, 0], [0, 4], [4, 4], [4, 0], [0, 0] + ], + [ // Hole + [1, 1], [1, 2], [2, 2], [2, 1], [1, 1] + ] + ])); + + // Invalid polygons + $this->assertFalse($validator->isValid([])); // Empty + $this->assertFalse($validator->isValid([ + [0, 0], [1, 1], [2, 2] // Not closed, less than 4 points + ])); + $this->assertFalse($validator->isValid([ + [[0, 0], [1, 1], [1, 0]] // Not closed + ])); + $this->assertFalse($validator->isValid([ + [[0, 0], [1, 1], [1, 'a'], [0, 0]] // Non-numeric + ])); + } + + public function testWKTStrings(): void + { + $this->assertTrue(Spatial::isWKTString('POINT(1 2)')); + $this->assertTrue(Spatial::isWKTString('LINESTRING(0 0,1 1)')); + $this->assertTrue(Spatial::isWKTString('POLYGON((0 0,1 0,1 1,0 1,0 0))')); + + $this->assertFalse(Spatial::isWKTString('CIRCLE(0 0,1)')); + $this->assertFalse(Spatial::isWKTString('POINT1(1 2)')); + } +} From 3336ebc4cccaef7a6e28b763fc83b1f773394d35 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 2 Sep 2025 20:45:15 +0530 Subject: [PATCH 18/69] updated phpdocs for handledistancespatial --- src/Database/Adapter/MySQL.php | 10 ++++++++++ src/Database/Adapter/Postgres.php | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index 84b3cb9e9..1603201f2 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -79,6 +79,16 @@ public function getSizeOfCollectionOnDisk(string $collection): int return $size; } + /** + * Handle distance spatial queries + * + * @param Query $query + * @param array $binds + * @param string $attribute + * @param string $alias + * @param string $placeholder + * @return string + */ protected function handleDistanceSpatialQueries(Query $query, array &$binds, string $attribute, string $alias, string $placeholder): string { $distanceParams = $query->getValues()[0]; diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index be462161b..2ace9f555 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1452,6 +1452,16 @@ public function getConnectionId(): string return $stmt->fetchColumn(); } + /** + * Handle distance spatial queries + * + * @param Query $query + * @param array $binds + * @param string $attribute + * @param string $alias + * @param string $placeholder + * @return string + */ protected function handleDistanceSpatialQueries(Query $query, array &$binds, string $attribute, string $alias, string $placeholder): string { $distanceParams = $query->getValues()[0]; From ddabbf8b4b326bbdc104a313b99ff7ea6c39acf6 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 2 Sep 2025 21:37:07 +0530 Subject: [PATCH 19/69] Refactor spatial query handling to improve clarity and consistency in distance calculations --- src/Database/Adapter/MariaDB.php | 4 ++-- src/Database/Adapter/MySQL.php | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index c7a989f2e..d9ea2d130 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1368,7 +1368,7 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($distanceParams[0]); $binds[":{$placeholder}_1"] = $distanceParams[1]; - $meters = isset($distanceParams[2]) && $distanceParams[2] === true; + $useMeters = isset($distanceParams[2]) && $distanceParams[2] === true; switch ($query->getMethod()) { case Query::TYPE_DISTANCE_EQUAL: @@ -1387,7 +1387,7 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str throw new DatabaseException('Unknown spatial query method: ' . $query->getMethod()); } - if ($meters) { + if ($useMeters) { return "ST_DISTANCE_SPHERE({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0), 6371000) {$operator} :{$placeholder}_1"; } return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0)) {$operator} :{$placeholder}_1"; diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index 1603201f2..682ce18ef 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -114,9 +114,14 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str throw new DatabaseException('Unknown spatial query method: ' . $query->getMethod()); } - $unit = $useMeters ? ", 'meter'" : ''; + if ($useMeters) { + $attr = "ST_SRID({$alias}.{$attribute}, " . Database::SRID . ")"; + $geom = "ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . ")"; + return "ST_Distance({$attr}, {$geom}, 'metre') {$operator} :{$placeholder}_1"; + } - return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0){$unit}) {$operator} :{$placeholder}_1"; + // Without meters, use default behavior + return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0)) {$operator} :{$placeholder}_1"; } public function getSupportForIndexArray(): bool From 49bb55af9303d5f8244003c602b128adb919eb23 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 2 Sep 2025 21:39:04 +0530 Subject: [PATCH 20/69] updated meter param for distance --- src/Database/Query.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Database/Query.php b/src/Database/Query.php index adc2b524f..8740820dd 100644 --- a/src/Database/Query.php +++ b/src/Database/Query.php @@ -920,7 +920,7 @@ public static function distanceEqual(string $attribute, array $values, int|float * @param bool $meters * @return Query */ - public static function distanceNotEqual(string $attribute, array $values, int|float $distance, $meters = false): self + public static function distanceNotEqual(string $attribute, array $values, int|float $distance, bool $meters = false): self { return new self(self::TYPE_DISTANCE_NOT_EQUAL, $attribute, [[$values,$distance,$meters]]); } @@ -934,7 +934,7 @@ public static function distanceNotEqual(string $attribute, array $values, int|fl * @param bool $meters * @return Query */ - public static function distanceGreaterThan(string $attribute, array $values, int|float $distance, $meters = false): self + public static function distanceGreaterThan(string $attribute, array $values, int|float $distance, bool $meters = false): self { return new self(self::TYPE_DISTANCE_GREATER_THAN, $attribute, [[$values,$distance, $meters]]); } From e2bdc83d8171a7c343da5d790810013195ea0960 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 4 Sep 2025 03:01:45 +1200 Subject: [PATCH 21/69] Don't wait to return empty --- src/Database/Database.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 516949fe5..a600421c6 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -4275,6 +4275,9 @@ public function updateDocument(string $collection, string $id, Document $documen $old = Authorization::skip(fn () => $this->silent( fn () => $this->getDocument($collection->getId(), $id, forUpdate: true) )); + if ($old->isEmpty()) { + return new Document(); + } $skipPermissionsUpdate = true; @@ -4414,10 +4417,6 @@ public function updateDocument(string $collection, string $id, Document $documen } } - if ($old->isEmpty()) { - return new Document(); - } - if ($shouldUpdate) { $document->setAttribute('$updatedAt', ($newUpdatedAt === null || !$this->preserveDates) ? $time : $newUpdatedAt); } From 42f35b7809155c875c50e60cb676e0268c329dcb Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 4 Sep 2025 03:24:02 +1200 Subject: [PATCH 22/69] Return empty on empty --- src/Database/Database.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Database/Database.php b/src/Database/Database.php index a600421c6..02211679b 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -4449,6 +4449,10 @@ public function updateDocument(string $collection, string $id, Document $documen return $document; }); + if ($document->isEmpty()) { + return $document; + } + if ($this->resolveRelationships) { $document = $this->silent(fn () => $this->populateDocumentRelationships($collection, $document)); } From 82aa0e8523419dd832350255b52514d316e41f91 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 4 Sep 2025 03:28:02 +1200 Subject: [PATCH 23/69] Only trigger deleted if deleted --- src/Database/Database.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 02211679b..9b7a60ff0 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -5576,7 +5576,9 @@ public function deleteDocument(string $collection, string $id): bool return $result; }); - $this->trigger(self::EVENT_DOCUMENT_DELETE, $document); + if ($deleted) { + $this->trigger(self::EVENT_DOCUMENT_DELETE, $document); + } return $deleted; } From 35b7744e2b61d6e5cdb7e783245eaae7d3820032 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 4 Sep 2025 22:55:30 +1200 Subject: [PATCH 24/69] Add time between queries --- src/Database/Query.php | 24 ++++++ tests/e2e/Adapter/Scopes/DocumentTests.php | 92 ++++++++++++++++++++++ tests/unit/QueryTest.php | 22 ++++++ 3 files changed, 138 insertions(+) diff --git a/src/Database/Query.php b/src/Database/Query.php index 8740820dd..5df9d6585 100644 --- a/src/Database/Query.php +++ b/src/Database/Query.php @@ -730,6 +730,30 @@ public static function updatedAfter(string $value): self return self::greaterThan('$updatedAt', $value); } + /** + * Helper method to create Query for documents created between two dates + * + * @param string $start + * @param string $end + * @return Query + */ + public static function createdBetween(string $start, string $end): self + { + return self::between('$createdAt', $start, $end); + } + + /** + * Helper method to create Query for documents updated between two dates + * + * @param string $start + * @param string $end + * @return Query + */ + public static function updatedBetween(string $start, string $end): self + { + return self::between('$updatedAt', $start, $end); + } + /** * @param array $queries * @return Query diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index bf7f2a905..79e2d8299 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -2745,6 +2745,98 @@ public function testFindUpdatedAfter(): void $this->assertEquals(0, count($documents)); } + public function testFindCreatedBetween(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + /** + * Test Query::createdBetween wrapper + */ + $pastDate = '1900-01-01T00:00:00.000Z'; + $futureDate = '2050-01-01T00:00:00.000Z'; + $recentPastDate = '2020-01-01T00:00:00.000Z'; + $nearFutureDate = '2025-01-01T00:00:00.000Z'; + + // All documents should be between past and future + $documents = $database->find('movies', [ + Query::createdBetween($pastDate, $futureDate), + Query::limit(25) + ]); + + $this->assertGreaterThan(0, count($documents)); + + // No documents should exist in this range + $documents = $database->find('movies', [ + Query::createdBetween($pastDate, $pastDate), + Query::limit(25) + ]); + + $this->assertEquals(0, count($documents)); + + // Documents created between recent past and near future + $documents = $database->find('movies', [ + Query::createdBetween($recentPastDate, $nearFutureDate), + Query::limit(25) + ]); + + $count = count($documents); + + // Same count should be returned with expanded range + $documents = $database->find('movies', [ + Query::createdBetween($pastDate, $nearFutureDate), + Query::limit(25) + ]); + + $this->assertGreaterThanOrEqual($count, count($documents)); + } + + public function testFindUpdatedBetween(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + /** + * Test Query::updatedBetween wrapper + */ + $pastDate = '1900-01-01T00:00:00.000Z'; + $futureDate = '2050-01-01T00:00:00.000Z'; + $recentPastDate = '2020-01-01T00:00:00.000Z'; + $nearFutureDate = '2025-01-01T00:00:00.000Z'; + + // All documents should be between past and future + $documents = $database->find('movies', [ + Query::updatedBetween($pastDate, $futureDate), + Query::limit(25) + ]); + + $this->assertGreaterThan(0, count($documents)); + + // No documents should exist in this range + $documents = $database->find('movies', [ + Query::updatedBetween($pastDate, $pastDate), + Query::limit(25) + ]); + + $this->assertEquals(0, count($documents)); + + // Documents updated between recent past and near future + $documents = $database->find('movies', [ + Query::updatedBetween($recentPastDate, $nearFutureDate), + Query::limit(25) + ]); + + $count = count($documents); + + // Same count should be returned with expanded range + $documents = $database->find('movies', [ + Query::updatedBetween($pastDate, $nearFutureDate), + Query::limit(25) + ]); + + $this->assertGreaterThanOrEqual($count, count($documents)); + } + public function testFindLimit(): void { /** @var Database $database */ diff --git a/tests/unit/QueryTest.php b/tests/unit/QueryTest.php index 3084abaa0..c48755cb2 100644 --- a/tests/unit/QueryTest.php +++ b/tests/unit/QueryTest.php @@ -141,6 +141,18 @@ public function testCreate(): void $this->assertEquals(Query::TYPE_GREATER, $query->getMethod()); $this->assertEquals('$updatedAt', $query->getAttribute()); $this->assertEquals(['2023-12-31T23:59:59.999Z'], $query->getValues()); + + $query = Query::createdBetween('2023-01-01T00:00:00.000Z', '2023-12-31T23:59:59.999Z'); + + $this->assertEquals(Query::TYPE_BETWEEN, $query->getMethod()); + $this->assertEquals('$createdAt', $query->getAttribute()); + $this->assertEquals(['2023-01-01T00:00:00.000Z', '2023-12-31T23:59:59.999Z'], $query->getValues()); + + $query = Query::updatedBetween('2023-01-01T00:00:00.000Z', '2023-12-31T23:59:59.999Z'); + + $this->assertEquals(Query::TYPE_BETWEEN, $query->getMethod()); + $this->assertEquals('$updatedAt', $query->getAttribute()); + $this->assertEquals(['2023-01-01T00:00:00.000Z', '2023-12-31T23:59:59.999Z'], $query->getValues()); } /** @@ -271,6 +283,16 @@ public function testParse(): void $this->assertEquals('$updatedAt', $query->getAttribute()); $this->assertEquals(['2023-12-31T23:59:59.999Z'], $query->getValues()); + $query = Query::parse(Query::createdBetween('2023-01-01T00:00:00.000Z', '2023-12-31T23:59:59.999Z')->toString()); + $this->assertEquals('between', $query->getMethod()); + $this->assertEquals('$createdAt', $query->getAttribute()); + $this->assertEquals(['2023-01-01T00:00:00.000Z', '2023-12-31T23:59:59.999Z'], $query->getValues()); + + $query = Query::parse(Query::updatedBetween('2023-01-01T00:00:00.000Z', '2023-12-31T23:59:59.999Z')->toString()); + $this->assertEquals('between', $query->getMethod()); + $this->assertEquals('$updatedAt', $query->getAttribute()); + $this->assertEquals(['2023-01-01T00:00:00.000Z', '2023-12-31T23:59:59.999Z'], $query->getValues()); + $query = Query::parse(Query::between('age', 15, 18)->toString()); $this->assertEquals('between', $query->getMethod()); $this->assertEquals('age', $query->getAttribute()); From fc01e45608d169bd8afd3b2c0224747da5bca54f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 4 Sep 2025 23:06:16 +1200 Subject: [PATCH 25/69] Pass old to update on next, rename to upsert --- composer.lock | 344 ++++++++++++++------- src/Database/Adapter.php | 2 +- src/Database/Adapter/Pool.php | 2 +- src/Database/Adapter/SQL.php | 2 +- src/Database/Database.php | 78 +++-- src/Database/Mirror.php | 17 +- tests/e2e/Adapter/Scopes/DocumentTests.php | 62 ++-- tests/e2e/Adapter/Scopes/GeneralTests.php | 6 +- tests/e2e/Adapter/Scopes/SpatialTests.php | 4 +- 9 files changed, 343 insertions(+), 174 deletions(-) diff --git a/composer.lock b/composer.lock index 2a5302cd2..5933f4fc9 100644 --- a/composer.lock +++ b/composer.lock @@ -68,16 +68,16 @@ }, { "name": "composer/semver", - "version": "3.4.3", + "version": "3.4.4", "source": { "type": "git", "url": "https://github.com/composer/semver.git", - "reference": "4313d26ada5e0c4edfbd1dc481a92ff7bff91f12" + "reference": "198166618906cb2de69b95d7d47e5fa8aa1b2b95" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/semver/zipball/4313d26ada5e0c4edfbd1dc481a92ff7bff91f12", - "reference": "4313d26ada5e0c4edfbd1dc481a92ff7bff91f12", + "url": "https://api.github.com/repos/composer/semver/zipball/198166618906cb2de69b95d7d47e5fa8aa1b2b95", + "reference": "198166618906cb2de69b95d7d47e5fa8aa1b2b95", "shasum": "" }, "require": { @@ -129,7 +129,7 @@ "support": { "irc": "ircs://irc.libera.chat:6697/composer", "issues": "https://github.com/composer/semver/issues", - "source": "https://github.com/composer/semver/tree/3.4.3" + "source": "https://github.com/composer/semver/tree/3.4.4" }, "funding": [ { @@ -139,33 +139,32 @@ { "url": "https://github.com/composer", "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/composer/composer", - "type": "tidelift" } ], - "time": "2024-09-19T14:15:21+00:00" + "time": "2025-08-20T19:15:30+00:00" }, { "name": "google/protobuf", - "version": "v4.31.1", + "version": "v4.32.0", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "2b028ce8876254e2acbeceea7d9b573faad41864" + "reference": "9a9a92ecbe9c671dc1863f6d4a91ea3ea12c8646" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/2b028ce8876254e2acbeceea7d9b573faad41864", - "reference": "2b028ce8876254e2acbeceea7d9b573faad41864", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/9a9a92ecbe9c671dc1863f6d4a91ea3ea12c8646", + "reference": "9a9a92ecbe9c671dc1863f6d4a91ea3ea12c8646", "shasum": "" }, "require": { - "php": ">=7.0.0" + "php": ">=8.1.0" + }, + "provide": { + "ext-protobuf": "*" }, "require-dev": { - "phpunit/phpunit": ">=5.0.0" + "phpunit/phpunit": ">=5.0.0 <8.5.27" }, "suggest": { "ext-bcmath": "Need to support JSON deserialization" @@ -187,9 +186,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.31.1" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.32.0" }, - "time": "2025-05-28T18:52:35+00:00" + "time": "2025-08-14T20:00:33+00:00" }, { "name": "nyholm/psr7", @@ -407,16 +406,16 @@ }, { "name": "open-telemetry/context", - "version": "1.2.1", + "version": "1.3.1", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/context.git", - "reference": "1eb2b837ee9362db064a6b65d5ecce15a9f9f020" + "reference": "438f71812242db3f196fb4c717c6f92cbc819be6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/context/zipball/1eb2b837ee9362db064a6b65d5ecce15a9f9f020", - "reference": "1eb2b837ee9362db064a6b65d5ecce15a9f9f020", + "url": "https://api.github.com/repos/opentelemetry-php/context/zipball/438f71812242db3f196fb4c717c6f92cbc819be6", + "reference": "438f71812242db3f196fb4c717c6f92cbc819be6", "shasum": "" }, "require": { @@ -462,7 +461,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-05-07T23:36:50+00:00" + "time": "2025-08-13T01:12:00+00:00" }, { "name": "open-telemetry/exporter-otlp", @@ -593,16 +592,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.6.0", + "version": "1.7.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "1c0371794e4c0700afd4a9d4d8511cb5e3f78e6a" + "reference": "86287cf30fd6549444d7b8f7d8758d92e24086ac" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/1c0371794e4c0700afd4a9d4d8511cb5e3f78e6a", - "reference": "1c0371794e4c0700afd4a9d4d8511cb5e3f78e6a", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/86287cf30fd6549444d7b8f7d8758d92e24086ac", + "reference": "86287cf30fd6549444d7b8f7d8758d92e24086ac", "shasum": "" }, "require": { @@ -621,7 +620,7 @@ "ramsey/uuid": "^3.0 || ^4.0", "symfony/polyfill-mbstring": "^1.23", "symfony/polyfill-php82": "^1.26", - "tbachert/spi": "^1.0.1" + "tbachert/spi": "^1.0.5" }, "suggest": { "ext-gmp": "To support unlimited number of synchronous metric readers", @@ -635,6 +634,9 @@ "OpenTelemetry\\API\\Instrumentation\\Configuration\\General\\ConfigEnv\\EnvComponentLoaderHttpConfig", "OpenTelemetry\\API\\Instrumentation\\Configuration\\General\\ConfigEnv\\EnvComponentLoaderPeerConfig" ], + "OpenTelemetry\\SDK\\Common\\Configuration\\Resolver\\ResolverInterface": [ + "OpenTelemetry\\SDK\\Common\\Configuration\\Resolver\\SdkConfigurationResolver" + ], "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\HookManagerInterface": [ "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\ExtensionHookManager" ] @@ -683,20 +685,20 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-06-19T23:36:51+00:00" + "time": "2025-08-06T03:07:06+00:00" }, { "name": "open-telemetry/sem-conv", - "version": "1.32.1", + "version": "1.37.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sem-conv.git", - "reference": "94daa85ea61a8e2b7e1b0af6be0e875bedda7c22" + "reference": "8da7ec497c881e39afa6657d72586e27efbd29a1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sem-conv/zipball/94daa85ea61a8e2b7e1b0af6be0e875bedda7c22", - "reference": "94daa85ea61a8e2b7e1b0af6be0e875bedda7c22", + "url": "https://api.github.com/repos/opentelemetry-php/sem-conv/zipball/8da7ec497c881e39afa6657d72586e27efbd29a1", + "reference": "8da7ec497c881e39afa6657d72586e27efbd29a1", "shasum": "" }, "require": { @@ -740,7 +742,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-06-24T02:32:27+00:00" + "time": "2025-09-03T12:08:10+00:00" }, { "name": "php-http/discovery", @@ -1307,16 +1309,16 @@ }, { "name": "symfony/http-client", - "version": "v7.3.1", + "version": "v7.3.3", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "4403d87a2c16f33345dca93407a8714ee8c05a64" + "reference": "333b9bd7639cbdaecd25a3a48a9d2dcfaa86e019" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/4403d87a2c16f33345dca93407a8714ee8c05a64", - "reference": "4403d87a2c16f33345dca93407a8714ee8c05a64", + "url": "https://api.github.com/repos/symfony/http-client/zipball/333b9bd7639cbdaecd25a3a48a9d2dcfaa86e019", + "reference": "333b9bd7639cbdaecd25a3a48a9d2dcfaa86e019", "shasum": "" }, "require": { @@ -1324,6 +1326,7 @@ "psr/log": "^1|^2|^3", "symfony/deprecation-contracts": "^2.5|^3", "symfony/http-client-contracts": "~3.4.4|^3.5.2", + "symfony/polyfill-php83": "^1.29", "symfony/service-contracts": "^2.5|^3" }, "conflict": { @@ -1382,7 +1385,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.3.1" + "source": "https://github.com/symfony/http-client/tree/v7.3.3" }, "funding": [ { @@ -1393,12 +1396,16 @@ "url": "https://github.com/fabpot", "type": "github" }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, { "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", "type": "tidelift" } ], - "time": "2025-06-28T07:58:39+00:00" + "time": "2025-08-27T07:45:05+00:00" }, { "name": "symfony/http-client-contracts", @@ -1480,7 +1487,7 @@ }, { "name": "symfony/polyfill-mbstring", - "version": "v1.32.0", + "version": "v1.33.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", @@ -1541,7 +1548,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.32.0" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.33.0" }, "funding": [ { @@ -1552,6 +1559,10 @@ "url": "https://github.com/fabpot", "type": "github" }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, { "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", "type": "tidelift" @@ -1561,7 +1572,7 @@ }, { "name": "symfony/polyfill-php82", - "version": "v1.32.0", + "version": "v1.33.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php82.git", @@ -1617,7 +1628,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php82/tree/v1.32.0" + "source": "https://github.com/symfony/polyfill-php82/tree/v1.33.0" }, "funding": [ { @@ -1628,6 +1639,10 @@ "url": "https://github.com/fabpot", "type": "github" }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, { "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", "type": "tidelift" @@ -1635,6 +1650,86 @@ ], "time": "2024-09-09T11:45:10+00:00" }, + { + "name": "symfony/polyfill-php83", + "version": "v1.33.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php83.git", + "reference": "17f6f9a6b1735c0f163024d959f700cfbc5155e5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php83/zipball/17f6f9a6b1735c0f163024d959f700cfbc5155e5", + "reference": "17f6f9a6b1735c0f163024d959f700cfbc5155e5", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php83\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 8.3+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php83/tree/v1.33.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2025-07-08T02:45:35+00:00" + }, { "name": "symfony/service-contracts", "version": "v3.6.0", @@ -1720,16 +1815,16 @@ }, { "name": "tbachert/spi", - "version": "v1.0.4", + "version": "v1.0.5", "source": { "type": "git", "url": "https://github.com/Nevay/spi.git", - "reference": "86e355edfdd57f9cb720bd2ac3af7dde521ca0e7" + "reference": "e7078767866d0a9e0f91d3f9d42a832df5e39002" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Nevay/spi/zipball/86e355edfdd57f9cb720bd2ac3af7dde521ca0e7", - "reference": "86e355edfdd57f9cb720bd2ac3af7dde521ca0e7", + "url": "https://api.github.com/repos/Nevay/spi/zipball/e7078767866d0a9e0f91d3f9d42a832df5e39002", + "reference": "e7078767866d0a9e0f91d3f9d42a832df5e39002", "shasum": "" }, "require": { @@ -1766,9 +1861,9 @@ ], "support": { "issues": "https://github.com/Nevay/spi/issues", - "source": "https://github.com/Nevay/spi/tree/v1.0.4" + "source": "https://github.com/Nevay/spi/tree/v1.0.5" }, - "time": "2025-06-28T20:18:22+00:00" + "time": "2025-06-29T15:42:06+00:00" }, { "name": "utopia-php/cache", @@ -1870,16 +1965,16 @@ }, { "name": "utopia-php/framework", - "version": "0.33.20", + "version": "0.33.24", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "e1c7ab4e0b5b0a9a70256b1e00912e101e76a131" + "reference": "5112b1023342163e3fbedec99f38fc32c8700aa0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/e1c7ab4e0b5b0a9a70256b1e00912e101e76a131", - "reference": "e1c7ab4e0b5b0a9a70256b1e00912e101e76a131", + "url": "https://api.github.com/repos/utopia-php/http/zipball/5112b1023342163e3fbedec99f38fc32c8700aa0", + "reference": "5112b1023342163e3fbedec99f38fc32c8700aa0", "shasum": "" }, "require": { @@ -1911,9 +2006,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.20" + "source": "https://github.com/utopia-php/http/tree/0.33.24" }, - "time": "2025-05-18T23:51:21+00:00" + "time": "2025-09-04T04:18:39+00:00" }, { "name": "utopia-php/pools", @@ -2154,16 +2249,16 @@ }, { "name": "laravel/pint", - "version": "v1.22.1", + "version": "v1.24.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "941d1927c5ca420c22710e98420287169c7bcaf7" + "reference": "0345f3b05f136801af8c339f9d16ef29e6b4df8a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/941d1927c5ca420c22710e98420287169c7bcaf7", - "reference": "941d1927c5ca420c22710e98420287169c7bcaf7", + "url": "https://api.github.com/repos/laravel/pint/zipball/0345f3b05f136801af8c339f9d16ef29e6b4df8a", + "reference": "0345f3b05f136801af8c339f9d16ef29e6b4df8a", "shasum": "" }, "require": { @@ -2174,10 +2269,10 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.75.0", - "illuminate/view": "^11.44.7", - "larastan/larastan": "^3.4.0", - "laravel-zero/framework": "^11.36.1", + "friendsofphp/php-cs-fixer": "^3.82.2", + "illuminate/view": "^11.45.1", + "larastan/larastan": "^3.5.0", + "laravel-zero/framework": "^11.45.0", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^2.3.1", "pestphp/pest": "^2.36.0" @@ -2187,6 +2282,9 @@ ], "type": "project", "autoload": { + "files": [ + "overrides/Runner/Parallel/ProcessFactory.php" + ], "psr-4": { "App\\": "app/", "Database\\Seeders\\": "database/seeders/", @@ -2216,20 +2314,20 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-05-08T08:38:12+00:00" + "time": "2025-07-10T18:09:32+00:00" }, { "name": "myclabs/deep-copy", - "version": "1.13.1", + "version": "1.13.4", "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "1720ddd719e16cf0db4eb1c6eca108031636d46c" + "reference": "07d290f0c47959fd5eed98c95ee5602db07e0b6a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/1720ddd719e16cf0db4eb1c6eca108031636d46c", - "reference": "1720ddd719e16cf0db4eb1c6eca108031636d46c", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/07d290f0c47959fd5eed98c95ee5602db07e0b6a", + "reference": "07d290f0c47959fd5eed98c95ee5602db07e0b6a", "shasum": "" }, "require": { @@ -2268,7 +2366,7 @@ ], "support": { "issues": "https://github.com/myclabs/DeepCopy/issues", - "source": "https://github.com/myclabs/DeepCopy/tree/1.13.1" + "source": "https://github.com/myclabs/DeepCopy/tree/1.13.4" }, "funding": [ { @@ -2276,7 +2374,7 @@ "type": "tidelift" } ], - "time": "2025-04-29T12:36:36+00:00" + "time": "2025-08-01T08:46:24+00:00" }, { "name": "nikic/php-parser", @@ -2488,16 +2586,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.12.27", + "version": "1.12.28", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "3a6e423c076ab39dfedc307e2ac627ef579db162" + "reference": "fcf8b71aeab4e1a1131d1783cef97b23a51b87a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/3a6e423c076ab39dfedc307e2ac627ef579db162", - "reference": "3a6e423c076ab39dfedc307e2ac627ef579db162", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/fcf8b71aeab4e1a1131d1783cef97b23a51b87a9", + "reference": "fcf8b71aeab4e1a1131d1783cef97b23a51b87a9", "shasum": "" }, "require": { @@ -2542,7 +2640,7 @@ "type": "github" } ], - "time": "2025-05-21T20:51:45+00:00" + "time": "2025-07-17T17:15:39+00:00" }, { "name": "phpunit/php-code-coverage", @@ -2865,16 +2963,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.23", + "version": "9.6.25", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "43d2cb18d0675c38bd44982a5d1d88f6d53d8d95" + "reference": "049c011e01be805202d8eebedef49f769a8ec7b7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/43d2cb18d0675c38bd44982a5d1d88f6d53d8d95", - "reference": "43d2cb18d0675c38bd44982a5d1d88f6d53d8d95", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/049c011e01be805202d8eebedef49f769a8ec7b7", + "reference": "049c011e01be805202d8eebedef49f769a8ec7b7", "shasum": "" }, "require": { @@ -2885,7 +2983,7 @@ "ext-mbstring": "*", "ext-xml": "*", "ext-xmlwriter": "*", - "myclabs/deep-copy": "^1.13.1", + "myclabs/deep-copy": "^1.13.4", "phar-io/manifest": "^2.0.4", "phar-io/version": "^3.2.1", "php": ">=7.3", @@ -2896,11 +2994,11 @@ "phpunit/php-timer": "^5.0.3", "sebastian/cli-parser": "^1.0.2", "sebastian/code-unit": "^1.0.8", - "sebastian/comparator": "^4.0.8", + "sebastian/comparator": "^4.0.9", "sebastian/diff": "^4.0.6", "sebastian/environment": "^5.1.5", "sebastian/exporter": "^4.0.6", - "sebastian/global-state": "^5.0.7", + "sebastian/global-state": "^5.0.8", "sebastian/object-enumerator": "^4.0.4", "sebastian/resource-operations": "^3.0.4", "sebastian/type": "^3.2.1", @@ -2948,7 +3046,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.23" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.25" }, "funding": [ { @@ -2972,7 +3070,7 @@ "type": "tidelift" } ], - "time": "2025-05-02T06:40:34+00:00" + "time": "2025-08-20T14:38:31+00:00" }, { "name": "rregeer/phpunit-coverage-check", @@ -3189,16 +3287,16 @@ }, { "name": "sebastian/comparator", - "version": "4.0.8", + "version": "4.0.9", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/comparator.git", - "reference": "fa0f136dd2334583309d32b62544682ee972b51a" + "reference": "67a2df3a62639eab2cc5906065e9805d4fd5dfc5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/fa0f136dd2334583309d32b62544682ee972b51a", - "reference": "fa0f136dd2334583309d32b62544682ee972b51a", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/67a2df3a62639eab2cc5906065e9805d4fd5dfc5", + "reference": "67a2df3a62639eab2cc5906065e9805d4fd5dfc5", "shasum": "" }, "require": { @@ -3251,15 +3349,27 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/comparator/issues", - "source": "https://github.com/sebastianbergmann/comparator/tree/4.0.8" + "source": "https://github.com/sebastianbergmann/comparator/tree/4.0.9" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/comparator", + "type": "tidelift" } ], - "time": "2022-09-14T12:41:17+00:00" + "time": "2025-08-10T06:51:50+00:00" }, { "name": "sebastian/complexity", @@ -3526,16 +3636,16 @@ }, { "name": "sebastian/global-state", - "version": "5.0.7", + "version": "5.0.8", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/global-state.git", - "reference": "bca7df1f32ee6fe93b4d4a9abbf69e13a4ada2c9" + "reference": "b6781316bdcd28260904e7cc18ec983d0d2ef4f6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/bca7df1f32ee6fe93b4d4a9abbf69e13a4ada2c9", - "reference": "bca7df1f32ee6fe93b4d4a9abbf69e13a4ada2c9", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/b6781316bdcd28260904e7cc18ec983d0d2ef4f6", + "reference": "b6781316bdcd28260904e7cc18ec983d0d2ef4f6", "shasum": "" }, "require": { @@ -3578,15 +3688,27 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/global-state/issues", - "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.7" + "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.8" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/global-state", + "type": "tidelift" } ], - "time": "2024-03-02T06:35:11+00:00" + "time": "2025-08-10T07:10:35+00:00" }, { "name": "sebastian/lines-of-code", @@ -3759,16 +3881,16 @@ }, { "name": "sebastian/recursion-context", - "version": "4.0.5", + "version": "4.0.6", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/recursion-context.git", - "reference": "e75bd0f07204fec2a0af9b0f3cfe97d05f92efc1" + "reference": "539c6691e0623af6dc6f9c20384c120f963465a0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/e75bd0f07204fec2a0af9b0f3cfe97d05f92efc1", - "reference": "e75bd0f07204fec2a0af9b0f3cfe97d05f92efc1", + "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/539c6691e0623af6dc6f9c20384c120f963465a0", + "reference": "539c6691e0623af6dc6f9c20384c120f963465a0", "shasum": "" }, "require": { @@ -3810,15 +3932,27 @@ "homepage": "https://github.com/sebastianbergmann/recursion-context", "support": { "issues": "https://github.com/sebastianbergmann/recursion-context/issues", - "source": "https://github.com/sebastianbergmann/recursion-context/tree/4.0.5" + "source": "https://github.com/sebastianbergmann/recursion-context/tree/4.0.6" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/recursion-context", + "type": "tidelift" } ], - "time": "2023-02-03T06:07:39+00:00" + "time": "2025-08-10T06:57:39+00:00" }, { "name": "sebastian/resource-operations", @@ -4121,7 +4255,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -4129,6 +4263,6 @@ "ext-pdo": "*", "ext-mbstring": "*" }, - "platform-dev": [], + "platform-dev": {}, "plugin-api-version": "2.6.0" } diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 1f2e8e306..b753ea523 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -749,7 +749,7 @@ abstract public function updateDocuments(Document $collection, Document $updates * @param array $changes * @return array */ - abstract public function createOrUpdateDocuments( + abstract public function upsertDocuments( Document $collection, string $attribute, array $changes diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index fc099178b..3c1c10a19 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -245,7 +245,7 @@ public function updateDocuments(Document $collection, Document $updates, array $ return $this->delegate(__FUNCTION__, \func_get_args()); } - public function createOrUpdateDocuments(Document $collection, string $attribute, array $changes): array + public function upsertDocuments(Document $collection, string $attribute, array $changes): array { return $this->delegate(__FUNCTION__, \func_get_args()); } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index cc241c445..9e08d4b93 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2090,7 +2090,7 @@ public function createDocuments(Document $collection, array $documents): array * @return array * @throws DatabaseException */ - public function createOrUpdateDocuments( + public function upsertDocuments( Document $collection, string $attribute, array $changes diff --git a/src/Database/Database.php b/src/Database/Database.php index 9b7a60ff0..019c51de5 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -60,7 +60,7 @@ class Database public const VAR_LINESTRING = 'linestring'; public const VAR_POLYGON = 'polygon'; - public const SPATIAL_TYPES = [self::VAR_POINT,self::VAR_LINESTRING, self::VAR_POLYGON]; + public const SPATIAL_TYPES = [self::VAR_POINT, self::VAR_LINESTRING, self::VAR_POLYGON]; // Index Types public const INDEX_KEY = 'key'; @@ -3826,7 +3826,7 @@ public function createDocument(string $collection, Document $document): Document * @param string $collection * @param array $documents * @param int $batchSize - * @param callable|null $onNext + * @param (callable(Document): void)|null $onNext * @return int * @throws AuthorizationException * @throws StructureException @@ -4283,7 +4283,7 @@ public function updateDocument(string $collection, string $id, Document $documen if ($document->offsetExists('$permissions')) { $originalPermissions = $old->getPermissions(); - $currentPermissions = $document->getPermissions(); + $currentPermissions = $document->getPermissions(); sort($originalPermissions); sort($currentPermissions); @@ -4473,8 +4473,8 @@ public function updateDocument(string $collection, string $id, Document $documen * @param Document $updates * @param array $queries * @param int $batchSize - * @param callable|null $onNext - * @param callable|null $onError + * @param (callable(Document $updated, Document $old): void)|null $onNext + * @param (callable(Throwable): void)|null $onError * @return int * @throws AuthorizationException * @throws ConflictException @@ -4592,17 +4592,17 @@ public function updateDocuments( array_merge($new, $queries), forPermission: Database::PERMISSION_UPDATE )); - + if (empty($batch)) { break; } - $currentPermissions = $updates->getPermissions(); + $old = array_map(fn ($doc) => clone $doc, $batch); + $currentPermissions = $updates->getPermissions(); sort($currentPermissions); $this->withTransaction(function () use ($collection, $updates, &$batch, $currentPermissions) { foreach ($batch as $index => $document) { - $skipPermissionsUpdate = true; if ($updates->offsetExists('$permissions')) { @@ -4647,13 +4647,12 @@ public function updateDocuments( ); }); - foreach ($batch as $doc) { + foreach ($batch as $index => $doc) { $doc->removeAttribute('$skipPermissionsUpdate'); - $this->purgeCachedDocument($collection->getId(), $doc->getId()); $doc = $this->decode($collection, $doc); try { - $onNext && $onNext($doc); + $onNext && $onNext($doc, $old[$index]); } catch (Throwable $th) { $onError ? $onError($th) : throw $th; } @@ -5069,28 +5068,58 @@ private function getJunctionCollection(Document $collection, Document $relatedCo : '_' . $relatedCollection->getSequence() . '_' . $collection->getSequence(); } + /** + * Create or update a document. + * + * @param string $collection + * @param Document $document + * @return Document + * @throws StructureException + * @throws Throwable + */ + public function upsertDocument( + string $collection, + Document $document, + ): Document { + $result = null; + + $this->upsertDocumentsWithIncrease( + $collection, + '', + [$document], + function (Document $doc) use (&$result) { + $result = $doc; + } + ); + + return $result; + } + /** * Create or update documents. * * @param string $collection * @param array $documents * @param int $batchSize - * @param callable|null $onNext + * @param (callable(Document): void)|null $onNext + * @param (callable(Throwable): void)|null $onError * @return int * @throws StructureException * @throws \Throwable */ - public function createOrUpdateDocuments( + public function upsertDocuments( string $collection, array $documents, int $batchSize = self::INSERT_BATCH_SIZE, ?callable $onNext = null, + ?callable $onError = null ): int { - return $this->createOrUpdateDocumentsWithIncrease( + return $this->upsertDocumentsWithIncrease( $collection, '', $documents, $onNext, + $onError, $batchSize ); } @@ -5108,11 +5137,12 @@ public function createOrUpdateDocuments( * @throws \Throwable * @throws Exception */ - public function createOrUpdateDocumentsWithIncrease( + public function upsertDocumentsWithIncrease( string $collection, string $attribute, array $documents, ?callable $onNext = null, + ?callable $onError = null, int $batchSize = self::INSERT_BATCH_SIZE ): int { if (empty($documents)) { @@ -5144,7 +5174,7 @@ public function createOrUpdateDocumentsWithIncrease( if ($document->offsetExists('$permissions')) { $originalPermissions = $old->getPermissions(); - $currentPermissions = $document->getPermissions(); + $currentPermissions = $document->getPermissions(); sort($originalPermissions); sort($currentPermissions); @@ -5274,7 +5304,7 @@ public function createOrUpdateDocumentsWithIncrease( /** * @var array $chunk */ - $batch = $this->withTransaction(fn () => Authorization::skip(fn () => $this->adapter->createOrUpdateDocuments( + $batch = $this->withTransaction(fn () => Authorization::skip(fn () => $this->adapter->upsertDocuments( $collection, $attribute, $chunk @@ -5972,8 +6002,8 @@ private function deleteCascade(Document $collection, Document $relatedCollection * @param string $collection * @param array $queries * @param int $batchSize - * @param callable|null $onNext - * @param callable|null $onError + * @param (callable(Document): void)|null $onNext + * @param (callable(Throwable): void)|null $onError * @return int * @throws AuthorizationException * @throws DatabaseException @@ -6520,7 +6550,7 @@ public static function addFilter(string $name, callable $encode, callable $decod public function encode(Document $collection, Document $document): Document { $attributes = $collection->getAttribute('attributes', []); - $internalDateAttributes = ['$createdAt','$updatedAt']; + $internalDateAttributes = ['$createdAt', '$updatedAt']; foreach ($this->getInternalAttributes() as $attribute) { $attributes[] = $attribute; } @@ -6937,7 +6967,7 @@ public static function convertQuery(Document $collection, Query $query): Query } } - if (! $attribute->isEmpty()) { + if (!$attribute->isEmpty()) { $query->setOnArray($attribute->getAttribute('array', false)); if ($attribute->getAttribute('type') == Database::VAR_DATETIME) { @@ -7195,7 +7225,7 @@ public function decodeSpatialData(string $wkt): array // POINT(x y) if (str_starts_with($upper, 'POINT(')) { $start = strpos($wkt, '(') + 1; - $end = strrpos($wkt, ')'); + $end = strrpos($wkt, ')'); $inside = substr($wkt, $start, $end - $start); $coords = explode(' ', trim($inside)); @@ -7205,7 +7235,7 @@ public function decodeSpatialData(string $wkt): array // LINESTRING(x1 y1, x2 y2, ...) if (str_starts_with($upper, 'LINESTRING(')) { $start = strpos($wkt, '(') + 1; - $end = strrpos($wkt, ')'); + $end = strrpos($wkt, ')'); $inside = substr($wkt, $start, $end - $start); $points = explode(',', $inside); @@ -7218,7 +7248,7 @@ public function decodeSpatialData(string $wkt): array // POLYGON((x1,y1),(x2,y2)) if (str_starts_with($upper, 'POLYGON((')) { $start = strpos($wkt, '((') + 2; - $end = strrpos($wkt, '))'); + $end = strrpos($wkt, '))'); $inside = substr($wkt, $start, $end - $start); $rings = explode('),(', $inside); diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php index 25df6c888..dc5f362f8 100644 --- a/src/Database/Mirror.php +++ b/src/Database/Mirror.php @@ -722,8 +722,8 @@ public function updateDocuments( $updates, $queries, $batchSize, - function ($doc) use ($onNext, &$modified) { - $onNext && $onNext($doc); + function ($doc, $old) use ($onNext, &$modified) { + $onNext && $onNext($doc, $old); $modified++; }, $onError @@ -781,10 +781,15 @@ function ($doc) use ($onNext, &$modified) { return $modified; } - public function createOrUpdateDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE, callable|null $onNext = null): int - { + public function upsertDocuments( + string $collection, + array $documents, + int $batchSize = Database::INSERT_BATCH_SIZE, + ?callable $onNext = null, + ?callable $onError = null, + ): int { $modified = 0; - $this->source->createOrUpdateDocuments( + $this->source->upsertDocuments( $collection, $documents, $batchSize, @@ -826,7 +831,7 @@ function ($doc) use ($onNext, &$modified) { $modified = $this->destination->withPreserveDates( fn () => - $this->destination->createOrUpdateDocuments( + $this->destination->upsertDocuments( $collection, $clones, $batchSize, diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index bf7f2a905..500966cbf 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -532,7 +532,7 @@ public function testSkipPermissions(): void Authorization::disable(); $results = []; - $count = $database->createOrUpdateDocuments( + $count = $database->upsertDocuments( __FUNCTION__, $documents, onNext: function ($doc) use (&$results) { @@ -594,7 +594,7 @@ public function testUpsertDocuments(): void ]; $results = []; - $count = $database->createOrUpdateDocuments( + $count = $database->upsertDocuments( __FUNCTION__, $documents, onNext: function ($doc) use (&$results) { @@ -637,7 +637,7 @@ public function testUpsertDocuments(): void $documents[1]->setAttribute('integer', 10); $results = []; - $count = $database->createOrUpdateDocuments(__FUNCTION__, $documents, onNext: function ($doc) use (&$results) { + $count = $database->upsertDocuments(__FUNCTION__, $documents, onNext: function ($doc) use (&$results) { $results[] = $doc; }); @@ -714,7 +714,7 @@ public function testUpsertDocumentsInc(): void $documents[0]->setAttribute('integer', 1); $documents[1]->setAttribute('integer', 1); - $database->createOrUpdateDocumentsWithIncrease( + $database->upsertDocumentsWithIncrease( collection: __FUNCTION__, attribute: 'integer', documents: $documents @@ -729,7 +729,7 @@ public function testUpsertDocumentsInc(): void $documents[0]->setAttribute('integer', -1); $documents[1]->setAttribute('integer', -1); - $database->createOrUpdateDocumentsWithIncrease( + $database->upsertDocumentsWithIncrease( collection: __FUNCTION__, attribute: 'integer', documents: $documents @@ -764,10 +764,10 @@ public function testUpsertDocumentsPermissions(): void ], ]); - $database->createOrUpdateDocuments(__FUNCTION__, [$document]); + $database->upsertDocuments(__FUNCTION__, [$document]); try { - $database->createOrUpdateDocuments(__FUNCTION__, [$document->setAttribute('string', 'updated')]); + $database->upsertDocuments(__FUNCTION__, [$document->setAttribute('string', 'updated')]); $this->fail('Failed to throw exception'); } catch (Exception $e) { $this->assertInstanceOf(AuthorizationException::class, $e); @@ -782,10 +782,10 @@ public function testUpsertDocumentsPermissions(): void ], ]); - $database->createOrUpdateDocuments(__FUNCTION__, [$document]); + $database->upsertDocuments(__FUNCTION__, [$document]); $results = []; - $count = $database->createOrUpdateDocuments( + $count = $database->upsertDocuments( __FUNCTION__, [$document->setAttribute('string', 'updated')], onNext: function ($doc) use (&$results) { @@ -806,7 +806,7 @@ public function testUpsertDocumentsPermissions(): void ], ]); - $database->createOrUpdateDocuments(__FUNCTION__, [$document]); + $database->upsertDocuments(__FUNCTION__, [$document]); $newPermissions = [ Permission::read(Role::any()), @@ -815,7 +815,7 @@ public function testUpsertDocumentsPermissions(): void ]; $results = []; - $count = $database->createOrUpdateDocuments( + $count = $database->upsertDocuments( __FUNCTION__, [$document->setAttribute('$permissions', $newPermissions)], onNext: function ($doc) use (&$results) { @@ -862,7 +862,7 @@ public function testUpsertDocumentsAttributeMismatch(): void ]); // Ensure missing optionals on new document is allowed - $docs = $database->createOrUpdateDocuments(__FUNCTION__, [ + $docs = $database->upsertDocuments(__FUNCTION__, [ $existingDocument->setAttribute('first', 'updated'), $newDocument, ]); @@ -874,7 +874,7 @@ public function testUpsertDocumentsAttributeMismatch(): void $this->assertEquals('', $newDocument->getAttribute('last')); try { - $database->createOrUpdateDocuments(__FUNCTION__, [ + $database->upsertDocuments(__FUNCTION__, [ $existingDocument->removeAttribute('first'), $newDocument ]); @@ -884,7 +884,7 @@ public function testUpsertDocumentsAttributeMismatch(): void } // Ensure missing optionals on existing document is allowed - $docs = $database->createOrUpdateDocuments(__FUNCTION__, [ + $docs = $database->upsertDocuments(__FUNCTION__, [ $existingDocument ->setAttribute('first', 'first') ->removeAttribute('last'), @@ -899,7 +899,7 @@ public function testUpsertDocumentsAttributeMismatch(): void $this->assertEquals('last', $newDocument->getAttribute('last')); // Ensure set null on existing document is allowed - $docs = $database->createOrUpdateDocuments(__FUNCTION__, [ + $docs = $database->upsertDocuments(__FUNCTION__, [ $existingDocument ->setAttribute('first', 'first') ->setAttribute('last', null), @@ -926,7 +926,7 @@ public function testUpsertDocumentsAttributeMismatch(): void ]); // Ensure mismatch of attribute orders is allowed - $docs = $database->createOrUpdateDocuments(__FUNCTION__, [ + $docs = $database->upsertDocuments(__FUNCTION__, [ $doc3, $doc4 ]); @@ -967,11 +967,11 @@ public function testUpsertDocumentsNoop(): void ], ]); - $count = static::getDatabase()->createOrUpdateDocuments(__FUNCTION__, [$document]); + $count = static::getDatabase()->upsertDocuments(__FUNCTION__, [$document]); $this->assertEquals(1, $count); // No changes, should return 0 - $count = static::getDatabase()->createOrUpdateDocuments(__FUNCTION__, [$document]); + $count = static::getDatabase()->upsertDocuments(__FUNCTION__, [$document]); $this->assertEquals(0, $count); } @@ -990,7 +990,7 @@ public function testUpsertDuplicateIds(): void $doc2 = new Document(['$id' => 'dup', 'num' => 2]); try { - $db->createOrUpdateDocuments(__FUNCTION__, [$doc1, $doc2]); + $db->upsertDocuments(__FUNCTION__, [$doc1, $doc2]); $this->fail('Failed to throw exception'); } catch (\Throwable $e) { $this->assertInstanceOf(DuplicateException::class, $e, $e->getMessage()); @@ -1032,7 +1032,7 @@ public function testUpsertMixedPermissionDelta(): void Permission::read(Role::any()) ]); - $db->createOrUpdateDocuments(__FUNCTION__, [$d1, $d2]); + $db->upsertDocuments(__FUNCTION__, [$d1, $d2]); $this->assertEquals([ Permission::read(Role::any()), @@ -5547,7 +5547,7 @@ public function testUpsertDateOperations(): void // Test 1: Upsert new document with custom createdAt $upsertResults = []; - $database->createOrUpdateDocuments($collection, [ + $database->upsertDocuments($collection, [ new Document([ '$id' => 'upsert1', '$permissions' => $permissions, @@ -5566,7 +5566,7 @@ public function testUpsertDateOperations(): void $upsertDoc1->setAttribute('string', 'upsert1_updated'); $upsertDoc1->setAttribute('$updatedAt', $updateDate); $updatedUpsertResults = []; - $database->createOrUpdateDocuments($collection, [$upsertDoc1], onNext: function ($doc) use (&$updatedUpsertResults) { + $database->upsertDocuments($collection, [$upsertDoc1], onNext: function ($doc) use (&$updatedUpsertResults) { $updatedUpsertResults[] = $doc; }); $updatedUpsertDoc1 = $updatedUpsertResults[0]; @@ -5576,7 +5576,7 @@ public function testUpsertDateOperations(): void // Test 3: Upsert new document with both custom dates $upsertResults2 = []; - $database->createOrUpdateDocuments($collection, [ + $database->upsertDocuments($collection, [ new Document([ '$id' => 'upsert2', '$permissions' => $permissions, @@ -5597,7 +5597,7 @@ public function testUpsertDateOperations(): void $upsertDoc2->setAttribute('$createdAt', $date3); $upsertDoc2->setAttribute('$updatedAt', $date3); $updatedUpsertResults2 = []; - $database->createOrUpdateDocuments($collection, [$upsertDoc2], onNext: function ($doc) use (&$updatedUpsertResults2) { + $database->upsertDocuments($collection, [$upsertDoc2], onNext: function ($doc) use (&$updatedUpsertResults2) { $updatedUpsertResults2[] = $doc; }); $updatedUpsertDoc2 = $updatedUpsertResults2[0]; @@ -5610,7 +5610,7 @@ public function testUpsertDateOperations(): void $customDate = '2000-01-01T10:00:00.000+00:00'; $upsertResults3 = []; - $database->createOrUpdateDocuments($collection, [ + $database->upsertDocuments($collection, [ new Document([ '$id' => 'upsert3', '$permissions' => $permissions, @@ -5631,7 +5631,7 @@ public function testUpsertDateOperations(): void $upsertDoc3->setAttribute('$createdAt', $customDate); $upsertDoc3->setAttribute('$updatedAt', $customDate); $updatedUpsertResults3 = []; - $database->createOrUpdateDocuments($collection, [$upsertDoc3], onNext: function ($doc) use (&$updatedUpsertResults3) { + $database->upsertDocuments($collection, [$upsertDoc3], onNext: function ($doc) use (&$updatedUpsertResults3) { $updatedUpsertResults3[] = $doc; }); $updatedUpsertDoc3 = $updatedUpsertResults3[0]; @@ -5671,7 +5671,7 @@ public function testUpsertDateOperations(): void ]; $bulkUpsertResults = []; - $database->createOrUpdateDocuments($collection, $upsertDocuments, onNext: function ($doc) use (&$bulkUpsertResults) { + $database->upsertDocuments($collection, $upsertDocuments, onNext: function ($doc) use (&$bulkUpsertResults) { $bulkUpsertResults[] = $doc; }); @@ -5738,7 +5738,7 @@ public function testUpsertDateOperations(): void $this->assertNotEmpty($doc->getAttribute('$updatedAt'), "updatedAt mismatch for $id"); } - // Test 11: Bulk upsert operations with createOrUpdateDocuments + // Test 11: Bulk upsert operations with upsertDocuments $upsertUpdateDocuments = []; foreach ($upsertDocuments as $doc) { $updatedDoc = clone $doc; @@ -5749,7 +5749,7 @@ public function testUpsertDateOperations(): void } $upsertUpdateResults = []; - $countUpsertUpdate = $database->createOrUpdateDocuments($collection, $upsertUpdateDocuments, onNext: function ($doc) use (&$upsertUpdateResults) { + $countUpsertUpdate = $database->upsertDocuments($collection, $upsertUpdateDocuments, onNext: function ($doc) use (&$upsertUpdateResults) { $upsertUpdateResults[] = $doc; }); $this->assertEquals(4, $countUpsertUpdate); @@ -5774,7 +5774,7 @@ public function testUpsertDateOperations(): void } $upsertDisabledResults = []; - $countUpsertDisabled = $database->createOrUpdateDocuments($collection, $upsertDisabledDocuments, onNext: function ($doc) use (&$upsertDisabledResults) { + $countUpsertDisabled = $database->upsertDocuments($collection, $upsertDisabledDocuments, onNext: function ($doc) use (&$upsertDisabledResults) { $upsertDisabledResults[] = $doc; }); $this->assertEquals(4, $countUpsertDisabled); @@ -5829,7 +5829,7 @@ public function testUpdateDocumentsCount(): void ]) ]; $upsertUpdateResults = []; - $count = $database->createOrUpdateDocuments($collectionName, $docs, onNext: function ($doc) use (&$upsertUpdateResults) { + $count = $database->upsertDocuments($collectionName, $docs, onNext: function ($doc) use (&$upsertUpdateResults) { $upsertUpdateResults[] = $doc; }); $this->assertCount(4, $upsertUpdateResults); diff --git a/tests/e2e/Adapter/Scopes/GeneralTests.php b/tests/e2e/Adapter/Scopes/GeneralTests.php index 795f4d096..1664273c1 100644 --- a/tests/e2e/Adapter/Scopes/GeneralTests.php +++ b/tests/e2e/Adapter/Scopes/GeneralTests.php @@ -512,7 +512,7 @@ public function testSharedTablesTenantPerDocument(): void $database ->setTenant(null) ->setTenantPerDocument(true) - ->createOrUpdateDocuments(__FUNCTION__, [new Document([ + ->upsertDocuments(__FUNCTION__, [new Document([ '$id' => $doc3Id, '$tenant' => 3, 'name' => 'Superman3', @@ -550,7 +550,7 @@ public function testSharedTablesTenantPerDocument(): void $database ->setTenant(null) ->setTenantPerDocument(true) - ->createOrUpdateDocuments(__FUNCTION__, [new Document([ + ->upsertDocuments(__FUNCTION__, [new Document([ '$id' => $doc4Id, '$tenant' => 4, 'name' => 'Superman4', @@ -582,7 +582,7 @@ public function testSharedTablesTenantPerDocument(): void $database ->setTenant(null) ->setTenantPerDocument(true) - ->createOrUpdateDocuments(__FUNCTION__, [new Document([ + ->upsertDocuments(__FUNCTION__, [new Document([ '$id' => $doc4Id, '$tenant' => 4, 'name' => 'Superman4 updated', diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index b1d3d9866..3d93d233e 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -1581,7 +1581,7 @@ public function testSpatialBulkOperation(): void ]], $document->getAttribute('area')); } - // Test 3: createOrUpdateDocuments with spatial data + // Test 3: upsertDocuments with spatial data $upsertDocuments = [ new Document([ '$id' => 'upsert1', @@ -1622,7 +1622,7 @@ public function testSpatialBulkOperation(): void ]; $upsertResults = []; - $upsertCount = $database->createOrUpdateDocuments($collectionName, $upsertDocuments, onNext: function ($doc) use (&$upsertResults) { + $upsertCount = $database->upsertDocuments($collectionName, $upsertDocuments, onNext: function ($doc) use (&$upsertResults) { $upsertResults[] = $doc; }); From 0c6e6931d21cbe754a40273b23d9d12b980dabc1 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 4 Sep 2025 23:46:08 +1200 Subject: [PATCH 26/69] Send old for upsert + delete --- src/Database/Database.php | 24 ++++++++++++++++-------- src/Database/Mirror.php | 8 ++++---- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 019c51de5..69b50afe1 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -4592,7 +4592,7 @@ public function updateDocuments( array_merge($new, $queries), forPermission: Database::PERMISSION_UPDATE )); - + if (empty($batch)) { break; } @@ -5101,7 +5101,7 @@ function (Document $doc) use (&$result) { * @param string $collection * @param array $documents * @param int $batchSize - * @param (callable(Document): void)|null $onNext + * @param (callable(Document, ?Document): void)|null $onNext * @param (callable(Throwable): void)|null $onError * @return int * @throws StructureException @@ -5130,7 +5130,8 @@ public function upsertDocuments( * @param string $collection * @param string $attribute * @param array $documents - * @param callable|null $onNext + * @param (callable(Document, ?Document): void)|null $onNext + * @param (callable(Throwable): void)|null $onError * @param int $batchSize * @return int * @throws StructureException @@ -5320,7 +5321,7 @@ public function upsertDocumentsWithIncrease( } } - foreach ($batch as $doc) { + foreach ($batch as $index => $doc) { if ($this->resolveRelationships) { $doc = $this->silent(fn () => $this->populateDocumentRelationships($collection, $doc)); } @@ -5335,7 +5336,13 @@ public function upsertDocumentsWithIncrease( $this->purgeCachedDocument($collection->getId(), $doc->getId()); } - $onNext && $onNext($doc); + $old = $chunk[$index]->getOld(); + + try { + $onNext && $onNext($doc, $old->isEmpty() ? null : $old); + } catch (\Throwable $th) { + $onError ? $onError($th) : throw $th; + } } } @@ -6002,7 +6009,7 @@ private function deleteCascade(Document $collection, Document $relatedCollection * @param string $collection * @param array $queries * @param int $batchSize - * @param (callable(Document): void)|null $onNext + * @param (callable(Document, Document): void)|null $onNext * @param (callable(Throwable): void)|null $onError * @return int * @throws AuthorizationException @@ -6095,6 +6102,7 @@ public function deleteDocuments( break; } + $old = array_map(fn ($doc) => clone $doc, $batch); $sequences = []; $permissionIds = []; @@ -6131,7 +6139,7 @@ public function deleteDocuments( ); }); - foreach ($batch as $document) { + foreach ($batch as $index => $document) { if ($this->getSharedTables() && $this->getTenantPerDocument()) { $this->withTenant($document->getTenant(), function () use ($collection, $document) { $this->purgeCachedDocument($collection->getId(), $document->getId()); @@ -6140,7 +6148,7 @@ public function deleteDocuments( $this->purgeCachedDocument($collection->getId(), $document->getId()); } try { - $onNext && $onNext($document); + $onNext && $onNext($document, $old[$index]); } catch (Throwable $th) { $onError ? $onError($th) : throw $th; } diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php index dc5f362f8..2ec18c197 100644 --- a/src/Database/Mirror.php +++ b/src/Database/Mirror.php @@ -793,8 +793,8 @@ public function upsertDocuments( $collection, $documents, $batchSize, - function ($doc) use ($onNext, &$modified) { - $onNext && $onNext($doc); + function ($doc, $old) use ($onNext, &$modified) { + $onNext && $onNext($doc, $old); $modified++; } ); @@ -911,8 +911,8 @@ public function deleteDocuments( $collection, $queries, $batchSize, - function ($doc) use (&$modified, $onNext) { - $onNext && $onNext($doc); + function ($doc, $old) use (&$modified, $onNext) { + $onNext && $onNext($doc, $old); $modified++; }, $onError From 48220cac848b59d7c448912c98ad73aba131f1db Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 5 Sep 2025 00:15:38 +1200 Subject: [PATCH 27/69] Update src/Database/Database.php Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/Database/Database.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 69b50afe1..eca06315f 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -5087,11 +5087,15 @@ public function upsertDocument( $collection, '', [$document], - function (Document $doc) use (&$result) { + function (Document $doc, ?Document $_old = null) use (&$result) { $result = $doc; } ); + if ($result === null) { + // No-op (unchanged): return the current persisted doc + $result = $this->getDocument($collection, $document->getId()); + } return $result; } From d3549853250354007f3975419573eecf977e0a5a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 5 Sep 2025 00:20:57 +1200 Subject: [PATCH 28/69] Improve mirror callbacks --- src/Database/Mirror.php | 55 ++++++++++++++--------------------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php index 2ec18c197..dac23a063 100644 --- a/src/Database/Mirror.php +++ b/src/Database/Mirror.php @@ -589,17 +589,14 @@ public function createDocuments( array $documents, int $batchSize = self::INSERT_BATCH_SIZE, ?callable $onNext = null, + ?callable $onError = null, ): int { - $modified = 0; - - $this->source->createDocuments( + $modified = $this->source->createDocuments( $collection, $documents, $batchSize, - function ($doc) use ($onNext, &$modified) { - $onNext && $onNext($doc); - $modified++; - }, + $onNext, + $onError, ); if ( @@ -638,7 +635,6 @@ function ($doc) use ($onNext, &$modified) { $collection, $clones, $batchSize, - null, ) ); @@ -715,18 +711,13 @@ public function updateDocuments( ?callable $onNext = null, ?callable $onError = null, ): int { - $modified = 0; - - $this->source->updateDocuments( + $modified = $this->source->updateDocuments( $collection, $updates, $queries, $batchSize, - function ($doc, $old) use ($onNext, &$modified) { - $onNext && $onNext($doc, $old); - $modified++; - }, - $onError + $onNext, + $onError, ); if ( @@ -754,14 +745,13 @@ function ($doc, $old) use ($onNext, &$modified) { ); } - $modified = $this->destination->withPreserveDates( + $this->destination->withPreserveDates( fn () => $this->destination->updateDocuments( $collection, $clone, $queries, $batchSize, - null, ) ); @@ -788,15 +778,12 @@ public function upsertDocuments( ?callable $onNext = null, ?callable $onError = null, ): int { - $modified = 0; - $this->source->upsertDocuments( + $modified = $this->source->upsertDocuments( $collection, $documents, $batchSize, - function ($doc, $old) use ($onNext, &$modified) { - $onNext && $onNext($doc, $old); - $modified++; - } + $onNext, + $onError, ); if ( @@ -829,13 +816,12 @@ function ($doc, $old) use ($onNext, &$modified) { $clones[] = $clone; } - $modified = $this->destination->withPreserveDates( + $this->destination->withPreserveDates( fn () => $this->destination->upsertDocuments( $collection, $clones, $batchSize, - null, ) ); @@ -850,8 +836,9 @@ function ($doc, $old) use ($onNext, &$modified) { } } } catch (\Throwable $err) { - $this->logError('createDocuments', $err); + $this->logError('upsertDocuments', $err); } + return $modified; } @@ -905,17 +892,12 @@ public function deleteDocuments( ?callable $onNext = null, ?callable $onError = null, ): int { - $modified = 0; - - $this->source->deleteDocuments( + $modified = $this->source->deleteDocuments( $collection, $queries, $batchSize, - function ($doc, $old) use (&$modified, $onNext) { - $onNext && $onNext($doc, $old); - $modified++; - }, - $onError + $onNext, + $onError, ); if ( @@ -940,11 +922,10 @@ function ($doc, $old) use (&$modified, $onNext) { ); } - $modified = $this->destination->deleteDocuments( + $this->destination->deleteDocuments( $collection, $queries, $batchSize, - null, ); foreach ($this->writeFilters as $filter) { From 55d04ee67d5c93a5f6f865ffaf545a121ef0ae6d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 5 Sep 2025 00:28:09 +1200 Subject: [PATCH 29/69] Fix create docs on error --- src/Database/Database.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index eca06315f..2fec661a1 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -3827,6 +3827,7 @@ public function createDocument(string $collection, Document $document): Document * @param array $documents * @param int $batchSize * @param (callable(Document): void)|null $onNext + * @param (callable(Throwable): void)|null $onError * @return int * @throws AuthorizationException * @throws StructureException @@ -3838,6 +3839,7 @@ public function createDocuments( array $documents, int $batchSize = self::INSERT_BATCH_SIZE, ?callable $onNext = null, + ?callable $onError = null, ): int { if (!$this->adapter->getSharedTables() && $this->adapter->getTenantPerDocument()) { throw new DatabaseException('Shared tables must be enabled if tenant per document is enabled.'); @@ -3914,7 +3916,13 @@ public function createDocuments( $document = $this->casting($collection, $document); $document = $this->decode($collection, $document); - $onNext && $onNext($document); + + try { + $onNext && $onNext($document); + } catch (\Throwable $e) { + $onError ? $onError($e) : throw $e; + } + $modified++; } } From b9070a2fcbb213ee8f12b0d59c40c6a2b3e3741c Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 5 Sep 2025 18:52:30 +0530 Subject: [PATCH 30/69] updated spatial get type --- src/Database/Validator/Spatial.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Validator/Spatial.php b/src/Database/Validator/Spatial.php index 2470562cb..8845b6550 100644 --- a/src/Database/Validator/Spatial.php +++ b/src/Database/Validator/Spatial.php @@ -141,7 +141,7 @@ public function isArray(): bool public function getType(): string { - return 'spatial'; + return self::TYPE_ARRAY; } /** From bf1aad026305d30517da9199b5b4cbc125a1d849 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sun, 7 Sep 2025 20:31:17 +0530 Subject: [PATCH 31/69] reduced the spatial query dimension to avoid nested array on the user sides --- src/Database/Query.php | 16 ++++++------ tests/e2e/Adapter/Scopes/SpatialTests.php | 32 +++++++++++------------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/Database/Query.php b/src/Database/Query.php index 5df9d6585..d8f1557d9 100644 --- a/src/Database/Query.php +++ b/src/Database/Query.php @@ -986,7 +986,7 @@ public static function distanceLessThan(string $attribute, array $values, int|fl */ public static function intersects(string $attribute, array $values): self { - return new self(self::TYPE_INTERSECTS, $attribute, $values); + return new self(self::TYPE_INTERSECTS, $attribute, [$values]); } /** @@ -998,7 +998,7 @@ public static function intersects(string $attribute, array $values): self */ public static function notIntersects(string $attribute, array $values): self { - return new self(self::TYPE_NOT_INTERSECTS, $attribute, $values); + return new self(self::TYPE_NOT_INTERSECTS, $attribute, [$values]); } /** @@ -1010,7 +1010,7 @@ public static function notIntersects(string $attribute, array $values): self */ public static function crosses(string $attribute, array $values): self { - return new self(self::TYPE_CROSSES, $attribute, $values); + return new self(self::TYPE_CROSSES, $attribute, [$values]); } /** @@ -1022,7 +1022,7 @@ public static function crosses(string $attribute, array $values): self */ public static function notCrosses(string $attribute, array $values): self { - return new self(self::TYPE_NOT_CROSSES, $attribute, $values); + return new self(self::TYPE_NOT_CROSSES, $attribute, [$values]); } /** @@ -1034,7 +1034,7 @@ public static function notCrosses(string $attribute, array $values): self */ public static function overlaps(string $attribute, array $values): self { - return new self(self::TYPE_OVERLAPS, $attribute, $values); + return new self(self::TYPE_OVERLAPS, $attribute, [$values]); } /** @@ -1046,7 +1046,7 @@ public static function overlaps(string $attribute, array $values): self */ public static function notOverlaps(string $attribute, array $values): self { - return new self(self::TYPE_NOT_OVERLAPS, $attribute, $values); + return new self(self::TYPE_NOT_OVERLAPS, $attribute, [$values]); } /** @@ -1058,7 +1058,7 @@ public static function notOverlaps(string $attribute, array $values): self */ public static function touches(string $attribute, array $values): self { - return new self(self::TYPE_TOUCHES, $attribute, $values); + return new self(self::TYPE_TOUCHES, $attribute, [$values]); } /** @@ -1070,6 +1070,6 @@ public static function touches(string $attribute, array $values): self */ public static function notTouches(string $attribute, array $values): self { - return new self(self::TYPE_NOT_TOUCHES, $attribute, $values); + return new self(self::TYPE_NOT_TOUCHES, $attribute, [$values]); } } diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index b1d3d9866..a095c9027 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -135,8 +135,8 @@ public function testSpatialTypeDocuments(): void 'notEquals' => Query::notEqual('pointAttr', [[1.0, 1.0]]), 'distanceEqual' => Query::distanceEqual('pointAttr', [5.0, 5.0], 1.4142135623730951), 'distanceNotEqual' => Query::distanceNotEqual('pointAttr', [1.0, 1.0], 0.0), - 'intersects' => Query::intersects('pointAttr', [[6.0, 6.0]]), - 'notIntersects' => Query::notIntersects('pointAttr', [[1.0, 1.0]]) + 'intersects' => Query::intersects('pointAttr', [6.0, 6.0]), + 'notIntersects' => Query::notIntersects('pointAttr', [1.0, 1.0]) ]; foreach ($pointQueries as $queryType => $query) { @@ -151,8 +151,8 @@ public function testSpatialTypeDocuments(): void 'notContains' => Query::notContains('lineAttr', [[5.0, 6.0]]), // Point not on the line 'equals' => query::equal('lineAttr', [[[1.0, 2.0], [3.0, 4.0]]]), // Exact same linestring 'notEquals' => query::notEqual('lineAttr', [[[5.0, 6.0], [7.0, 8.0]]]), // Different linestring - 'intersects' => Query::intersects('lineAttr', [[1.0, 2.0]]), // Point on the line should intersect - 'notIntersects' => Query::notIntersects('lineAttr', [[5.0, 6.0]]) // Point not on the line should not intersect + 'intersects' => Query::intersects('lineAttr', [1.0, 2.0]), // Point on the line should intersect + 'notIntersects' => Query::notIntersects('lineAttr', [5.0, 6.0]) // Point not on the line should not intersect ]; foreach ($lineQueries as $queryType => $query) { @@ -182,12 +182,12 @@ public function testSpatialTypeDocuments(): void $polyQueries = [ 'contains' => Query::contains('polyAttr', [[5.0, 5.0]]), // Point inside polygon 'notContains' => Query::notContains('polyAttr', [[15.0, 15.0]]), // Point outside polygon - 'intersects' => Query::intersects('polyAttr', [[5.0, 5.0]]), // Point inside polygon should intersect - 'notIntersects' => Query::notIntersects('polyAttr', [[15.0, 15.0]]), // Point outside polygon should not intersect + 'intersects' => Query::intersects('polyAttr', [5.0, 5.0]), // Point inside polygon should intersect + 'notIntersects' => Query::notIntersects('polyAttr', [15.0, 15.0]), // Point outside polygon should not intersect 'equals' => query::equal('polyAttr', [[[[0.0, 0.0], [0.0, 10.0], [10.0, 10.0], [0.0, 0.0]]]]), // Exact same polygon 'notEquals' => query::notEqual('polyAttr', [[[[20.0, 20.0], [20.0, 30.0], [30.0, 30.0], [20.0, 20.0]]]]), // Different polygon - 'overlaps' => Query::overlaps('polyAttr', [[[[5.0, 5.0], [5.0, 15.0], [15.0, 15.0], [15.0, 5.0], [5.0, 5.0]]]]), // Overlapping polygon - 'notOverlaps' => Query::notOverlaps('polyAttr', [[[[20.0, 20.0], [20.0, 30.0], [30.0, 30.0], [30.0, 20.0], [20.0, 20.0]]]]) // Non-overlapping polygon + 'overlaps' => Query::overlaps('polyAttr', [[[5.0, 5.0], [5.0, 15.0], [15.0, 15.0], [15.0, 5.0], [5.0, 5.0]]]), // Overlapping polygon + 'notOverlaps' => Query::notOverlaps('polyAttr', [[[20.0, 20.0], [20.0, 30.0], [30.0, 30.0], [30.0, 20.0], [20.0, 20.0]]]) // Non-overlapping polygon ]; foreach ($polyQueries as $queryType => $query) { @@ -935,8 +935,8 @@ public function testComplexGeometricShapes(): void // Test rectangle intersects with another rectangle $overlappingRect = $database->find($collectionName, [ Query::and([ - Query::intersects('rectangle', [[[15, 5], [15, 15], [25, 15], [25, 5], [15, 5]]]), - Query::notTouches('rectangle', [[[15, 5], [15, 15], [25, 15], [25, 5], [15, 5]]]) + Query::intersects('rectangle', [[15, 5], [15, 15], [25, 15], [25, 5], [15, 5]]), + Query::notTouches('rectangle', [[15, 5], [15, 15], [25, 15], [25, 5], [15, 5]]) ]), ], Database::PERMISSION_READ); $this->assertNotEmpty($overlappingRect); @@ -1042,7 +1042,7 @@ public function testComplexGeometricShapes(): void ], Database::PERMISSION_READ); } else { $exactSquare = $database->find($collectionName, [ - Query::intersects('square', [[[5, 5], [5, 15], [15, 15], [15, 5], [5, 5]]]) + Query::intersects('square', [[5, 5], [5, 15], [15, 15], [15, 5], [5, 5]]) ], Database::PERMISSION_READ); } $this->assertNotEmpty($exactSquare); @@ -1089,13 +1089,13 @@ public function testComplexGeometricShapes(): void // Test triangle intersects with point $intersectingTriangle = $database->find($collectionName, [ - Query::intersects('triangle', [[25, 10]]) // Point inside triangle should intersect + Query::intersects('triangle', [25, 10]) // Point inside triangle should intersect ], Database::PERMISSION_READ); $this->assertNotEmpty($intersectingTriangle); // Test triangle doesn't intersect with distant point $nonIntersectingTriangle = $database->find($collectionName, [ - Query::notIntersects('triangle', [[100, 100]]) // Distant point should not intersect + Query::notIntersects('triangle', [100, 100]) // Distant point should not intersect ], Database::PERMISSION_READ); $this->assertNotEmpty($nonIntersectingTriangle); @@ -1159,7 +1159,7 @@ public function testComplexGeometricShapes(): void // Test complex polygon intersects with line $intersectingLine = $database->find($collectionName, [ - Query::intersects('complex_polygon', [[[0, 10], [20, 10]]]) // Horizontal line through L-shape + Query::intersects('complex_polygon', [[0, 10], [20, 10]]) // Horizontal line through L-shape ], Database::PERMISSION_READ); $this->assertNotEmpty($intersectingLine); @@ -1181,13 +1181,13 @@ public function testComplexGeometricShapes(): void // Test linestring intersects with point $intersectingPoint = $database->find($collectionName, [ - Query::intersects('multi_linestring', [[10, 10]]) // Point on diagonal line + Query::intersects('multi_linestring', [10, 10]) // Point on diagonal line ], Database::PERMISSION_READ); $this->assertNotEmpty($intersectingPoint); // Test linestring intersects with a horizontal line coincident at y=20 $touchingLine = $database->find($collectionName, [ - Query::intersects('multi_linestring', [[[0, 20], [20, 20]]]) + Query::intersects('multi_linestring', [[0, 20], [20, 20]]) ], Database::PERMISSION_READ); $this->assertNotEmpty($touchingLine); From 46f1a467c0659b953af40b7387096a643aadf787 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 8 Sep 2025 17:10:57 +0530 Subject: [PATCH 32/69] * Add support for distance calculation between multidimension geometries in meters * Exprted a method from spatial validator to know the type of the spatial object --- src/Database/Adapter.php | 7 + src/Database/Adapter/MariaDB.php | 28 +++- src/Database/Adapter/MySQL.php | 13 +- src/Database/Adapter/Pool.php | 9 + src/Database/Adapter/Postgres.php | 15 +- src/Database/Adapter/SQL.php | 7 + src/Database/Adapter/SQLite.php | 10 ++ src/Database/Validator/Spatial.php | 5 + tests/e2e/Adapter/Scopes/SpatialTests.php | 191 ++++++++++++++++++++++ 9 files changed, 276 insertions(+), 9 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 1f2e8e306..5e416d036 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -1077,6 +1077,13 @@ abstract public function getSupportForSpatialIndexOrder(): bool; */ abstract public function getSupportForBoundaryInclusiveContains(): bool; + /** + * Does the adapter support calculating distance(in meters) between multidimension geometry(line, polygon,etc)? + * + * @return bool + */ + abstract public function getSupportForDistanceBetweenMultiDimensionGeometryInMeters(): bool; + /** * Get current attribute count from collection document * diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index d9ea2d130..cb9cd3d84 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1355,6 +1355,7 @@ public function deleteDocument(string $collection, string $id): bool /** * Handle distance spatial queries * + * @param string $spatialAttributeType * @param Query $query * @param array $binds * @param string $attribute @@ -1362,10 +1363,11 @@ public function deleteDocument(string $collection, string $id): bool * @param string $placeholder * @return string */ - protected function handleDistanceSpatialQueries(Query $query, array &$binds, string $attribute, string $alias, string $placeholder): string + protected function handleDistanceSpatialQueries(string $spatialAttributeType, Query $query, array &$binds, string $attribute, string $alias, string $placeholder): string { $distanceParams = $query->getValues()[0]; - $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($distanceParams[0]); + $wkt = $this->convertArrayToWKT($distanceParams[0]); + $binds[":{$placeholder}_0"] = $wkt; $binds[":{$placeholder}_1"] = $distanceParams[1]; $useMeters = isset($distanceParams[2]) && $distanceParams[2] === true; @@ -1388,6 +1390,11 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str } if ($useMeters) { + $wktType = $this->getSpatialTypeFromWKT($wkt); + $attrType = strtolower($spatialAttributeType); + if ($wktType != Database::VAR_POINT || $attrType != Database::VAR_POINT) { + throw new DatabaseException('Distance in meters is not supported between '.$attrType . ' and '. $wkt); + } return "ST_DISTANCE_SPHERE({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0), 6371000) {$operator} :{$placeholder}_1"; } return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0)) {$operator} :{$placeholder}_1"; @@ -1396,6 +1403,7 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str /** * Handle spatial queries * + * @param string $type * @param Query $query * @param array $binds * @param string $attribute @@ -1403,7 +1411,7 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str * @param string $placeholder * @return string */ - protected function handleSpatialQueries(Query $query, array &$binds, string $attribute, string $alias, string $placeholder): string + protected function handleSpatialQueries(string $type, Query $query, array &$binds, string $attribute, string $alias, string $placeholder): string { switch ($query->getMethod()) { case Query::TYPE_CROSSES: @@ -1418,7 +1426,7 @@ protected function handleSpatialQueries(Query $query, array &$binds, string $att case Query::TYPE_DISTANCE_NOT_EQUAL: case Query::TYPE_DISTANCE_GREATER_THAN: case Query::TYPE_DISTANCE_LESS_THAN: - return $this->handleDistanceSpatialQueries($query, $binds, $attribute, $alias, $placeholder); + return $this->handleDistanceSpatialQueries($type, $query, $binds, $attribute, $alias, $placeholder); case Query::TYPE_INTERSECTS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); @@ -1487,7 +1495,7 @@ protected function getSQLCondition(Query $query, array &$binds, array $attribute $attributeType = $this->getAttributeType($query->getAttribute(), $attributes); if (in_array($attributeType, Database::SPATIAL_TYPES)) { - return $this->handleSpatialQueries($query, $binds, $attribute, $alias, $placeholder); + return $this->handleSpatialQueries($attributeType, $query, $binds, $attribute, $alias, $placeholder); } switch ($query->getMethod()) { @@ -1868,4 +1876,14 @@ public function getSupportForSpatialIndexOrder(): bool { return true; } + + /** + * Does the adapter support calculating distance(in meters) between multidimension geometry(line, polygon,etc)? + * + * @return bool + */ + public function getSupportForDistanceBetweenMultiDimensionGeometryInMeters(): bool + { + return false; + } } diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index 682ce18ef..332a9b064 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -82,6 +82,7 @@ public function getSizeOfCollectionOnDisk(string $collection): int /** * Handle distance spatial queries * + * @param string $spatialAttributeType * @param Query $query * @param array $binds * @param string $attribute @@ -89,7 +90,7 @@ public function getSizeOfCollectionOnDisk(string $collection): int * @param string $placeholder * @return string */ - protected function handleDistanceSpatialQueries(Query $query, array &$binds, string $attribute, string $alias, string $placeholder): string + protected function handleDistanceSpatialQueries(string $spatialAttributeType, Query $query, array &$binds, string $attribute, string $alias, string $placeholder): string { $distanceParams = $query->getValues()[0]; $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($distanceParams[0]); @@ -173,4 +174,14 @@ public function getSupportForSpatialIndexOrder(): bool { return false; } + + /** + * Does the adapter support calculating distance(in meters) between multidimension geometry(line, polygon,etc)? + * + * @return bool + */ + public function getSupportForDistanceBetweenMultiDimensionGeometryInMeters(): bool + { + return true; + } } diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index fc099178b..636e37555 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -530,4 +530,13 @@ public function getSupportForSpatialIndexOrder(): bool { return $this->delegate(__FUNCTION__, \func_get_args()); } + /** + * Does the adapter support calculating distance(in meters) between multidimension geometry(line, polygon,etc)? + * + * @return bool + */ + public function getSupportForDistanceBetweenMultiDimensionGeometryInMeters(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 2ace9f555..b183c8c4e 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1488,9 +1488,8 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str } if ($meters) { - // Transform both attribute and input geometry to 3857 (meters) for distance calculation - $attr = "ST_Transform({$alias}.{$attribute}, 3857)"; - $geom = "ST_Transform(ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "), 3857)"; + $attr = "({$alias}.{$attribute}::geography)"; + $geom = "ST_SetSRID(ST_GeomFromText(:{$placeholder}_0), " . Database::SRID . ")::geography"; return "ST_Distance({$attr}, {$geom}) {$operator} :{$placeholder}_1"; } @@ -1982,4 +1981,14 @@ public function getSupportForSpatialIndexOrder(): bool { return false; } + + /** + * Does the adapter support calculating distance(in meters) between multidimension geometry(line, polygon,etc)? + * + * @return bool + */ + public function getSupportForDistanceBetweenMultiDimensionGeometryInMeters(): bool + { + return true; + } } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index cc241c445..3f654b0a9 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2657,4 +2657,11 @@ public function sum(Document $collection, string $attribute, array $queries = [] return $result['sum'] ?? 0; } + + public function getSpatialTypeFromWKT(string $wkt): string + { + $wkt = trim($wkt); + $pos = strpos($wkt, '('); + return strtolower(trim(substr($wkt, 0, $pos))); + } } diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index e9a2800aa..4f9a2afdd 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -1263,4 +1263,14 @@ public function getSupportForBoundaryInclusiveContains(): bool { return false; } + + /** + * Does the adapter support calculating distance(in meters) between multidimension geometry(line, polygon,etc)? + * + * @return bool + */ + public function getSupportForDistanceBetweenMultiDimensionGeometryInMeters(): bool + { + return false; + } } diff --git a/src/Database/Validator/Spatial.php b/src/Database/Validator/Spatial.php index 8845b6550..ab3bd8be4 100644 --- a/src/Database/Validator/Spatial.php +++ b/src/Database/Validator/Spatial.php @@ -144,6 +144,11 @@ public function getType(): string return self::TYPE_ARRAY; } + public function getSptialType(): string + { + return $this->spatialType; + } + /** * Main validation entrypoint */ diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index b1d3d9866..7bb361ed1 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -2158,4 +2158,195 @@ public function testSpatialDistanceInMeter(): void $database->deleteCollection($collectionName); } } + + public function testSpatialDistanceInMeterForMultiDimensionGeometry(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + if (!$database->getAdapter()->getSupportForSpatialAttributes()) { + $this->markTestSkipped('Adapter does not support spatial attributes'); + } + + if (!$database->getAdapter()->getSupportForDistanceBetweenMultiDimensionGeometryInMeters()) { + $this->markTestSkipped('Adapter does not support spatial distance(in meter) for multidimension'); + } + + $multiCollection = 'spatial_distance_meters_multi_'; + try { + $database->createCollection($multiCollection); + + // Create spatial attributes + $this->assertEquals(true, $database->createAttribute($multiCollection, 'loc', Database::VAR_POINT, 0, true)); + $this->assertEquals(true, $database->createAttribute($multiCollection, 'line', Database::VAR_LINESTRING, 0, true)); + $this->assertEquals(true, $database->createAttribute($multiCollection, 'poly', Database::VAR_POLYGON, 0, true)); + + // Create indexes + $this->assertEquals(true, $database->createIndex($multiCollection, 'idx_loc', Database::INDEX_SPATIAL, ['loc'])); + $this->assertEquals(true, $database->createIndex($multiCollection, 'idx_line', Database::INDEX_SPATIAL, ['line'])); + $this->assertEquals(true, $database->createIndex($multiCollection, 'idx_poly', Database::INDEX_SPATIAL, ['poly'])); + + // Geometry sets: near origin and far east + $docNear = $database->createDocument($multiCollection, new Document([ + '$id' => 'near', + 'loc' => [0.0000, 0.0000], + 'line' => [[0.0000, 0.0000], [0.0010, 0.0000]], // ~111m + 'poly' => [[ + [-0.0010, -0.0010], + [-0.0010, 0.0010], + [ 0.0010, 0.0010], + [ 0.0010, -0.0010], + [-0.0010, -0.0010] // closed + ]], + '$permissions' => [Permission::read(Role::any()), Permission::update(Role::any())] + ])); + + $docFar = $database->createDocument($multiCollection, new Document([ + '$id' => 'far', + 'loc' => [0.2000, 0.0000], // ~22 km east + 'line' => [[0.2000, 0.0000], [0.2020, 0.0000]], + 'poly' => [[ + [0.1980, -0.0020], + [0.1980, 0.0020], + [0.2020, 0.0020], + [0.2020, -0.0020], + [0.1980, -0.0020] // closed + ]], + '$permissions' => [Permission::read(Role::any()), Permission::update(Role::any())] + ])); + + $this->assertInstanceOf(Document::class, $docNear); + $this->assertInstanceOf(Document::class, $docFar); + + // polygon vs polygon (~1 km from near, ~22 km from far) + $polyPolyWithin3km = $database->find($multiCollection, [ + Query::distanceLessThan('poly', [[ + [0.0080, -0.0010], + [0.0080, 0.0010], + [0.0110, 0.0010], + [0.0110, -0.0010], + [0.0080, -0.0010] // closed + ]], 3000, true) + ], Database::PERMISSION_READ); + $this->assertCount(1, $polyPolyWithin3km); + $this->assertEquals('near', $polyPolyWithin3km[0]->getId()); + + $polyPolyGreater3km = $database->find($multiCollection, [ + Query::distanceGreaterThan('poly', [[ + [0.0080, -0.0010], + [0.0080, 0.0010], + [0.0110, 0.0010], + [0.0110, -0.0010], + [0.0080, -0.0010] // closed + ]], 3000, true) + ], Database::PERMISSION_READ); + $this->assertCount(1, $polyPolyGreater3km); + $this->assertEquals('far', $polyPolyGreater3km[0]->getId()); + + // point vs polygon (~0 km near, ~22 km far) + $ptPolyWithin500 = $database->find($multiCollection, [ + Query::distanceLessThan('loc', [[ + [-0.0010, -0.0010], + [-0.0010, 0.0020], + [ 0.0020, 0.0020], + [-0.0010, -0.0010] + ]], 500, true) + ], Database::PERMISSION_READ); + $this->assertCount(1, $ptPolyWithin500); + $this->assertEquals('near', $ptPolyWithin500[0]->getId()); + + $ptPolyGreater500 = $database->find($multiCollection, [ + Query::distanceGreaterThan('loc', [[ + [-0.0010, -0.0010], + [-0.0010, 0.0020], + [ 0.0020, 0.0020], + [-0.0010, -0.0010] + ]], 500, true) + ], Database::PERMISSION_READ); + $this->assertCount(1, $ptPolyGreater500); + $this->assertEquals('far', $ptPolyGreater500[0]->getId()); + + // Zero-distance checks + $lineEqualZero = $database->find($multiCollection, [ + Query::distanceEqual('line', [[0.0000, 0.0000], [0.0010, 0.0000]], 0, true) + ], Database::PERMISSION_READ); + $this->assertNotEmpty($lineEqualZero); + $this->assertEquals('near', $lineEqualZero[0]->getId()); + + $polyEqualZero = $database->find($multiCollection, [ + Query::distanceEqual('poly', [[ + [-0.0010, -0.0010], + [-0.0010, 0.0010], + [ 0.0010, 0.0010], + [ 0.0010, -0.0010], + [-0.0010, -0.0010] + ]], 0, true) + ], Database::PERMISSION_READ); + $this->assertNotEmpty($polyEqualZero); + $this->assertEquals('near', $polyEqualZero[0]->getId()); + + } finally { + $database->deleteCollection($multiCollection); + } + } + + public function testSpatialDistanceInMeterError() + { + /** @var Database $database */ + $database = static::getDatabase(); + if (!$database->getAdapter()->getSupportForSpatialAttributes()) { + $this->markTestSkipped('Adapter does not support spatial attributes'); + } + + if ($database->getAdapter()->getSupportForDistanceBetweenMultiDimensionGeometryInMeters()) { + $this->markTestSkipped('Adapter supports spatial distance (in meter) for multidimension geometries'); + } + + $collection = 'spatial_distance_error_test'; + $database->createCollection($collection); + $this->assertEquals(true, $database->createAttribute($collection, 'loc', Database::VAR_POINT, 0, true)); + $this->assertEquals(true, $database->createAttribute($collection, 'line', Database::VAR_LINESTRING, 0, true)); + $this->assertEquals(true, $database->createAttribute($collection, 'poly', Database::VAR_POLYGON, 0, true)); + + $doc = $database->createDocument($collection, new Document([ + '$id' => 'doc1', + 'loc' => [0.0, 0.0], + 'line' => [[0.0, 0.0], [0.001, 0.0]], + 'poly' => [[[ -0.001, -0.001 ], [ -0.001, 0.001 ], [ 0.001, 0.001 ], [ -0.001, -0.001 ]]], + '$permissions' => [] + ])); + $this->assertInstanceOf(Document::class, $doc); + + // Invalid geometry pairs (all combinations except POINT vs POINT) + $cases = [ + // Point compared against wrong types + ['attr' => 'line', 'geom' => [0.002, 0.0], 'msg' => 'Point vs LineString'], + ['attr' => 'poly', 'geom' => [0.002, 0.0], 'msg' => 'Point vs Polygon'], + + // LineString compared against wrong types + ['attr' => 'loc', 'geom' => [[0.0, 0.0], [0.001, 0.001]], 'msg' => 'LineString vs Point'], + ['attr' => 'poly', 'geom' => [[0.0, 0.0], [0.001, 0.001]], 'msg' => 'LineString vs Polygon'], + + // Polygon compared against wrong types + ['attr' => 'loc', 'geom' => [[[0.0, 0.0], [0.001, 0.0], [0.001, 0.001], [0.0, 0.0]]], 'msg' => 'Polygon vs Point'], + ['attr' => 'line', 'geom' => [[[0.0, 0.0], [0.001, 0.0], [0.001, 0.001], [0.0, 0.0]]], 'msg' => 'Polygon vs LineString'], + + // Polygon vs Polygon (still invalid for "meters") + ['attr' => 'poly', 'geom' => [[[0.002, -0.001], [0.002, 0.001], [0.004, 0.001], [0.002, -0.001]]], 'msg' => 'Polygon vs Polygon'], + + // LineString vs LineString (invalid for "meters") + ['attr' => 'line', 'geom' => [[0.002, 0.0], [0.003, 0.0]], 'msg' => 'LineString vs LineString'], + ]; + + foreach ($cases as $case) { + try { + $database->find($collection, [ + Query::distanceLessThan($case['attr'], $case['geom'], 1000, true) + ]); + $this->fail('Expected Exception not thrown for ' . $case['msg']); + } catch (\Exception $e) { + $this->assertInstanceOf(\Exception::class, $e, $case['msg']); + } + } + } + } From 158e7c1cd8f870e65ec38132eebc917d620d9aaa Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 8 Sep 2025 17:21:08 +0530 Subject: [PATCH 33/69] Fix spatial distance error messages and improve validation for geometry types --- src/Database/Adapter/MariaDB.php | 2 +- src/Database/Adapter/SQL.php | 3 ++ tests/e2e/Adapter/Scopes/SpatialTests.php | 40 ++++++++++------------- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index cb9cd3d84..a70e1dc73 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1393,7 +1393,7 @@ protected function handleDistanceSpatialQueries(string $spatialAttributeType, Qu $wktType = $this->getSpatialTypeFromWKT($wkt); $attrType = strtolower($spatialAttributeType); if ($wktType != Database::VAR_POINT || $attrType != Database::VAR_POINT) { - throw new DatabaseException('Distance in meters is not supported between '.$attrType . ' and '. $wkt); + throw new DatabaseException('Distance in meters is not supported between '.$attrType . ' and '. $wktType); } return "ST_DISTANCE_SPHERE({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0), 6371000) {$operator} :{$placeholder}_1"; } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 3f654b0a9..2dfa8e138 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2662,6 +2662,9 @@ public function getSpatialTypeFromWKT(string $wkt): string { $wkt = trim($wkt); $pos = strpos($wkt, '('); + if ($pos === false) { + throw new Exception("Not a valid spatialtype"); + } return strtolower(trim(substr($wkt, 0, $pos))); } } diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 7bb361ed1..5ece6cc84 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -2289,7 +2289,7 @@ public function testSpatialDistanceInMeterForMultiDimensionGeometry(): void } } - public function testSpatialDistanceInMeterError() + public function testSpatialDistanceInMeterError(): void { /** @var Database $database */ $database = static::getDatabase(); @@ -2316,25 +2316,16 @@ public function testSpatialDistanceInMeterError() ])); $this->assertInstanceOf(Document::class, $doc); - // Invalid geometry pairs (all combinations except POINT vs POINT) + // Invalid geometry pairs $cases = [ - // Point compared against wrong types - ['attr' => 'line', 'geom' => [0.002, 0.0], 'msg' => 'Point vs LineString'], - ['attr' => 'poly', 'geom' => [0.002, 0.0], 'msg' => 'Point vs Polygon'], - - // LineString compared against wrong types - ['attr' => 'loc', 'geom' => [[0.0, 0.0], [0.001, 0.001]], 'msg' => 'LineString vs Point'], - ['attr' => 'poly', 'geom' => [[0.0, 0.0], [0.001, 0.001]], 'msg' => 'LineString vs Polygon'], - - // Polygon compared against wrong types - ['attr' => 'loc', 'geom' => [[[0.0, 0.0], [0.001, 0.0], [0.001, 0.001], [0.0, 0.0]]], 'msg' => 'Polygon vs Point'], - ['attr' => 'line', 'geom' => [[[0.0, 0.0], [0.001, 0.0], [0.001, 0.001], [0.0, 0.0]]], 'msg' => 'Polygon vs LineString'], - - // Polygon vs Polygon (still invalid for "meters") - ['attr' => 'poly', 'geom' => [[[0.002, -0.001], [0.002, 0.001], [0.004, 0.001], [0.002, -0.001]]], 'msg' => 'Polygon vs Polygon'], - - // LineString vs LineString (invalid for "meters") - ['attr' => 'line', 'geom' => [[0.002, 0.0], [0.003, 0.0]], 'msg' => 'LineString vs LineString'], + ['attr' => 'line', 'geom' => [0.002, 0.0], 'expected' => ['linestring', 'point']], + ['attr' => 'poly', 'geom' => [0.002, 0.0], 'expected' => ['polygon', 'point']], + ['attr' => 'loc', 'geom' => [[0.0, 0.0], [0.001, 0.001]], 'expected' => ['point', 'linestring']], + ['attr' => 'poly', 'geom' => [[0.0, 0.0], [0.001, 0.001]], 'expected' => ['polygon', 'linestring']], + ['attr' => 'loc', 'geom' => [[[0.0, 0.0], [0.001, 0.0], [0.001, 0.001], [0.0, 0.0]]], 'expected' => ['point', 'polygon']], + ['attr' => 'line', 'geom' => [[[0.0, 0.0], [0.001, 0.0], [0.001, 0.001], [0.0, 0.0]]], 'expected' => ['linestring', 'polygon']], + ['attr' => 'poly', 'geom' => [[[0.002, -0.001], [0.002, 0.001], [0.004, 0.001], [0.002, -0.001]]], 'expected' => ['polygon', 'polygon']], + ['attr' => 'line', 'geom' => [[0.002, 0.0], [0.003, 0.0]], 'expected' => ['linestring', 'linestring']], ]; foreach ($cases as $case) { @@ -2342,11 +2333,16 @@ public function testSpatialDistanceInMeterError() $database->find($collection, [ Query::distanceLessThan($case['attr'], $case['geom'], 1000, true) ]); - $this->fail('Expected Exception not thrown for ' . $case['msg']); + $this->fail('Expected Exception not thrown for ' . implode(' vs ', $case['expected'])); } catch (\Exception $e) { - $this->assertInstanceOf(\Exception::class, $e, $case['msg']); + $this->assertInstanceOf(\Exception::class, $e); + + // Validate exception message contains correct type names + $msg = strtolower($e->getMessage()); + var_dump($msg); + $this->assertStringContainsString($case['expected'][0], $msg, 'Attr type missing in exception'); + $this->assertStringContainsString($case['expected'][1], $msg, 'Geom type missing in exception'); } } } - } From ab3a2350906f79472e6987ec2e0eba3ff35f56e5 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 8 Sep 2025 17:30:33 +0530 Subject: [PATCH 34/69] Refactor spatial query handling and improve error messages in database adapters --- src/Database/Adapter/MariaDB.php | 8 ++++---- src/Database/Adapter/MySQL.php | 4 ++-- src/Database/Adapter/SQL.php | 2 +- src/Database/Validator/Spatial.php | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index a70e1dc73..080ca34ba 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1355,15 +1355,15 @@ public function deleteDocument(string $collection, string $id): bool /** * Handle distance spatial queries * - * @param string $spatialAttributeType * @param Query $query * @param array $binds * @param string $attribute + * @param string $type * @param string $alias * @param string $placeholder * @return string */ - protected function handleDistanceSpatialQueries(string $spatialAttributeType, Query $query, array &$binds, string $attribute, string $alias, string $placeholder): string + protected function handleDistanceSpatialQueries(Query $query, array &$binds, string $attribute, string $type, string $alias, string $placeholder): string { $distanceParams = $query->getValues()[0]; $wkt = $this->convertArrayToWKT($distanceParams[0]); @@ -1391,7 +1391,7 @@ protected function handleDistanceSpatialQueries(string $spatialAttributeType, Qu if ($useMeters) { $wktType = $this->getSpatialTypeFromWKT($wkt); - $attrType = strtolower($spatialAttributeType); + $attrType = strtolower($type); if ($wktType != Database::VAR_POINT || $attrType != Database::VAR_POINT) { throw new DatabaseException('Distance in meters is not supported between '.$attrType . ' and '. $wktType); } @@ -1426,7 +1426,7 @@ protected function handleSpatialQueries(string $type, Query $query, array &$bind case Query::TYPE_DISTANCE_NOT_EQUAL: case Query::TYPE_DISTANCE_GREATER_THAN: case Query::TYPE_DISTANCE_LESS_THAN: - return $this->handleDistanceSpatialQueries($type, $query, $binds, $attribute, $alias, $placeholder); + return $this->handleDistanceSpatialQueries($query, $binds, $attribute, $type, $alias, $placeholder); case Query::TYPE_INTERSECTS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index 332a9b064..27a4028f3 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -82,15 +82,15 @@ public function getSizeOfCollectionOnDisk(string $collection): int /** * Handle distance spatial queries * - * @param string $spatialAttributeType * @param Query $query * @param array $binds * @param string $attribute + * @param string $type * @param string $alias * @param string $placeholder * @return string */ - protected function handleDistanceSpatialQueries(string $spatialAttributeType, Query $query, array &$binds, string $attribute, string $alias, string $placeholder): string + protected function handleDistanceSpatialQueries(Query $query, array &$binds, string $attribute, string $type, string $alias, string $placeholder): string { $distanceParams = $query->getValues()[0]; $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($distanceParams[0]); diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 2dfa8e138..0522bfc0d 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2663,7 +2663,7 @@ public function getSpatialTypeFromWKT(string $wkt): string $wkt = trim($wkt); $pos = strpos($wkt, '('); if ($pos === false) { - throw new Exception("Not a valid spatialtype"); + throw new DatabaseException("Invalid spatial type"); } return strtolower(trim(substr($wkt, 0, $pos))); } diff --git a/src/Database/Validator/Spatial.php b/src/Database/Validator/Spatial.php index ab3bd8be4..23886ffae 100644 --- a/src/Database/Validator/Spatial.php +++ b/src/Database/Validator/Spatial.php @@ -144,7 +144,7 @@ public function getType(): string return self::TYPE_ARRAY; } - public function getSptialType(): string + public function getSpatialType(): string { return $this->spatialType; } From 8d85d669019d99ec72aae7d19b9ff5562375e049 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 8 Sep 2025 17:40:08 +0530 Subject: [PATCH 35/69] updated types --- src/Database/Adapter/MariaDB.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 080ca34ba..b9b42a2cf 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1403,15 +1403,15 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str /** * Handle spatial queries * - * @param string $type * @param Query $query * @param array $binds * @param string $attribute + * @param string $type * @param string $alias * @param string $placeholder * @return string */ - protected function handleSpatialQueries(string $type, Query $query, array &$binds, string $attribute, string $alias, string $placeholder): string + protected function handleSpatialQueries(Query $query, array &$binds, string $attribute,string $type, string $alias, string $placeholder): string { switch ($query->getMethod()) { case Query::TYPE_CROSSES: @@ -1495,7 +1495,7 @@ protected function getSQLCondition(Query $query, array &$binds, array $attribute $attributeType = $this->getAttributeType($query->getAttribute(), $attributes); if (in_array($attributeType, Database::SPATIAL_TYPES)) { - return $this->handleSpatialQueries($attributeType, $query, $binds, $attribute, $alias, $placeholder); + return $this->handleSpatialQueries($query, $binds, $attribute, $attributeType,$alias, $placeholder); } switch ($query->getMethod()) { From 297b29a07e156d32fca5e76962f5be8643d089d9 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 8 Sep 2025 17:42:23 +0530 Subject: [PATCH 36/69] lint --- src/Database/Adapter/MariaDB.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index b9b42a2cf..4b4580afd 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1411,7 +1411,7 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str * @param string $placeholder * @return string */ - protected function handleSpatialQueries(Query $query, array &$binds, string $attribute,string $type, string $alias, string $placeholder): string + protected function handleSpatialQueries(Query $query, array &$binds, string $attribute, string $type, string $alias, string $placeholder): string { switch ($query->getMethod()) { case Query::TYPE_CROSSES: @@ -1495,7 +1495,7 @@ protected function getSQLCondition(Query $query, array &$binds, array $attribute $attributeType = $this->getAttributeType($query->getAttribute(), $attributes); if (in_array($attributeType, Database::SPATIAL_TYPES)) { - return $this->handleSpatialQueries($query, $binds, $attribute, $attributeType,$alias, $placeholder); + return $this->handleSpatialQueries($query, $binds, $attribute, $attributeType, $alias, $placeholder); } switch ($query->getMethod()) { From 285f3276db1b6b842362bced2fe1adeee15cd482 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 8 Sep 2025 21:49:16 +0300 Subject: [PATCH 37/69] Update Key validator to limit length to 36 characters for improved MongoDB ID support --- src/Database/Validator/Key.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Database/Validator/Key.php b/src/Database/Validator/Key.php index 18ed1dd02..920ff7b92 100644 --- a/src/Database/Validator/Key.php +++ b/src/Database/Validator/Key.php @@ -76,8 +76,8 @@ public function isValid($value): bool if (\preg_match('/[^A-Za-z0-9\_\-\.]/', $value)) { return false; } - // Updated to 100 to suport mongodb ids - if (\mb_strlen($value) > 100) { + // At most 36 chars + if (\mb_strlen($value) > 36) { return false; } From 79e7eba8644250b3a5a03cdfb8daed34c88c14ab Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 9 Sep 2025 22:50:01 +0530 Subject: [PATCH 38/69] updated spatial update attribute --- src/Database/Adapter.php | 3 +- src/Database/Adapter/MariaDB.php | 6 +-- src/Database/Adapter/Pool.php | 2 +- src/Database/Adapter/Postgres.php | 5 ++- src/Database/Adapter/SQLite.php | 3 +- src/Database/Database.php | 6 ++- tests/e2e/Adapter/Scopes/SpatialTests.php | 47 +++++++++++++++++++++++ 7 files changed, 63 insertions(+), 9 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 5e416d036..db58e4fa4 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -578,10 +578,11 @@ abstract public function createAttributes(string $collection, array $attributes) * @param bool $signed * @param bool $array * @param string|null $newKey + * @param bool $required * * @return bool */ - abstract public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false, ?string $newKey = null): bool; + abstract public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false, ?string $newKey = null, bool $required = false): bool; /** * Delete Attribute diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 4b4580afd..752a60eca 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -409,16 +409,16 @@ public function getSchemaAttributes(string $collection): array * @param bool $signed * @param bool $array * @param string|null $newKey + * @param bool $required * @return bool * @throws DatabaseException */ - public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false, ?string $newKey = null): bool + public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false, ?string $newKey = null, bool $required = false): bool { $name = $this->filter($collection); $id = $this->filter($id); $newKey = empty($newKey) ? null : $this->filter($newKey); - $type = $this->getSQLType($type, $size, $signed, $array, false); - + $type = $this->getSQLType($type, $size, $signed, $array, $required); if (!empty($newKey)) { $sql = "ALTER TABLE {$this->getSQLTable($name)} CHANGE COLUMN `{$id}` `{$newKey}` {$type};"; } else { diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index 636e37555..27dfd6bb8 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -175,7 +175,7 @@ public function createAttributes(string $collection, array $attributes): bool return $this->delegate(__FUNCTION__, \func_get_args()); } - public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false, ?string $newKey = null): bool + public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false, ?string $newKey = null, bool $required = false): bool { return $this->delegate(__FUNCTION__, \func_get_args()); } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index b183c8c4e..1276cea57 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -535,16 +535,17 @@ public function renameAttribute(string $collection, string $old, string $new): b * @param bool $signed * @param bool $array * @param string|null $newKey + * @param bool $required * @return bool * @throws Exception * @throws PDOException */ - public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false, ?string $newKey = null): bool + public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false, ?string $newKey = null, bool $required = false): bool { $name = $this->filter($collection); $id = $this->filter($id); $newKey = empty($newKey) ? null : $this->filter($newKey); - $type = $this->getSQLType($type, $size, $signed, $array, false); + $type = $this->getSQLType($type, $size, $signed, $array, $required); if ($type == 'TIMESTAMP(3)') { $type = "TIMESTAMP(3) without time zone USING TO_TIMESTAMP(\"$id\", 'YYYY-MM-DD HH24:MI:SS.MS')"; diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 4f9a2afdd..54ffb47dd 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -327,11 +327,12 @@ public function analyzeCollection(string $collection): bool * @param bool $signed * @param bool $array * @param string|null $newKey + * @param bool $required * @return bool * @throws Exception * @throws PDOException */ - public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false, ?string $newKey = null): bool + public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false, ?string $newKey = null, bool $required = false): bool { if (!empty($newKey) && $newKey !== $id) { return $this->renameAttribute($collection, $id, $newKey); diff --git a/src/Database/Database.php b/src/Database/Database.php index 9b7a60ff0..3f74ef10b 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -2161,6 +2161,10 @@ public function updateAttribute(string $collection, string $id, ?string $type = $default = null; } + if ($required === true && in_array($type, Database::SPATIAL_TYPES)) { + $altering = true; + } + switch ($type) { case self::VAR_STRING: if (empty($size)) { @@ -2322,7 +2326,7 @@ public function updateAttribute(string $collection, string $id, ?string $type = } } - $updated = $this->adapter->updateAttribute($collection, $id, $type, $size, $signed, $array, $newKey); + $updated = $this->adapter->updateAttribute($collection, $id, $type, $size, $signed, $array, $newKey, $required); if (!$updated) { throw new DatabaseException('Failed to update attribute'); diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 6af26eedc..74f4055d4 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -840,6 +840,53 @@ public function testSpatialIndex(): void } finally { $database->deleteCollection($collNullIndex); } + + $collUpdateNull = 'spatial_idx_update_null_' . uniqid(); + try { + $database->createCollection($collUpdateNull); + + $database->createAttribute($collUpdateNull, 'loc', Database::VAR_POINT, 0, false); + if (!$nullSupported) { + try { + $database->createIndex($collUpdateNull, 'idx_loc', Database::INDEX_SPATIAL, ['loc']); + $this->fail('Expected exception when creating spatial index on NULL-able attribute'); + } catch (\Throwable $e) { + $this->assertInstanceOf(Exception::class, $e); + } + } else { + $this->assertTrue($database->createIndex($collUpdateNull, 'idx_loc', Database::INDEX_SPATIAL, ['loc'])); + } + + $database->updateAttribute($collUpdateNull, 'loc', required: true); + + $this->assertTrue($database->createIndex($collUpdateNull, 'idx_loc_req', Database::INDEX_SPATIAL, ['loc'])); + } finally { + $database->deleteCollection($collUpdateNull); + } + + + $collUpdateNull = 'spatial_idx_index_null_' . uniqid(); + try { + $database->createCollection($collUpdateNull); + + $database->createAttribute($collUpdateNull, 'loc', Database::VAR_POINT, 0, false); + if (!$nullSupported) { + try { + $database->createIndex($collUpdateNull, 'idx_loc', Database::INDEX_SPATIAL, ['loc']); + $this->fail('Expected exception when creating spatial index on NULL-able attribute'); + } catch (\Throwable $e) { + $this->assertInstanceOf(Exception::class, $e); + } + } else { + $this->assertTrue($database->createIndex($collUpdateNull, 'idx_loc', Database::INDEX_SPATIAL, ['loc'])); + } + + $database->updateAttribute($collUpdateNull, 'loc', required: true); + + $this->assertTrue($database->createIndex($collUpdateNull, 'idx_loc_req', Database::INDEX_SPATIAL, ['loc'])); + } finally { + $database->deleteCollection($collUpdateNull); + } } public function testComplexGeometricShapes(): void From 78ca0c70a546693690db6707ddd2fbbc891087f6 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 9 Sep 2025 23:15:05 +0530 Subject: [PATCH 39/69] replaced database exception with query exception --- src/Database/Adapter/MariaDB.php | 3 ++- tests/e2e/Adapter/Scopes/SpatialTests.php | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 752a60eca..cb4b27fed 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -9,6 +9,7 @@ use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\NotFound as NotFoundException; +use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\Exception\Timeout as TimeoutException; use Utopia\Database\Exception\Truncate as TruncateException; use Utopia\Database\Helpers\ID; @@ -1393,7 +1394,7 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str $wktType = $this->getSpatialTypeFromWKT($wkt); $attrType = strtolower($type); if ($wktType != Database::VAR_POINT || $attrType != Database::VAR_POINT) { - throw new DatabaseException('Distance in meters is not supported between '.$attrType . ' and '. $wktType); + throw new QueryException('Distance in meters is not supported between '.$attrType . ' and '. $wktType); } return "ST_DISTANCE_SPHERE({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0), 6371000) {$operator} :{$placeholder}_1"; } diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 74f4055d4..8053a6d18 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -5,6 +5,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception; +use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\Exception\Structure as StructureException; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -2382,11 +2383,10 @@ public function testSpatialDistanceInMeterError(): void ]); $this->fail('Expected Exception not thrown for ' . implode(' vs ', $case['expected'])); } catch (\Exception $e) { - $this->assertInstanceOf(\Exception::class, $e); + $this->assertInstanceOf(QueryException::class, $e); // Validate exception message contains correct type names $msg = strtolower($e->getMessage()); - var_dump($msg); $this->assertStringContainsString($case['expected'][0], $msg, 'Attr type missing in exception'); $this->assertStringContainsString($case['expected'][1], $msg, 'Geom type missing in exception'); } From 2df73eadfcfa39ad7c9af86152291cef48e1e1b1 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 9 Sep 2025 23:22:24 +0530 Subject: [PATCH 40/69] updated tests --- tests/e2e/Adapter/Scopes/SpatialTests.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 8053a6d18..281edf070 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -842,14 +842,14 @@ public function testSpatialIndex(): void $database->deleteCollection($collNullIndex); } - $collUpdateNull = 'spatial_idx_update_null_' . uniqid(); + $collUpdateNull = 'spatial_idx_req'; try { $database->createCollection($collUpdateNull); $database->createAttribute($collUpdateNull, 'loc', Database::VAR_POINT, 0, false); if (!$nullSupported) { try { - $database->createIndex($collUpdateNull, 'idx_loc', Database::INDEX_SPATIAL, ['loc']); + $database->createIndex($collUpdateNull, 'idx_loc_required', Database::INDEX_SPATIAL, ['loc']); $this->fail('Expected exception when creating spatial index on NULL-able attribute'); } catch (\Throwable $e) { $this->assertInstanceOf(Exception::class, $e); @@ -866,7 +866,7 @@ public function testSpatialIndex(): void } - $collUpdateNull = 'spatial_idx_index_null_' . uniqid(); + $collUpdateNull = 'spatial_idx_index_null_required_true'; try { $database->createCollection($collUpdateNull); @@ -884,7 +884,7 @@ public function testSpatialIndex(): void $database->updateAttribute($collUpdateNull, 'loc', required: true); - $this->assertTrue($database->createIndex($collUpdateNull, 'idx_loc_req', Database::INDEX_SPATIAL, ['loc'])); + $this->assertTrue($database->createIndex($collUpdateNull, 'new index', Database::INDEX_SPATIAL, ['loc'])); } finally { $database->deleteCollection($collUpdateNull); } From 0d4f2ec6a365d97f4e6f5e9ea6917fc5ea6c60d7 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 10 Sep 2025 15:45:33 +0530 Subject: [PATCH 41/69] added long-lat order to constant in the mariadb/mysql adapter --- src/Database/Adapter/MariaDB.php | 24 ++++++++++++------------ src/Database/Adapter/MySQL.php | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index cb4b27fed..0c2f339bb 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1417,11 +1417,11 @@ protected function handleSpatialQueries(Query $query, array &$binds, string $att switch ($query->getMethod()) { case Query::TYPE_CROSSES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Crosses({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "ST_Crosses({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; case Query::TYPE_NOT_CROSSES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Crosses({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "NOT ST_Crosses({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; case Query::TYPE_DISTANCE_EQUAL: case Query::TYPE_DISTANCE_NOT_EQUAL: @@ -1431,43 +1431,43 @@ protected function handleSpatialQueries(Query $query, array &$binds, string $att case Query::TYPE_INTERSECTS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; case Query::TYPE_NOT_INTERSECTS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "NOT ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; case Query::TYPE_OVERLAPS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; case Query::TYPE_NOT_OVERLAPS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "NOT ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; case Query::TYPE_TOUCHES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; case Query::TYPE_NOT_TOUCHES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "NOT ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; case Query::TYPE_EQUAL: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; case Query::TYPE_NOT_EQUAL: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "NOT ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; case Query::TYPE_CONTAINS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Contains({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "ST_Contains({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; case Query::TYPE_NOT_CONTAINS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Contains({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "NOT ST_Contains({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; default: throw new DatabaseException('Unknown spatial query method: ' . $query->getMethod()); diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index 27a4028f3..f15e5d76c 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -117,7 +117,7 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str if ($useMeters) { $attr = "ST_SRID({$alias}.{$attribute}, " . Database::SRID . ")"; - $geom = "ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . ")"; + $geom = "ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . ",'axis-order=long-lat')"; return "ST_Distance({$attr}, {$geom}, 'metre') {$operator} :{$placeholder}_1"; } From 796310b430b59b09b4b179136d7598336af0b597 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 10 Sep 2025 17:01:49 +0530 Subject: [PATCH 42/69] spatial types filter --- src/Database/Database.php | 113 +++++++++++++++++++--- tests/e2e/Adapter/Scopes/SpatialTests.php | 68 +++++++++++++ 2 files changed, 170 insertions(+), 11 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 3f74ef10b..85bce93b8 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -477,6 +477,91 @@ function (?string $value) { return DateTime::formatTz($value); } ); + + self::addFilter( + Database::VAR_POINT, + /** + * @param mixed $value + * @return mixed + */ + function (mixed $value) { + if (is_null($value)) { + return; + } + try { + return self::encodeSpatialData($value, Database::VAR_POINT); + } catch (\Throwable) { + return $value; + } + }, + /** + * @param string|null $value + * @return string|null + */ + function (?string $value) { + if (is_null($value)) { + return $value; + } + return self::decodeSpatialData($value); + } + ); + self::addFilter( + Database::VAR_LINESTRING, + /** + * @param mixed $value + * @return mixed + */ + function (mixed $value) { + if (is_null($value)) { + return; + } + try { + return self::encodeSpatialData($value, Database::VAR_LINESTRING); + } catch (\Throwable) { + if (is_null($value)) { + return $value; + } + return $value; + } + }, + /** + * @param string|null $value + * @return string|null + */ + function (?string $value) { + if (is_null($value)) { + return $value; + } + return self::decodeSpatialData($value); + } + ); + self::addFilter( + Database::VAR_POLYGON, + /** + * @param mixed $value + * @return mixed + */ + function (mixed $value) { + if (is_null($value)) { + return; + } + try { + return self::encodeSpatialData($value, Database::VAR_POLYGON); + } catch (\Throwable) { + return $value; + } + }, + /** + * @param string|null $value + * @return string|null + */ + function (?string $value) { + if (is_null($value)) { + return $value; + } + return self::decodeSpatialData($value); + } + ); } /** @@ -1242,6 +1327,19 @@ public function delete(?string $database = null): bool */ public function createCollection(string $id, array $attributes = [], array $indexes = [], ?array $permissions = null, bool $documentSecurity = true): Document { + foreach ($attributes as &$attribute) { + if (in_array($attribute['type'], Database::SPATIAL_TYPES)) { + $existingFilters = $attribute['filters'] ?? []; + if (!is_array($existingFilters)) { + $existingFilters = [$existingFilters]; + } + $attribute['filters'] = array_values( + array_unique(array_merge($existingFilters, [$attribute['type']])) + ); + } + } + unset($attribute); + $permissions ??= [ Permission::create(Role::any()), ]; @@ -1598,6 +1696,10 @@ public function createAttribute(string $collection, string $id, string $type, in if ($collection->isEmpty()) { throw new NotFoundException('Collection not found'); } + if (in_array($type, Database::SPATIAL_TYPES)) { + $filters[] = $type; + $filters = array_unique($filters); + } $attribute = $this->validateAttribute( $collection, @@ -6561,14 +6663,6 @@ public function encode(Document $collection, Document $document): Document foreach ($value as $index => $node) { if ($node !== null) { - // Handle spatial data encoding - $attributeType = $attribute['type'] ?? ''; - if (in_array($attributeType, Database::SPATIAL_TYPES)) { - if (is_array($node)) { - $node = $this->encodeSpatialData($node, $attributeType); - } - } - foreach ($filters as $filter) { $node = $this->encodeAttribute($filter, $node, $document); } @@ -6647,9 +6741,6 @@ public function decode(Document $collection, Document $document, array $selectio $value = (is_null($value)) ? [] : $value; foreach ($value as $index => $node) { - if (is_string($node) && in_array($type, Database::SPATIAL_TYPES)) { - $node = $this->decodeSpatialData($node); - } foreach (array_reverse($filters) as $filter) { $node = $this->decodeAttribute($filter, $node, $document, $key); diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 281edf070..983851a71 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -2392,4 +2392,72 @@ public function testSpatialDistanceInMeterError(): void } } } + public function testSpatialEncodeDecode(): void + { + $collection = new Document([ + '$collection' => ID::custom(Database::METADATA), + '$id' => ID::custom('users'), + 'name' => 'Users', + 'attributes' => [ + [ + '$id' => ID::custom('point'), + 'type' => Database::VAR_POINT, + 'required' => false, + 'filters' => [Database::VAR_POINT], + ], + [ + '$id' => ID::custom('line'), + 'type' => Database::VAR_LINESTRING, + 'format' => '', + 'required' => false, + 'filters' => [Database::VAR_LINESTRING], + ], + [ + '$id' => ID::custom('poly'), + 'type' => Database::VAR_POLYGON, + 'format' => '', + 'required' => false, + 'filters' => [Database::VAR_POLYGON], + ] + ] + ]); + + /** @var Database $database */ + $database = static::getDatabase(); + if (!$database->getAdapter()->getSupportForSpatialAttributes()) { + $this->markTestSkipped('Adapter does not support spatial attributes'); + } + $point = "POINT(1 2)"; + $line = "LINESTRING(1 2, 1 2)"; + $poly = "POLYGON((0 0, 0 10, 10 10, 0 0))"; + + $pointArr = [1,2]; + $lineArr = [[1,2],[1,2]]; + $polyArr = [[[0.0, 0.0], [0.0, 10.0], [10.0, 10.0], [0.0, 0.0]]]; + $doc = new Document(['point' => $pointArr ,'line' => $lineArr, 'poly' => $polyArr]); + + $result = $database->encode($collection, $doc); + + $this->assertEquals($result->getAttribute('point'), $point); + $this->assertEquals($result->getAttribute('line'), $line); + $this->assertEquals($result->getAttribute('poly'), $poly); + + + $result = $database->decode($collection, $doc); + $this->assertEquals($result->getAttribute('point'), $pointArr); + $this->assertEquals($result->getAttribute('line'), $lineArr); + $this->assertEquals($result->getAttribute('poly'), $polyArr); + + $stringDoc = new Document(['point' => $point,'line' => $line, 'poly' => $poly]); + $result = $database->decode($collection, $stringDoc); + $this->assertEquals($result->getAttribute('point'), $pointArr); + $this->assertEquals($result->getAttribute('line'), $lineArr); + $this->assertEquals($result->getAttribute('poly'), $polyArr); + + $nullDoc = new Document(['point' => null,'line' => null, 'poly' => null]); + $result = $database->decode($collection, $nullDoc); + $this->assertEquals($result->getAttribute('point'), null); + $this->assertEquals($result->getAttribute('line'), null); + $this->assertEquals($result->getAttribute('poly'), null); + } } From 7f08617ce0c838e7e37825c8aa350664363fb1d8 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 10 Sep 2025 17:25:31 +0530 Subject: [PATCH 43/69] updated condition for returning value in the filters encode/decode --- src/Database/Database.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 85bce93b8..b1d9053e2 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -485,8 +485,8 @@ function (?string $value) { * @return mixed */ function (mixed $value) { - if (is_null($value)) { - return; + if (!is_array($value)) { + return $value; } try { return self::encodeSpatialData($value, Database::VAR_POINT); @@ -499,7 +499,7 @@ function (mixed $value) { * @return string|null */ function (?string $value) { - if (is_null($value)) { + if (!is_string($value)) { return $value; } return self::decodeSpatialData($value); @@ -4742,7 +4742,6 @@ public function updateDocuments( if (!is_null($this->timestamp) && $oldUpdatedAt > $this->timestamp) { throw new ConflictException('Document was updated after the request timestamp'); } - $batch[$index] = $this->encode($collection, $document); } From adb82de6663419e882ea1f6bbe47ebf385d4a497 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 10 Sep 2025 18:30:11 +0530 Subject: [PATCH 44/69] refactor spatial attribute handling to support NULL values and improve index validation * fixed long-lat order for geo function * added spatial types as filters * index validation updates * test update for index createion edge cases and filter encoding and decoding --- src/Database/Adapter/MariaDB.php | 32 +++++++- src/Database/Database.php | 7 +- src/Database/Validator/Index.php | 14 ++-- tests/e2e/Adapter/Scopes/SpatialTests.php | 92 ++++++++++++++++++++++- 4 files changed, 131 insertions(+), 14 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 0c2f339bb..08afe6452 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1641,13 +1641,39 @@ protected function getSQLType(string $type, int $size, bool $signed = true, bool case Database::VAR_POINT: - return 'POINT' . ($required && !$this->getSupportForSpatialIndexNull() ? ' NOT NULL' : ''); + $type = 'POINT'; + if (!$this->getSupportForSpatialIndexNull()) { + if ($required) { + $type .= ' NOT NULL'; + } else { + $type .= ' NULL'; + } + } + return $type; case Database::VAR_LINESTRING: - return 'LINESTRING' . ($required && !$this->getSupportForSpatialIndexNull() ? ' NOT NULL' : ''); + $type = 'LINESTRING'; + if (!$this->getSupportForSpatialIndexNull()) { + if ($required) { + $type .= ' NOT NULL'; + } else { + $type .= ' NULL'; + } + } + return $type; + case Database::VAR_POLYGON: - return 'POLYGON' . ($required && !$this->getSupportForSpatialIndexNull() ? ' NOT NULL' : ''); + $type = 'POLYGON'; + if (!$this->getSupportForSpatialIndexNull()) { + if ($required) { + $type .= ' NOT NULL'; + } else { + $type .= ' NULL'; + } + } + return $type; + default: throw new DatabaseException('Unknown type: ' . $type . '. Must be one of ' . Database::VAR_STRING . ', ' . Database::VAR_INTEGER . ', ' . Database::VAR_FLOAT . ', ' . Database::VAR_BOOLEAN . ', ' . Database::VAR_DATETIME . ', ' . Database::VAR_RELATIONSHIP . ', ' . ', ' . Database::VAR_POINT . ', ' . Database::VAR_LINESTRING . ', ' . Database::VAR_POLYGON); diff --git a/src/Database/Database.php b/src/Database/Database.php index b1d9053e2..471c7277b 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -2263,7 +2263,8 @@ public function updateAttribute(string $collection, string $id, ?string $type = $default = null; } - if ($required === true && in_array($type, Database::SPATIAL_TYPES)) { + // we need to alter table attribute type to NOT NULL/NULL for change in required + if (!$this->adapter->getSupportForSpatialIndexNull() && in_array($type, Database::SPATIAL_TYPES)) { $altering = true; } @@ -3323,12 +3324,12 @@ public function createIndex(string $collection, string $id, string $type, array if ($type === self::INDEX_SPATIAL) { foreach ($attributes as $attr) { if (!isset($indexAttributesWithTypes[$attr])) { - throw new DatabaseException('Attribute "' . $attr . '" not found in collection'); + throw new IndexException('Attribute "' . $attr . '" not found in collection'); } $attributeType = $indexAttributesWithTypes[$attr]; if (!in_array($attributeType, [self::VAR_POINT, self::VAR_LINESTRING, self::VAR_POLYGON])) { - throw new DatabaseException('Spatial index can only be created on spatial attributes (point, linestring, polygon). Attribute "' . $attr . '" is of type "' . $attributeType . '"'); + throw new IndexException('Spatial index can only be created on spatial attributes (point, linestring, polygon). Attribute "' . $attr . '" is of type "' . $attributeType . '"'); } } diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php index 87fa51e78..2385888aa 100644 --- a/src/Database/Validator/Index.php +++ b/src/Database/Validator/Index.php @@ -339,9 +339,6 @@ public function getType(): string public function checkSpatialIndex(Document $index): bool { $type = $index->getAttribute('type'); - if ($type !== Database::INDEX_SPATIAL) { - return true; - } if (!$this->spatialIndexSupport) { $this->message = 'Spatial indexes are not supported'; @@ -351,15 +348,22 @@ public function checkSpatialIndex(Document $index): bool $attributes = $index->getAttribute('attributes', []); $orders = $index->getAttribute('orders', []); + if (count($attributes) !== 1) { + $this->message = 'Spatial index can be created on a single spatial attribute'; + return false; + } + foreach ($attributes as $attributeName) { $attribute = $this->attributes[\strtolower($attributeName)] ?? new Document(); $attributeType = $attribute->getAttribute('type', ''); - if (!\in_array($attributeType, Database::SPATIAL_TYPES, true)) { + continue; + } + + if ($type !== Database::INDEX_SPATIAL) { $this->message = 'Spatial index can only be created on spatial attributes (point, linestring, polygon). Attribute "' . $attributeName . '" is of type "' . $attributeType . '"'; return false; } - $required = (bool) $attribute->getAttribute('required', false); if (!$required && !$this->spatialIndexNullSupport) { $this->message = 'Spatial indexes do not allow null values. Mark the attribute "' . $attributeName . '" as required or create the index on a column with no null values.'; diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 983851a71..7b8512e2e 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -5,6 +5,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception; +use Utopia\Database\Exception\Index as IndexException; use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\Exception\Structure as StructureException; use Utopia\Database\Helpers\ID; @@ -1469,7 +1470,6 @@ public function testSpatialBulkOperation(): void 'required' => true, 'signed' => true, 'array' => false, - 'filters' => [], ]), new Document([ '$id' => ID::custom('location'), @@ -1478,7 +1478,6 @@ public function testSpatialBulkOperation(): void 'required' => true, 'signed' => true, 'array' => false, - 'filters' => [], ]), new Document([ '$id' => ID::custom('area'), @@ -1487,7 +1486,6 @@ public function testSpatialBulkOperation(): void 'required' => false, 'signed' => true, 'array' => false, - 'filters' => [], ]) ]; @@ -2460,4 +2458,92 @@ public function testSpatialEncodeDecode(): void $this->assertEquals($result->getAttribute('line'), null); $this->assertEquals($result->getAttribute('poly'), null); } + + public function testSpatialIndexSingleAttributeOnly(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + if (!$database->getAdapter()->getSupportForSpatialAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $collectionName = 'spatial_idx_single_attr_' . uniqid(); + try { + $database->createCollection($collectionName); + + // Create a spatial attribute + $database->createAttribute($collectionName, 'loc', Database::VAR_POINT, 0, true); + $database->createAttribute($collectionName, 'loc2', Database::VAR_POINT, 0, true); + $database->createAttribute($collectionName, 'title', Database::VAR_STRING, 255, true); + + // Case 1: Valid spatial index on a single spatial attribute + $this->assertTrue( + $database->createIndex($collectionName, 'idx_loc', Database::INDEX_SPATIAL, ['loc']) + ); + + // Case 2: Fail when trying to create spatial index with multiple attributes + try { + $database->createIndex($collectionName, 'idx_multi', Database::INDEX_SPATIAL, ['loc', 'loc2']); + $this->fail('Expected exception when creating spatial index on multiple attributes'); + } catch (\Throwable $e) { + $this->assertInstanceOf(IndexException::class, $e); + } + + // Case 3: Fail when trying to create non-spatial index on a spatial attribute + try { + $database->createIndex($collectionName, 'idx_wrong_type', Database::INDEX_KEY, ['loc']); + $this->fail('Expected exception when creating non-spatial index on spatial attribute'); + } catch (\Throwable $e) { + $this->assertInstanceOf(IndexException::class, $e); + } + + // Case 4: Fail when trying to mix spatial + non-spatial attributes in a spatial index + try { + $database->createIndex($collectionName, 'idx_mix', Database::INDEX_SPATIAL, ['loc', 'title']); + $this->fail('Expected exception when creating spatial index with mixed attribute types'); + } catch (\Throwable $e) { + $this->assertInstanceOf(IndexException::class, $e); + } + + } finally { + $database->deleteCollection($collectionName); + } + } + + public function testSpatialIndexRequiredToggling(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + if (!$database->getAdapter()->getSupportForSpatialAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + if ($database->getAdapter()->getSupportForSpatialIndexNull()) { + $this->expectNotToPerformAssertions(); + return; + } + + try { + $collUpdateNull = 'spatial_idx_toggle'; + $database->createCollection($collUpdateNull); + + $database->createAttribute($collUpdateNull, 'loc', Database::VAR_POINT, 0, false); + try { + $database->createIndex($collUpdateNull, 'idx_loc', Database::INDEX_SPATIAL, ['loc']); + $this->fail('Expected exception when creating spatial index on NULL-able attribute'); + } catch (\Throwable $e) { + $this->assertInstanceOf(Exception::class, $e); + } + $database->updateAttribute($collUpdateNull, 'loc', required: true); + $this->assertTrue($database->createIndex($collUpdateNull, 'new index', Database::INDEX_SPATIAL, ['loc'])); + $this->assertTrue($database->deleteIndex($collUpdateNull, 'new index')); + $database->updateAttribute($collUpdateNull, 'loc', required: false); + + $database->createDocument($collUpdateNull, new Document(['loc' => null])); + } finally { + $database->deleteCollection($collUpdateNull); + } + } + } From d291d6ae1e4b3988090d0229f8873d0131e76cfb Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 10 Sep 2025 18:34:57 +0530 Subject: [PATCH 45/69] linting --- src/Database/Database.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 471c7277b..91ca5c1ca 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -518,9 +518,6 @@ function (mixed $value) { try { return self::encodeSpatialData($value, Database::VAR_LINESTRING); } catch (\Throwable) { - if (is_null($value)) { - return $value; - } return $value; } }, From 7ed30f442ba3957ddf982d679e6e3e92d7cf8004 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 10 Sep 2025 19:30:24 +0530 Subject: [PATCH 46/69] updated index validator for spatial types --- src/Database/Validator/Index.php | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php index 2385888aa..bab80c173 100644 --- a/src/Database/Validator/Index.php +++ b/src/Database/Validator/Index.php @@ -340,26 +340,27 @@ public function checkSpatialIndex(Document $index): bool { $type = $index->getAttribute('type'); - if (!$this->spatialIndexSupport) { - $this->message = 'Spatial indexes are not supported'; - return false; - } - $attributes = $index->getAttribute('attributes', []); $orders = $index->getAttribute('orders', []); - if (count($attributes) !== 1) { - $this->message = 'Spatial index can be created on a single spatial attribute'; - return false; - } - foreach ($attributes as $attributeName) { $attribute = $this->attributes[\strtolower($attributeName)] ?? new Document(); $attributeType = $attribute->getAttribute('type', ''); + if (!\in_array($attributeType, Database::SPATIAL_TYPES, true)) { continue; } + if (!$this->spatialIndexSupport) { + $this->message = 'Spatial indexes are not supported'; + return false; + } + + if (count($attributes) !== 1) { + $this->message = 'Spatial index can be created on a single spatial attribute'; + return false; + } + if ($type !== Database::INDEX_SPATIAL) { $this->message = 'Spatial index can only be created on spatial attributes (point, linestring, polygon). Attribute "' . $attributeName . '" is of type "' . $attributeType . '"'; return false; @@ -369,13 +370,14 @@ public function checkSpatialIndex(Document $index): bool $this->message = 'Spatial indexes do not allow null values. Mark the attribute "' . $attributeName . '" as required or create the index on a column with no null values.'; return false; } - } - if (!empty($orders) && !$this->spatialIndexOrderSupport) { - $this->message = 'Spatial indexes with explicit orders are not supported. Remove the orders to create this index.'; - return false; + if (!empty($orders) && !$this->spatialIndexOrderSupport) { + $this->message = 'Spatial indexes with explicit orders are not supported. Remove the orders to create this index.'; + return false; + } } + return true; } } From 4f7bae74abf3968ea24609378b34a2675778e999 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 10 Sep 2025 19:38:34 +0530 Subject: [PATCH 47/69] updated spatial type tests with spatial index and non spatial combiation --- tests/e2e/Adapter/Scopes/SpatialTests.php | 62 +++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 7b8512e2e..1aadeeed3 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -12,6 +12,7 @@ use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; +use Utopia\Database\Validator\Index; trait SpatialTests { @@ -2546,4 +2547,65 @@ public function testSpatialIndexRequiredToggling(): void } } + public function testSpatialIndexOnNonSpatial(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + if (!$database->getAdapter()->getSupportForSpatialAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + try { + $collUpdateNull = 'spatial_idx_toggle'; + $database->createCollection($collUpdateNull); + + $database->createAttribute($collUpdateNull, 'loc', Database::VAR_POINT, 0, true); + $database->createAttribute($collUpdateNull, 'name', Database::VAR_STRING, 0, true); + try { + $database->createIndex($collUpdateNull, 'idx_loc', Database::INDEX_SPATIAL, ['name']); + $this->fail('Expected exception when creating spatial index on NULL-able attribute'); + } catch (\Throwable $e) { + $this->assertInstanceOf(IndexException::class, $e); + } + + try { + $database->createIndex($collUpdateNull, 'idx_loc', Database::INDEX_KEY, ['loc']); + $this->fail('Expected exception when creating non spatial index on spatial attribute'); + } catch (\Throwable $e) { + $this->assertInstanceOf(IndexException::class, $e); + } + + try { + $database->createIndex($collUpdateNull, 'idx_loc', Database::INDEX_KEY, ['loc,name']); + $this->fail('Expected exception when creating index'); + } catch (\Throwable $e) { + $this->assertInstanceOf(IndexException::class, $e); + } + + try { + $database->createIndex($collUpdateNull, 'idx_loc', Database::INDEX_KEY, ['name,loc']); + $this->fail('Expected exception when creating index'); + } catch (\Throwable $e) { + $this->assertInstanceOf(IndexException::class, $e); + } + + try { + $database->createIndex($collUpdateNull, 'idx_loc', Database::INDEX_SPATIAL, ['name,loc']); + $this->fail('Expected exception when creating index'); + } catch (\Throwable $e) { + $this->assertInstanceOf(IndexException::class, $e); + } + + try { + $database->createIndex($collUpdateNull, 'idx_loc', Database::INDEX_SPATIAL, ['loc,name']); + $this->fail('Expected exception when creating index'); + } catch (\Throwable $e) { + $this->assertInstanceOf(IndexException::class, $e); + } + + } finally { + $database->deleteCollection($collUpdateNull); + } + } } From 7123d2870bb15571b18c95631f0daf1c762f76f3 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 10 Sep 2025 19:51:28 +0530 Subject: [PATCH 48/69] fixed postgres test failing due to 0 length varchar --- tests/e2e/Adapter/Scopes/SpatialTests.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 1aadeeed3..5e1a4f4e4 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -2561,7 +2561,7 @@ public function testSpatialIndexOnNonSpatial(): void $database->createCollection($collUpdateNull); $database->createAttribute($collUpdateNull, 'loc', Database::VAR_POINT, 0, true); - $database->createAttribute($collUpdateNull, 'name', Database::VAR_STRING, 0, true); + $database->createAttribute($collUpdateNull, 'name', Database::VAR_STRING, 4, true); try { $database->createIndex($collUpdateNull, 'idx_loc', Database::INDEX_SPATIAL, ['name']); $this->fail('Expected exception when creating spatial index on NULL-able attribute'); From 4abe0ef9085227919c4ab14aa8e415ef82dbe082 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 11 Sep 2025 11:15:56 +0530 Subject: [PATCH 49/69] updated array conditon for filter --- src/Database/Database.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 91ca5c1ca..b6163fe4f 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -512,8 +512,8 @@ function (?string $value) { * @return mixed */ function (mixed $value) { - if (is_null($value)) { - return; + if (!is_array($value)) { + return $value; } try { return self::encodeSpatialData($value, Database::VAR_LINESTRING); @@ -539,8 +539,8 @@ function (?string $value) { * @return mixed */ function (mixed $value) { - if (is_null($value)) { - return; + if (!is_array($value)) { + return $value; } try { return self::encodeSpatialData($value, Database::VAR_POLYGON); From 1b6f0af29bf54bd9be51df6a6976fa588b298013 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 11 Sep 2025 09:44:54 +0300 Subject: [PATCH 50/69] sync with main --- src/Database/Adapter/Mongo.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index dc3c67b69..ad47a8185 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -1157,7 +1157,7 @@ public function updateDocuments(Document $collection, Document $updates, array $ * @param array $changes * @return array */ - public function createOrUpdateDocuments(Document $collection, string $attribute, array $changes): array + public function upsertDocuments(Document $collection, string $attribute, array $changes): array { if (empty($changes)) { return $changes; From 34a6001f718cd58f8fb96cc1e7052d4a05ceef20 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 11 Sep 2025 09:58:59 +0300 Subject: [PATCH 51/69] dding mongo to tests workflow --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index eea0e7842..93868fb80 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -72,12 +72,14 @@ jobs: matrix: adapter: [ + Mongo, MariaDB, MySQL, Postgres, SQLite, Mirror, Pool, + SharedTables/Mongo, SharedTables/MariaDB, SharedTables/MySQL, SharedTables/Postgres, From 8f662ea5aaf8ed88d4e0f2eedd5c75c8c85004a9 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 11 Sep 2025 10:04:22 +0300 Subject: [PATCH 52/69] adding mongo to tests workflow --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 93868fb80..025894dd5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -72,14 +72,14 @@ jobs: matrix: adapter: [ - Mongo, + MongoDB, MariaDB, MySQL, Postgres, SQLite, Mirror, Pool, - SharedTables/Mongo, + SharedTables/MongoDB, SharedTables/MariaDB, SharedTables/MySQL, SharedTables/Postgres, From fc2871fe739a44f9b3cac2d3a8782427ea31f7a0 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 11 Sep 2025 10:10:13 +0300 Subject: [PATCH 53/69] adding mongo to tests workflow --- tests/e2e/Adapter/Scopes/DocumentTests.php | 146 ++++++++++----------- 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 688b77beb..4476ad9c2 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -3484,79 +3484,79 @@ public function testFindNotContains(): void // $this->assertLessThanOrEqual(5, count($documents)); // But still excluding Marvel movies // } - public function testFindNotBetween(): void - { - /** @var Database $database */ - $database = static::getDatabase(); - - // Test notBetween with price range - should return documents outside the range - $documents = $database->find('movies', [ - Query::notBetween('price', 25.94, 25.99), - ]); - $this->assertEquals(4, count($documents)); // All movies except the 2 in the price range - - // Test notBetween with range that includes no documents - should return all documents - $documents = $database->find('movies', [ - Query::notBetween('price', 30, 35), - ]); - $this->assertEquals(6, count($documents)); - - // Test notBetween with date range - $documents = $database->find('movies', [ - Query::notBetween('$createdAt', '1975-12-06', '2050-12-06'), - ]); - $this->assertEquals(0, count($documents)); // No movies outside this wide date range - - // Test notBetween with narrower date range - $documents = $database->find('movies', [ - Query::notBetween('$createdAt', '2000-01-01', '2001-01-01'), - ]); - $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range - - // Test notBetween with updated date range - $documents = $database->find('movies', [ - Query::notBetween('$updatedAt', '2000-01-01T00:00:00.000+00:00', '2001-01-01T00:00:00.000+00:00'), - ]); - $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range - - // Test notBetween with year range (integer values) - $documents = $database->find('movies', [ - Query::notBetween('year', 2005, 2007), - ]); - $this->assertLessThanOrEqual(6, count($documents)); // Movies outside 2005-2007 range - - // Test notBetween with reversed range (start > end) - should still work - $documents = $database->find('movies', [ - Query::notBetween('price', 25.99, 25.94), // Note: reversed order - ]); - $this->assertGreaterThanOrEqual(4, count($documents)); // Should handle reversed range gracefully - - // Test notBetween with same start and end values - $documents = $database->find('movies', [ - Query::notBetween('year', 2006, 2006), - ]); - $this->assertGreaterThanOrEqual(5, count($documents)); // All movies except those from exactly 2006 - - // Test notBetween combined with other filters - $documents = $database->find('movies', [ - Query::notBetween('price', 25.94, 25.99), - Query::orderDesc('year'), - Query::limit(2) - ]); - $this->assertEquals(2, count($documents)); // Limited results, ordered, excluding price range - - // Test notBetween with extreme ranges - $documents = $database->find('movies', [ - Query::notBetween('year', -1000, 1000), // Very wide range - ]); - $this->assertLessThanOrEqual(6, count($documents)); // Movies outside this range - - // Test notBetween with float precision - $documents = $database->find('movies', [ - Query::notBetween('price', 25.945, 25.955), // Very narrow range - ]); - $this->assertGreaterThanOrEqual(4, count($documents)); // Most movies should be outside this narrow range - } +// public function testFindNotBetween(): void +// { +// /** @var Database $database */ +// $database = static::getDatabase(); +// +// // Test notBetween with price range - should return documents outside the range +// $documents = $database->find('movies', [ +// Query::notBetween('price', 25.94, 25.99), +// ]); +// $this->assertEquals(4, count($documents)); // All movies except the 2 in the price range +// +// // Test notBetween with range that includes no documents - should return all documents +// $documents = $database->find('movies', [ +// Query::notBetween('price', 30, 35), +// ]); +// $this->assertEquals(6, count($documents)); +// +// // Test notBetween with date range +// $documents = $database->find('movies', [ +// Query::notBetween('$createdAt', '1975-12-06', '2050-12-06'), +// ]); +// $this->assertEquals(0, count($documents)); // No movies outside this wide date range +// +// // Test notBetween with narrower date range +// $documents = $database->find('movies', [ +// Query::notBetween('$createdAt', '2000-01-01', '2001-01-01'), +// ]); +// $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range +// +// // Test notBetween with updated date range +// $documents = $database->find('movies', [ +// Query::notBetween('$updatedAt', '2000-01-01T00:00:00.000+00:00', '2001-01-01T00:00:00.000+00:00'), +// ]); +// $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range +// +// // Test notBetween with year range (integer values) +// $documents = $database->find('movies', [ +// Query::notBetween('year', 2005, 2007), +// ]); +// $this->assertLessThanOrEqual(6, count($documents)); // Movies outside 2005-2007 range +// +// // Test notBetween with reversed range (start > end) - should still work +// $documents = $database->find('movies', [ +// Query::notBetween('price', 25.99, 25.94), // Note: reversed order +// ]); +// $this->assertGreaterThanOrEqual(4, count($documents)); // Should handle reversed range gracefully +// +// // Test notBetween with same start and end values +// $documents = $database->find('movies', [ +// Query::notBetween('year', 2006, 2006), +// ]); +// $this->assertGreaterThanOrEqual(5, count($documents)); // All movies except those from exactly 2006 +// +// // Test notBetween combined with other filters +// $documents = $database->find('movies', [ +// Query::notBetween('price', 25.94, 25.99), +// Query::orderDesc('year'), +// Query::limit(2) +// ]); +// $this->assertEquals(2, count($documents)); // Limited results, ordered, excluding price range +// +// // Test notBetween with extreme ranges +// $documents = $database->find('movies', [ +// Query::notBetween('year', -1000, 1000), // Very wide range +// ]); +// $this->assertLessThanOrEqual(6, count($documents)); // Movies outside this range +// +// // Test notBetween with float precision +// $documents = $database->find('movies', [ +// Query::notBetween('price', 25.945, 25.955), // Very narrow range +// ]); +// $this->assertGreaterThanOrEqual(4, count($documents)); // Most movies should be outside this narrow range +// } public function testFindSelect(): void { From 77bc5fdbe901305b5f6cc2157373ba14f5f399ea Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 11 Sep 2025 14:42:05 +0530 Subject: [PATCH 54/69] added srid fix for the mysql --- src/Database/Adapter/MariaDB.php | 73 +++++++++++++++++------ src/Database/Adapter/MySQL.php | 52 ++++++++++++++-- src/Database/Adapter/SQL.php | 35 ++++++++++- tests/e2e/Adapter/Base.php | 14 ++--- tests/e2e/Adapter/Scopes/SpatialTests.php | 2 + 5 files changed, 145 insertions(+), 31 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 08afe6452..807d569b7 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -846,7 +846,7 @@ public function createDocument(Document $collection, Document $document): Docume $bindKey = 'key_' . $bindIndex; $columns .= "`{$column}`, "; if (in_array($attribute, $spatialAttributes)) { - $columnNames .= 'ST_GeomFromText(:' . $bindKey . '), '; + $columnNames .= 'ST_GeomFromText(:' . $bindKey . ', ' . Database::SRID . ", 'axis-order=lat-long'), "; } else { $columnNames .= ':' . $bindKey . ', '; } @@ -1116,7 +1116,7 @@ public function updateDocument(Document $collection, string $id, Document $docum $bindKey = 'key_' . $bindIndex; if (in_array($attribute, $spatialAttributes)) { - $columns .= "`{$column}`" . '=ST_GeomFromText(:' . $bindKey . '),'; + $columns .= "`{$column}`" . '=ST_GeomFromText(:' . $bindKey . ' , ' . Database::SRID . ' ),'; } else { $columns .= "`{$column}`" . '=:' . $bindKey . ','; } @@ -1417,11 +1417,11 @@ protected function handleSpatialQueries(Query $query, array &$binds, string $att switch ($query->getMethod()) { case Query::TYPE_CROSSES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Crosses({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; + return "ST_Crosses({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; case Query::TYPE_NOT_CROSSES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Crosses({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; + return "NOT ST_Crosses({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; case Query::TYPE_DISTANCE_EQUAL: case Query::TYPE_DISTANCE_NOT_EQUAL: @@ -1431,43 +1431,43 @@ protected function handleSpatialQueries(Query $query, array &$binds, string $att case Query::TYPE_INTERSECTS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; + return "ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; case Query::TYPE_NOT_INTERSECTS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; + return "NOT ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; case Query::TYPE_OVERLAPS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; + return "ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; case Query::TYPE_NOT_OVERLAPS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; + return "NOT ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; case Query::TYPE_TOUCHES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; + return "ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; case Query::TYPE_NOT_TOUCHES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; + return "NOT ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; case Query::TYPE_EQUAL: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; + return "ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326))"; case Query::TYPE_NOT_EQUAL: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; + return "NOT ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; case Query::TYPE_CONTAINS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Contains({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; + return "ST_Contains({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; case Query::TYPE_NOT_CONTAINS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Contains({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 'axis-order=long-lat'))"; + return "NOT ST_Contains({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; default: throw new DatabaseException('Unknown spatial query method: ' . $query->getMethod()); @@ -1641,7 +1641,7 @@ protected function getSQLType(string $type, int $size, bool $signed = true, bool case Database::VAR_POINT: - $type = 'POINT'; + $type = 'POINT SRID 4326'; if (!$this->getSupportForSpatialIndexNull()) { if ($required) { $type .= ' NOT NULL'; @@ -1652,7 +1652,7 @@ protected function getSQLType(string $type, int $size, bool $signed = true, bool return $type; case Database::VAR_LINESTRING: - $type = 'LINESTRING'; + $type = 'LINESTRING SRID 4326'; if (!$this->getSupportForSpatialIndexNull()) { if ($required) { $type .= ' NOT NULL'; @@ -1664,7 +1664,7 @@ protected function getSQLType(string $type, int $size, bool $signed = true, bool case Database::VAR_POLYGON: - $type = 'POLYGON'; + $type = 'POLYGON SRID 4326'; if (!$this->getSupportForSpatialIndexNull()) { if ($required) { $type .= ' NOT NULL'; @@ -1913,4 +1913,43 @@ public function getSupportForDistanceBetweenMultiDimensionGeometryInMeters(): bo { return false; } + + public function getSpatialSQLType(string $type, bool $required): string{ + switch ($type) { + case Database::VAR_POINT: + $type = 'POINT SRID 4326'; + if (!$this->getSupportForSpatialIndexNull()) { + if ($required) { + $type .= ' NOT NULL'; + } else { + $type .= ' NULL'; + } + } + return $type; + + case Database::VAR_LINESTRING: + $type = 'LINESTRING SRID 4326'; + if (!$this->getSupportForSpatialIndexNull()) { + if ($required) { + $type .= ' NOT NULL'; + } else { + $type .= ' NULL'; + } + } + return $type; + + + case Database::VAR_POLYGON: + $type = 'POLYGON SRID 4326'; + if (!$this->getSupportForSpatialIndexNull()) { + if ($required) { + $type .= ' NOT NULL'; + } else { + $type .= ' NULL'; + } + } + return $type; + } + return ''; + } } diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index f15e5d76c..136da70a2 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -117,12 +117,14 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str if ($useMeters) { $attr = "ST_SRID({$alias}.{$attribute}, " . Database::SRID . ")"; - $geom = "ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . ",'axis-order=long-lat')"; + $geom = "ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . ", 'axis-order=lat-long' )"; return "ST_Distance({$attr}, {$geom}, 'metre') {$operator} :{$placeholder}_1"; } - + $attr = "ST_GeomFromText(ST_AsText({$alias}.{$attribute}), 0, 'axis-order=lat-long')"; + $geom = "ST_GeomFromText(:{$placeholder}_0, 0, 'axis-order=lat-long')"; + return "ST_Distance({$attr}, {$geom}) {$operator} :{$placeholder}_1"; // Without meters, use default behavior - return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0)) {$operator} :{$placeholder}_1"; + // return "ST_Distance(ST_GeomFromText({$alias}.{$attribute}, 0, 'axis-order=lat-long')), ST_GeomFromText(:{$placeholder}_0, 0, 'axis-order=lat-long')) {$operator} :{$placeholder}_1"; } public function getSupportForIndexArray(): bool @@ -184,4 +186,46 @@ public function getSupportForDistanceBetweenMultiDimensionGeometryInMeters(): bo { return true; } -} + + /** + * Spatial type attribute + */ + public function getSpatialSQLType(string $type, bool $required): string{ + switch ($type) { + case Database::VAR_POINT: + $type = 'POINT SRID 4326'; + if (!$this->getSupportForSpatialIndexNull()) { + if ($required) { + $type .= ' NOT NULL'; + } else { + $type .= ' NULL'; + } + } + return $type; + + case Database::VAR_LINESTRING: + $type = 'LINESTRING SRID 4326'; + if (!$this->getSupportForSpatialIndexNull()) { + if ($required) { + $type .= ' NOT NULL'; + } else { + $type .= ' NULL'; + } + } + return $type; + + + case Database::VAR_POLYGON: + $type = 'POLYGON SRID 4326'; + if (!$this->getSupportForSpatialIndexNull()) { + if ($required) { + $type .= ' NOT NULL'; + } else { + $type .= ' NULL'; + } + } + return $type; + } + return ''; + } + } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 0522bfc0d..bf959648a 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -483,7 +483,7 @@ public function updateDocuments(Document $collection, Document $updates, array $ $column = $this->filter($attribute); if (in_array($attribute, $spatialAttributes)) { - $columns .= "{$this->quote($column)} = ST_GeomFromText(:key_{$bindIndex})"; + $columns .= "{$this->quote($column)} = ST_GeomFromText(:key_{$bindIndex} , 4326, 'axis-order=lat-long')"; } else { $columns .= "{$this->quote($column)} = :key_{$bindIndex}"; } @@ -2019,7 +2019,7 @@ public function createDocuments(Document $collection, array $documents): array } if (in_array($key, $spatialAttributes)) { $bindKey = 'key_' . $bindIndex; - $bindKeys[] = "ST_GeomFromText(:" . $bindKey . ")"; + $bindKeys[] = "ST_GeomFromText(:" . $bindKey . ", 4326 , 'axis-order=lat-long')"; } else { $value = (\is_bool($value)) ? (int)$value : $value; $bindKey = 'key_' . $bindIndex; @@ -2145,7 +2145,7 @@ public function createOrUpdateDocuments( if (in_array($attributeKey, $spatialAttributes)) { $bindKey = 'key_' . $bindIndex; - $bindKeys[] = "ST_GeomFromText(:" . $bindKey . ")"; + $bindKeys[] = "ST_GeomFromText(:" . $bindKey . " , 4326, 'axis-order=lat-long')"; } else { $attrValue = (\is_bool($attrValue)) ? (int)$attrValue : $attrValue; $bindKey = 'key_' . $bindIndex; @@ -2667,4 +2667,33 @@ public function getSpatialTypeFromWKT(string $wkt): string } return strtolower(trim(substr($wkt, 0, $pos))); } + + /** + * Interpolate SQL with its bound parameters. + * + * @param string $query The SQL query with placeholders. + * @param array $params The key => value bindings. + * @return string The interpolated SQL query (for debugging/logging only). + */ +function interpolateQuery(string $query, array $params): string { + $keys = []; + $values = []; + + foreach ($params as $key => $value) { + // Handle named or positional placeholders + $keys[] = is_string($key) ? '/:' . preg_quote($key, '/') . '/' : '/[?]/'; + + // Quote values properly + if (is_null($value)) { + $values[] = 'NULL'; + } elseif (is_numeric($value)) { + $values[] = $value; + } else { + // Escape single quotes inside the string + $values[] = "'" . str_replace("'", "''", $value) . "'"; + } + } + + return preg_replace($keys, $values, $query, 1); +} } diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 37ad7cce3..e278ef180 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -18,14 +18,14 @@ abstract class Base extends TestCase { - use CollectionTests; - use DocumentTests; - use AttributeTests; - use IndexTests; - use PermissionTests; - use RelationshipTests; + // use CollectionTests; + // use DocumentTests; + // use AttributeTests; + // use IndexTests; + // use PermissionTests; + // use RelationshipTests; + // use GeneralTests; use SpatialTests; - use GeneralTests; protected static string $namespace; diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 5e1a4f4e4..53158d8f4 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -129,8 +129,10 @@ public function testSpatialTypeDocuments(): void // Update spatial data $doc1->setAttribute('pointAttr', [6.0, 6.0]); $updatedDoc = $database->updateDocument($collectionName, 'doc1', $doc1); + $this->assertEquals([6.0, 6.0], $updatedDoc->getAttribute('pointAttr')); + // Test spatial queries with appropriate operations for each geometry type // Point attribute tests - use operations valid for points $pointQueries = [ From 4d84e6cdc7d82506481c67ed3156ec3875197e23 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 11 Sep 2025 12:39:24 +0300 Subject: [PATCH 55/69] Refactor database cursor handling and update docker-compose for MongoDB integration. Cleaned up code in Database.php for better cursor encoding and ensured proper initialization. Adjusted DocumentTests for improved readability and consistency. --- src/Database/Database.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 2610e7564..7c195354e 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -6347,11 +6347,13 @@ public function find(string $collection, array $queries = [], string $forPermiss } if (!empty($cursor)) { + $cursor = $this->encode($collection, $cursor); $cursor = $this->adapter->castingBefore($collection, $cursor); + $cursor = $cursor->getArrayCopy(); + } else { + $cursor = []; } - - $cursor = empty($cursor) ? [] : $this->encode($collection, $cursor)->getArrayCopy(); - + /** @var array $queries */ $queries = \array_merge( $selects, From aac79fe12375c4fe904dca281d1664ebd8eab746 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 11 Sep 2025 12:40:50 +0300 Subject: [PATCH 56/69] Comment out the testFindNotBetween method in DocumentTests.php to temporarily disable it for further review and refinement. --- tests/e2e/Adapter/Scopes/DocumentTests.php | 146 ++++++++++----------- 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 4476ad9c2..24e3b173a 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -3484,79 +3484,79 @@ public function testFindNotContains(): void // $this->assertLessThanOrEqual(5, count($documents)); // But still excluding Marvel movies // } -// public function testFindNotBetween(): void -// { -// /** @var Database $database */ -// $database = static::getDatabase(); -// -// // Test notBetween with price range - should return documents outside the range -// $documents = $database->find('movies', [ -// Query::notBetween('price', 25.94, 25.99), -// ]); -// $this->assertEquals(4, count($documents)); // All movies except the 2 in the price range -// -// // Test notBetween with range that includes no documents - should return all documents -// $documents = $database->find('movies', [ -// Query::notBetween('price', 30, 35), -// ]); -// $this->assertEquals(6, count($documents)); -// -// // Test notBetween with date range -// $documents = $database->find('movies', [ -// Query::notBetween('$createdAt', '1975-12-06', '2050-12-06'), -// ]); -// $this->assertEquals(0, count($documents)); // No movies outside this wide date range -// -// // Test notBetween with narrower date range -// $documents = $database->find('movies', [ -// Query::notBetween('$createdAt', '2000-01-01', '2001-01-01'), -// ]); -// $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range -// -// // Test notBetween with updated date range -// $documents = $database->find('movies', [ -// Query::notBetween('$updatedAt', '2000-01-01T00:00:00.000+00:00', '2001-01-01T00:00:00.000+00:00'), -// ]); -// $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range -// -// // Test notBetween with year range (integer values) -// $documents = $database->find('movies', [ -// Query::notBetween('year', 2005, 2007), -// ]); -// $this->assertLessThanOrEqual(6, count($documents)); // Movies outside 2005-2007 range -// -// // Test notBetween with reversed range (start > end) - should still work -// $documents = $database->find('movies', [ -// Query::notBetween('price', 25.99, 25.94), // Note: reversed order -// ]); -// $this->assertGreaterThanOrEqual(4, count($documents)); // Should handle reversed range gracefully -// -// // Test notBetween with same start and end values -// $documents = $database->find('movies', [ -// Query::notBetween('year', 2006, 2006), -// ]); -// $this->assertGreaterThanOrEqual(5, count($documents)); // All movies except those from exactly 2006 -// -// // Test notBetween combined with other filters -// $documents = $database->find('movies', [ -// Query::notBetween('price', 25.94, 25.99), -// Query::orderDesc('year'), -// Query::limit(2) -// ]); -// $this->assertEquals(2, count($documents)); // Limited results, ordered, excluding price range -// -// // Test notBetween with extreme ranges -// $documents = $database->find('movies', [ -// Query::notBetween('year', -1000, 1000), // Very wide range -// ]); -// $this->assertLessThanOrEqual(6, count($documents)); // Movies outside this range -// -// // Test notBetween with float precision -// $documents = $database->find('movies', [ -// Query::notBetween('price', 25.945, 25.955), // Very narrow range -// ]); -// $this->assertGreaterThanOrEqual(4, count($documents)); // Most movies should be outside this narrow range -// } + // public function testFindNotBetween(): void + // { + // /** @var Database $database */ + // $database = static::getDatabase(); + // + // // Test notBetween with price range - should return documents outside the range + // $documents = $database->find('movies', [ + // Query::notBetween('price', 25.94, 25.99), + // ]); + // $this->assertEquals(4, count($documents)); // All movies except the 2 in the price range + // + // // Test notBetween with range that includes no documents - should return all documents + // $documents = $database->find('movies', [ + // Query::notBetween('price', 30, 35), + // ]); + // $this->assertEquals(6, count($documents)); + // + // // Test notBetween with date range + // $documents = $database->find('movies', [ + // Query::notBetween('$createdAt', '1975-12-06', '2050-12-06'), + // ]); + // $this->assertEquals(0, count($documents)); // No movies outside this wide date range + // + // // Test notBetween with narrower date range + // $documents = $database->find('movies', [ + // Query::notBetween('$createdAt', '2000-01-01', '2001-01-01'), + // ]); + // $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range + // + // // Test notBetween with updated date range + // $documents = $database->find('movies', [ + // Query::notBetween('$updatedAt', '2000-01-01T00:00:00.000+00:00', '2001-01-01T00:00:00.000+00:00'), + // ]); + // $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range + // + // // Test notBetween with year range (integer values) + // $documents = $database->find('movies', [ + // Query::notBetween('year', 2005, 2007), + // ]); + // $this->assertLessThanOrEqual(6, count($documents)); // Movies outside 2005-2007 range + // + // // Test notBetween with reversed range (start > end) - should still work + // $documents = $database->find('movies', [ + // Query::notBetween('price', 25.99, 25.94), // Note: reversed order + // ]); + // $this->assertGreaterThanOrEqual(4, count($documents)); // Should handle reversed range gracefully + // + // // Test notBetween with same start and end values + // $documents = $database->find('movies', [ + // Query::notBetween('year', 2006, 2006), + // ]); + // $this->assertGreaterThanOrEqual(5, count($documents)); // All movies except those from exactly 2006 + // + // // Test notBetween combined with other filters + // $documents = $database->find('movies', [ + // Query::notBetween('price', 25.94, 25.99), + // Query::orderDesc('year'), + // Query::limit(2) + // ]); + // $this->assertEquals(2, count($documents)); // Limited results, ordered, excluding price range + // + // // Test notBetween with extreme ranges + // $documents = $database->find('movies', [ + // Query::notBetween('year', -1000, 1000), // Very wide range + // ]); + // $this->assertLessThanOrEqual(6, count($documents)); // Movies outside this range + // + // // Test notBetween with float precision + // $documents = $database->find('movies', [ + // Query::notBetween('price', 25.945, 25.955), // Very narrow range + // ]); + // $this->assertGreaterThanOrEqual(4, count($documents)); // Most movies should be outside this narrow range + // } public function testFindSelect(): void { From c967e720efb4b5ad6f14b8313c85faac3d7bdb16 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 11 Sep 2025 12:50:34 +0300 Subject: [PATCH 57/69] linter --- src/Database/Database.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 7c195354e..4e501f193 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -6353,7 +6353,7 @@ public function find(string $collection, array $queries = [], string $forPermiss } else { $cursor = []; } - + /** @var array $queries */ $queries = \array_merge( $selects, From 32bf039d536a64ada3644053e19ab69d7af9f1e1 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 11 Sep 2025 15:57:47 +0530 Subject: [PATCH 58/69] SRID and axis order updates * SRID fix for mariadb/mysql * Axis order updates for mysql --- src/Database/Adapter.php | 7 ++ src/Database/Adapter/MariaDB.php | 133 ++++++++-------------- src/Database/Adapter/MySQL.php | 79 +++++++------ src/Database/Adapter/Pool.php | 10 ++ src/Database/Adapter/Postgres.php | 38 ++++--- src/Database/Adapter/SQL.php | 76 +++++++------ src/Database/Adapter/SQLite.php | 10 ++ tests/e2e/Adapter/Scopes/SpatialTests.php | 2 +- 8 files changed, 190 insertions(+), 165 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index db58e4fa4..4e2990e70 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -1071,6 +1071,13 @@ abstract public function getSupportForSpatialIndexNull(): bool; */ abstract public function getSupportForSpatialIndexOrder(): bool; + /** + * Does the adapter support spatial axis order specification? + * + * @return bool + */ + abstract public function getSupportForSpatialAxisOrder(): bool; + /** * Does the adapter includes boundary during spatial contains? * diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 807d569b7..d13a133b9 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -846,7 +846,7 @@ public function createDocument(Document $collection, Document $document): Docume $bindKey = 'key_' . $bindIndex; $columns .= "`{$column}`, "; if (in_array($attribute, $spatialAttributes)) { - $columnNames .= 'ST_GeomFromText(:' . $bindKey . ', ' . Database::SRID . ", 'axis-order=lat-long'), "; + $columnNames .= $this->getSpatialGeomFromText(':' . $bindKey) . ", "; } else { $columnNames .= ':' . $bindKey . ', '; } @@ -1116,7 +1116,7 @@ public function updateDocument(Document $collection, string $id, Document $docum $bindKey = 'key_' . $bindIndex; if (in_array($attribute, $spatialAttributes)) { - $columns .= "`{$column}`" . '=ST_GeomFromText(:' . $bindKey . ' , ' . Database::SRID . ' ),'; + $columns .= "`{$column}`" . '=' . $this->getSpatialGeomFromText(':' . $bindKey) . ','; } else { $columns .= "`{$column}`" . '=:' . $bindKey . ','; } @@ -1396,9 +1396,9 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str if ($wktType != Database::VAR_POINT || $attrType != Database::VAR_POINT) { throw new QueryException('Distance in meters is not supported between '.$attrType . ' and '. $wktType); } - return "ST_DISTANCE_SPHERE({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0), 6371000) {$operator} :{$placeholder}_1"; + return "ST_DISTANCE_SPHERE({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ", 6371000) {$operator} :{$placeholder}_1"; } - return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0)) {$operator} :{$placeholder}_1"; + return "ST_Distance({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ") {$operator} :{$placeholder}_1"; } /** @@ -1417,11 +1417,11 @@ protected function handleSpatialQueries(Query $query, array &$binds, string $att switch ($query->getMethod()) { case Query::TYPE_CROSSES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Crosses({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; + return "ST_Crosses({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_NOT_CROSSES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Crosses({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; + return "NOT ST_Crosses({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_DISTANCE_EQUAL: case Query::TYPE_DISTANCE_NOT_EQUAL: @@ -1431,43 +1431,43 @@ protected function handleSpatialQueries(Query $query, array &$binds, string $att case Query::TYPE_INTERSECTS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; + return "ST_Intersects({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_NOT_INTERSECTS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; + return "NOT ST_Intersects({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_OVERLAPS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; + return "ST_Overlaps({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_NOT_OVERLAPS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; + return "NOT ST_Overlaps({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_TOUCHES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; + return "ST_Touches({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_NOT_TOUCHES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; + return "NOT ST_Touches({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_EQUAL: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326))"; + return "ST_Equals({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_NOT_EQUAL: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; + return "NOT ST_Equals({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_CONTAINS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Contains({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; + return "ST_Contains({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_NOT_CONTAINS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Contains({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, 4326, 'axis-order=lat-long'))"; + return "NOT ST_Contains({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; default: throw new DatabaseException('Unknown spatial query method: ' . $query->getMethod()); @@ -1593,6 +1593,9 @@ protected function getSQLCondition(Query $query, array &$binds, array $attribute */ protected function getSQLType(string $type, int $size, bool $signed = true, bool $array = false, bool $required = false): string { + if (in_array($type, Database::SPATIAL_TYPES)) { + return $this->getSpatialSQLType($type, $required); + } if ($array === true) { return 'JSON'; } @@ -1639,42 +1642,6 @@ protected function getSQLType(string $type, int $size, bool $signed = true, bool case Database::VAR_DATETIME: return 'DATETIME(3)'; - - case Database::VAR_POINT: - $type = 'POINT SRID 4326'; - if (!$this->getSupportForSpatialIndexNull()) { - if ($required) { - $type .= ' NOT NULL'; - } else { - $type .= ' NULL'; - } - } - return $type; - - case Database::VAR_LINESTRING: - $type = 'LINESTRING SRID 4326'; - if (!$this->getSupportForSpatialIndexNull()) { - if ($required) { - $type .= ' NOT NULL'; - } else { - $type .= ' NULL'; - } - } - return $type; - - - case Database::VAR_POLYGON: - $type = 'POLYGON SRID 4326'; - if (!$this->getSupportForSpatialIndexNull()) { - if ($required) { - $type .= ' NOT NULL'; - } else { - $type .= ' NULL'; - } - } - return $type; - - default: throw new DatabaseException('Unknown type: ' . $type . '. Must be one of ' . Database::VAR_STRING . ', ' . Database::VAR_INTEGER . ', ' . Database::VAR_FLOAT . ', ' . Database::VAR_BOOLEAN . ', ' . Database::VAR_DATETIME . ', ' . Database::VAR_RELATIONSHIP . ', ' . ', ' . Database::VAR_POINT . ', ' . Database::VAR_LINESTRING . ', ' . Database::VAR_POLYGON); } @@ -1914,42 +1881,40 @@ public function getSupportForDistanceBetweenMultiDimensionGeometryInMeters(): bo return false; } - public function getSpatialSQLType(string $type, bool $required): string{ - switch ($type) { - case Database::VAR_POINT: - $type = 'POINT SRID 4326'; - if (!$this->getSupportForSpatialIndexNull()) { - if ($required) { - $type .= ' NOT NULL'; - } else { - $type .= ' NULL'; - } - } - return $type; + public function getSpatialSQLType(string $type, bool $required): string + { + $srid = Database::SRID; + $nullability = ''; - case Database::VAR_LINESTRING: - $type = 'LINESTRING SRID 4326'; - if (!$this->getSupportForSpatialIndexNull()) { - if ($required) { - $type .= ' NOT NULL'; - } else { - $type .= ' NULL'; - } + if (!$this->getSupportForSpatialIndexNull()) { + if ($required) { + $nullability = ' NOT NULL'; + } else { + $nullability = ' NULL'; } - return $type; + } + switch ($type) { + case Database::VAR_POINT: + return "POINT($srid)$nullability"; - case Database::VAR_POLYGON: - $type = 'POLYGON SRID 4326'; - if (!$this->getSupportForSpatialIndexNull()) { - if ($required) { - $type .= ' NOT NULL'; - } else { - $type .= ' NULL'; - } - } - return $type; + case Database::VAR_LINESTRING: + return "LINESTRING($srid)$nullability"; + + case Database::VAR_POLYGON: + return "POLYGON($srid)$nullability"; + } + + return ''; } - return ''; + + /** + * Does the adapter support spatial axis order specification? + * + * @return bool + */ + public function getSupportForSpatialAxisOrder(): bool + { + return false; } } diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index 136da70a2..4e749b6e7 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -117,11 +117,11 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str if ($useMeters) { $attr = "ST_SRID({$alias}.{$attribute}, " . Database::SRID . ")"; - $geom = "ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . ", 'axis-order=lat-long' )"; + $geom = $this->getSpatialGeomFromText(":{$placeholder}_0"); return "ST_Distance({$attr}, {$geom}, 'metre') {$operator} :{$placeholder}_1"; } - $attr = "ST_GeomFromText(ST_AsText({$alias}.{$attribute}), 0, 'axis-order=lat-long')"; - $geom = "ST_GeomFromText(:{$placeholder}_0, 0, 'axis-order=lat-long')"; + $attr = "ST_GeomFromText(ST_AsText({$alias}.{$attribute}), 0)"; + $geom = $this->getSpatialGeomFromText(":{$placeholder}_0", 0); return "ST_Distance({$attr}, {$geom}) {$operator} :{$placeholder}_1"; // Without meters, use default behavior // return "ST_Distance(ST_GeomFromText({$alias}.{$attribute}, 0, 'axis-order=lat-long')), ST_GeomFromText(:{$placeholder}_0, 0, 'axis-order=lat-long')) {$operator} :{$placeholder}_1"; @@ -190,42 +190,53 @@ public function getSupportForDistanceBetweenMultiDimensionGeometryInMeters(): bo /** * Spatial type attribute */ - public function getSpatialSQLType(string $type, bool $required): string{ + public function getSpatialSQLType(string $type, bool $required): string + { switch ($type) { - case Database::VAR_POINT: - $type = 'POINT SRID 4326'; - if (!$this->getSupportForSpatialIndexNull()) { - if ($required) { - $type .= ' NOT NULL'; - } else { - $type .= ' NULL'; + case Database::VAR_POINT: + $type = 'POINT SRID 4326'; + if (!$this->getSupportForSpatialIndexNull()) { + if ($required) { + $type .= ' NOT NULL'; + } else { + $type .= ' NULL'; + } } - } - return $type; - - case Database::VAR_LINESTRING: - $type = 'LINESTRING SRID 4326'; - if (!$this->getSupportForSpatialIndexNull()) { - if ($required) { - $type .= ' NOT NULL'; - } else { - $type .= ' NULL'; + return $type; + + case Database::VAR_LINESTRING: + $type = 'LINESTRING SRID 4326'; + if (!$this->getSupportForSpatialIndexNull()) { + if ($required) { + $type .= ' NOT NULL'; + } else { + $type .= ' NULL'; + } } - } - return $type; + return $type; - case Database::VAR_POLYGON: - $type = 'POLYGON SRID 4326'; - if (!$this->getSupportForSpatialIndexNull()) { - if ($required) { - $type .= ' NOT NULL'; - } else { - $type .= ' NULL'; + case Database::VAR_POLYGON: + $type = 'POLYGON SRID 4326'; + if (!$this->getSupportForSpatialIndexNull()) { + if ($required) { + $type .= ' NOT NULL'; + } else { + $type .= ' NULL'; + } } - } - return $type; - } - return ''; + return $type; + } + return ''; } + + /** + * Does the adapter support spatial axis order specification? + * + * @return bool + */ + public function getSupportForSpatialAxisOrder(): bool + { + return false; // Temporarily disable to test } +} diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index 27dfd6bb8..e4d85f9e8 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -539,4 +539,14 @@ public function getSupportForDistanceBetweenMultiDimensionGeometryInMeters(): bo { return $this->delegate(__FUNCTION__, \func_get_args()); } + + /** + * Does the adapter support spatial axis order specification? + * + * @return bool + */ + public function getSupportForSpatialAxisOrder(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 1276cea57..f666d1184 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1490,12 +1490,12 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str if ($meters) { $attr = "({$alias}.{$attribute}::geography)"; - $geom = "ST_SetSRID(ST_GeomFromText(:{$placeholder}_0), " . Database::SRID . ")::geography"; + $geom = "ST_SetSRID(" . $this->getSpatialGeomFromText(":{$placeholder}_0", null) . ", " . Database::SRID . ")::geography"; return "ST_Distance({$attr}, {$geom}) {$operator} :{$placeholder}_1"; } // Without meters, use the original SRID (e.g., 4326) - return "ST_Distance({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . ")) {$operator} :{$placeholder}_1"; + return "ST_Distance({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ") {$operator} :{$placeholder}_1"; } @@ -1514,11 +1514,11 @@ protected function handleSpatialQueries(Query $query, array &$binds, string $att switch ($query->getMethod()) { case Query::TYPE_CROSSES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Crosses({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "ST_Crosses({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_NOT_CROSSES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Crosses({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0))"; + return "NOT ST_Crosses({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_DISTANCE_EQUAL: case Query::TYPE_DISTANCE_NOT_EQUAL: @@ -1527,35 +1527,35 @@ protected function handleSpatialQueries(Query $query, array &$binds, string $att return $this->handleDistanceSpatialQueries($query, $binds, $attribute, $alias, $placeholder); case Query::TYPE_EQUAL: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; + return "ST_Equals({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_NOT_EQUAL: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Equals({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; + return "NOT ST_Equals({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_INTERSECTS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; + return "ST_Intersects({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_NOT_INTERSECTS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Intersects({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; + return "NOT ST_Intersects({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_OVERLAPS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; + return "ST_Overlaps({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_NOT_OVERLAPS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Overlaps({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; + return "NOT ST_Overlaps({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_TOUCHES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; + return "ST_Touches({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_NOT_TOUCHES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Touches({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; + return "NOT ST_Touches({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; case Query::TYPE_CONTAINS: case Query::TYPE_NOT_CONTAINS: @@ -1564,8 +1564,8 @@ protected function handleSpatialQueries(Query $query, array &$binds, string $att $isNot = $query->getMethod() === Query::TYPE_NOT_CONTAINS; $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); return $isNot - ? "NOT ST_Covers({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))" - : "ST_Covers({$alias}.{$attribute}, ST_GeomFromText(:{$placeholder}_0, " . Database::SRID . "))"; + ? "NOT ST_Covers({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")" + : "ST_Covers({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; default: throw new DatabaseException('Unknown spatial query method: ' . $query->getMethod()); @@ -1992,4 +1992,14 @@ public function getSupportForDistanceBetweenMultiDimensionGeometryInMeters(): bo { return true; } + + /** + * Does the adapter support spatial axis order specification? + * + * @return bool + */ + public function getSupportForSpatialAxisOrder(): bool + { + return false; + } } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index bf959648a..189f099c1 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -483,7 +483,7 @@ public function updateDocuments(Document $collection, Document $updates, array $ $column = $this->filter($attribute); if (in_array($attribute, $spatialAttributes)) { - $columns .= "{$this->quote($column)} = ST_GeomFromText(:key_{$bindIndex} , 4326, 'axis-order=lat-long')"; + $columns .= "{$this->quote($column)} = " . $this->getSpatialGeomFromText(":key_{$bindIndex}"); } else { $columns .= "{$this->quote($column)} = :key_{$bindIndex}"; } @@ -1513,6 +1513,47 @@ public function getSupportForSpatialIndexOrder(): bool return false; } + /** + * Does the adapter support spatial axis order specification? + * + * @return bool + */ + public function getSupportForSpatialAxisOrder(): bool + { + return false; + } + + /** + * Generate ST_GeomFromText call with proper SRID and axis order support + * + * @param string $wktPlaceholder + * @param int|null $srid + * @return string + */ + protected function getSpatialGeomFromText(string $wktPlaceholder, ?int $srid = null): string + { + $srid = $srid ?? Database::SRID; + $geomFromText = "ST_GeomFromText({$wktPlaceholder}, {$srid}"; + + if ($this->getSupportForSpatialAxisOrder()) { + $geomFromText .= ", " . $this->getSpatialAxisOrderSpec(); + } + + $geomFromText .= ")"; + + return $geomFromText; + } + + /** + * Get the spatial axis order specification string + * + * @return string + */ + protected function getSpatialAxisOrderSpec(): string + { + return "'axis-order=long-lat'"; + } + /** * @param string $tableName * @param string $columns @@ -2019,7 +2060,7 @@ public function createDocuments(Document $collection, array $documents): array } if (in_array($key, $spatialAttributes)) { $bindKey = 'key_' . $bindIndex; - $bindKeys[] = "ST_GeomFromText(:" . $bindKey . ", 4326 , 'axis-order=lat-long')"; + $bindKeys[] = $this->getSpatialGeomFromText(":" . $bindKey); } else { $value = (\is_bool($value)) ? (int)$value : $value; $bindKey = 'key_' . $bindIndex; @@ -2145,7 +2186,7 @@ public function createOrUpdateDocuments( if (in_array($attributeKey, $spatialAttributes)) { $bindKey = 'key_' . $bindIndex; - $bindKeys[] = "ST_GeomFromText(:" . $bindKey . " , 4326, 'axis-order=lat-long')"; + $bindKeys[] = $this->getSpatialGeomFromText(":" . $bindKey); } else { $attrValue = (\is_bool($attrValue)) ? (int)$attrValue : $attrValue; $bindKey = 'key_' . $bindIndex; @@ -2667,33 +2708,4 @@ public function getSpatialTypeFromWKT(string $wkt): string } return strtolower(trim(substr($wkt, 0, $pos))); } - - /** - * Interpolate SQL with its bound parameters. - * - * @param string $query The SQL query with placeholders. - * @param array $params The key => value bindings. - * @return string The interpolated SQL query (for debugging/logging only). - */ -function interpolateQuery(string $query, array $params): string { - $keys = []; - $values = []; - - foreach ($params as $key => $value) { - // Handle named or positional placeholders - $keys[] = is_string($key) ? '/:' . preg_quote($key, '/') . '/' : '/[?]/'; - - // Quote values properly - if (is_null($value)) { - $values[] = 'NULL'; - } elseif (is_numeric($value)) { - $values[] = $value; - } else { - // Escape single quotes inside the string - $values[] = "'" . str_replace("'", "''", $value) . "'"; - } - } - - return preg_replace($keys, $values, $query, 1); -} } diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 54ffb47dd..ff0246265 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -1274,4 +1274,14 @@ public function getSupportForDistanceBetweenMultiDimensionGeometryInMeters(): bo { return false; } + + /** + * Does the adapter support spatial axis order specification? + * + * @return bool + */ + public function getSupportForSpatialAxisOrder(): bool + { + return false; + } } diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 53158d8f4..b2ef0ae79 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -1147,7 +1147,7 @@ public function testComplexGeometricShapes(): void // Test triangle doesn't intersect with distant point $nonIntersectingTriangle = $database->find($collectionName, [ - Query::notIntersects('triangle', [100, 100]) // Distant point should not intersect + Query::notIntersects('triangle', [10, 10]) // Distant point should not intersect ], Database::PERMISSION_READ); $this->assertNotEmpty($nonIntersectingTriangle); From f74bc0954a8803d3ef5368ed5cbda18627213813 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 11 Sep 2025 16:12:45 +0530 Subject: [PATCH 59/69] using earth radius through a const --- src/Database/Adapter/MariaDB.php | 2 +- src/Database/Database.php | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index d13a133b9..571d74158 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1396,7 +1396,7 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str if ($wktType != Database::VAR_POINT || $attrType != Database::VAR_POINT) { throw new QueryException('Distance in meters is not supported between '.$attrType . ' and '. $wktType); } - return "ST_DISTANCE_SPHERE({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ", 6371000) {$operator} :{$placeholder}_1"; + return "ST_DISTANCE_SPHERE({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ", " . Database::EARTH_RADIUS . ") {$operator} :{$placeholder}_1"; } return "ST_Distance({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ") {$operator} :{$placeholder}_1"; } diff --git a/src/Database/Database.php b/src/Database/Database.php index b6163fe4f..72fd8574c 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -51,6 +51,7 @@ class Database // Global SRID for geographic coordinates (WGS84) public const SRID = 4326; + public const EARTH_RADIUS = 6371000; // Relationship Types public const VAR_RELATIONSHIP = 'relationship'; From cbabdbf141be34a384f72c49ba9556a7dae0c2ca Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 11 Sep 2025 16:17:32 +0530 Subject: [PATCH 60/69] reenabled tests --- tests/e2e/Adapter/Base.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index e278ef180..50b7b3fee 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -18,13 +18,13 @@ abstract class Base extends TestCase { - // use CollectionTests; - // use DocumentTests; - // use AttributeTests; - // use IndexTests; - // use PermissionTests; - // use RelationshipTests; - // use GeneralTests; + use CollectionTests; + use DocumentTests; + use AttributeTests; + use IndexTests; + use PermissionTests; + use RelationshipTests; + use GeneralTests; use SpatialTests; protected static string $namespace; From 0c9f864a36f733bbbac13d465470d829a23cf217 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 11 Sep 2025 17:39:15 +0530 Subject: [PATCH 61/69] updated tests --- src/Database/Adapter/MariaDB.php | 28 +++++++++++------------ src/Database/Adapter/MySQL.php | 20 ++++++++++++---- tests/e2e/Adapter/Scopes/SpatialTests.php | 22 +++++++++++++++--- 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 571d74158..c78d6637c 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1396,9 +1396,9 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str if ($wktType != Database::VAR_POINT || $attrType != Database::VAR_POINT) { throw new QueryException('Distance in meters is not supported between '.$attrType . ' and '. $wktType); } - return "ST_DISTANCE_SPHERE({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ", " . Database::EARTH_RADIUS . ") {$operator} :{$placeholder}_1"; + return "ST_DISTANCE_SPHERE({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0", null) . ", " . Database::EARTH_RADIUS . ") {$operator} :{$placeholder}_1"; } - return "ST_Distance({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ") {$operator} :{$placeholder}_1"; + return "ST_Distance({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0", null) . ") {$operator} :{$placeholder}_1"; } /** @@ -1417,11 +1417,11 @@ protected function handleSpatialQueries(Query $query, array &$binds, string $att switch ($query->getMethod()) { case Query::TYPE_CROSSES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Crosses({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; + return "ST_Crosses({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0", null) . ")"; case Query::TYPE_NOT_CROSSES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Crosses({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; + return "NOT ST_Crosses({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0", null) . ")"; case Query::TYPE_DISTANCE_EQUAL: case Query::TYPE_DISTANCE_NOT_EQUAL: @@ -1431,43 +1431,43 @@ protected function handleSpatialQueries(Query $query, array &$binds, string $att case Query::TYPE_INTERSECTS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Intersects({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; + return "ST_Intersects({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0", null) . ")"; case Query::TYPE_NOT_INTERSECTS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Intersects({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; + return "NOT ST_Intersects({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0", null) . ")"; case Query::TYPE_OVERLAPS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Overlaps({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; + return "ST_Overlaps({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0", null) . ")"; case Query::TYPE_NOT_OVERLAPS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Overlaps({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; + return "NOT ST_Overlaps({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0", null) . ")"; case Query::TYPE_TOUCHES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Touches({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; + return "ST_Touches({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0", null) . ")"; case Query::TYPE_NOT_TOUCHES: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Touches({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; + return "NOT ST_Touches({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0", null) . ")"; case Query::TYPE_EQUAL: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Equals({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; + return "ST_Equals({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0", null) . ")"; case Query::TYPE_NOT_EQUAL: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Equals({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; + return "NOT ST_Equals({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0", null) . ")"; case Query::TYPE_CONTAINS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "ST_Contains({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; + return "ST_Contains({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0", null) . ")"; case Query::TYPE_NOT_CONTAINS: $binds[":{$placeholder}_0"] = $this->convertArrayToWKT($query->getValues()[0]); - return "NOT ST_Contains({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0") . ")"; + return "NOT ST_Contains({$alias}.{$attribute}, " . $this->getSpatialGeomFromText(":{$placeholder}_0", null) . ")"; default: throw new DatabaseException('Unknown spatial query method: ' . $query->getMethod()); diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index 4e749b6e7..bd3da3d0b 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -117,14 +117,13 @@ protected function handleDistanceSpatialQueries(Query $query, array &$binds, str if ($useMeters) { $attr = "ST_SRID({$alias}.{$attribute}, " . Database::SRID . ")"; - $geom = $this->getSpatialGeomFromText(":{$placeholder}_0"); + $geom = $this->getSpatialGeomFromText(":{$placeholder}_0", null); return "ST_Distance({$attr}, {$geom}, 'metre') {$operator} :{$placeholder}_1"; } - $attr = "ST_GeomFromText(ST_AsText({$alias}.{$attribute}), 0)"; + // need to use srid 0 because of geometric distance + $attr = "ST_SRID({$alias}.{$attribute}, " . 0 . ")"; $geom = $this->getSpatialGeomFromText(":{$placeholder}_0", 0); return "ST_Distance({$attr}, {$geom}) {$operator} :{$placeholder}_1"; - // Without meters, use default behavior - // return "ST_Distance(ST_GeomFromText({$alias}.{$attribute}, 0, 'axis-order=lat-long')), ST_GeomFromText(:{$placeholder}_0, 0, 'axis-order=lat-long')) {$operator} :{$placeholder}_1"; } public function getSupportForIndexArray(): bool @@ -237,6 +236,17 @@ public function getSpatialSQLType(string $type, bool $required): string */ public function getSupportForSpatialAxisOrder(): bool { - return false; // Temporarily disable to test + return true; + } + + /** + * Get the spatial axis order specification string for MySQL + * MySQL with SRID 4326 expects lat-long by default, but our data is in long-lat format + * + * @return string + */ + protected function getSpatialAxisOrderSpec(): string + { + return "'axis-order=long-lat'"; } } diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index b2ef0ae79..62969097a 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -119,7 +119,15 @@ public function testSpatialTypeDocuments(): void '$id' => 'doc1', 'pointAttr' => [5.0, 5.0], 'lineAttr' => [[1.0, 2.0], [3.0, 4.0]], - 'polyAttr' => [[[0.0, 0.0], [0.0, 10.0], [10.0, 10.0], [0.0, 0.0]]], + 'polyAttr' => [ + [ + [0.0, 0.0], + [0.0, 10.0], + [10.0, 10.0], + [10.0, 0.0], + [0.0, 0.0] + ] + ], '$permissions' => [Permission::update(Role::any()), Permission::read(Role::any())] ]); $createdDoc = $database->createDocument($collectionName, $doc1); @@ -187,9 +195,17 @@ public function testSpatialTypeDocuments(): void $polyQueries = [ 'contains' => Query::contains('polyAttr', [[5.0, 5.0]]), // Point inside polygon 'notContains' => Query::notContains('polyAttr', [[15.0, 15.0]]), // Point outside polygon - 'intersects' => Query::intersects('polyAttr', [5.0, 5.0]), // Point inside polygon should intersect + 'intersects' => Query::intersects('polyAttr', [0.0, 0.0]), // Point inside polygon should intersect 'notIntersects' => Query::notIntersects('polyAttr', [15.0, 15.0]), // Point outside polygon should not intersect - 'equals' => query::equal('polyAttr', [[[[0.0, 0.0], [0.0, 10.0], [10.0, 10.0], [0.0, 0.0]]]]), // Exact same polygon + 'equals' => query::equal('polyAttr', [[ + [ + [0.0, 0.0], + [0.0, 10.0], + [10.0, 10.0], + [10.0, 0.0], + [0.0, 0.0] + ] + ]]), // Exact same polygon 'notEquals' => query::notEqual('polyAttr', [[[[20.0, 20.0], [20.0, 30.0], [30.0, 30.0], [20.0, 20.0]]]]), // Different polygon 'overlaps' => Query::overlaps('polyAttr', [[[5.0, 5.0], [5.0, 15.0], [15.0, 15.0], [15.0, 5.0], [5.0, 5.0]]]), // Overlapping polygon 'notOverlaps' => Query::notOverlaps('polyAttr', [[[20.0, 20.0], [20.0, 30.0], [30.0, 30.0], [30.0, 20.0], [20.0, 20.0]]]) // Non-overlapping polygon From f9f312758f70ebcdfb4ebeed669e034aae7fdf51 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 11 Sep 2025 17:53:49 +0530 Subject: [PATCH 62/69] added cleanups --- tests/e2e/Adapter/Scopes/SpatialTests.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 62969097a..1ccd22a48 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -234,7 +234,7 @@ public function testSpatialTypeDocuments(): void $this->assertEquals('doc1', $result[0]->getId(), sprintf('Incorrect document for distance %s on polyAttr', $queryType)); } } finally { - // $database->deleteCollection($collectionName); + $database->deleteCollection($collectionName); } } From 005d79b7f1f97b1f93279ff3be778f4407dfce2b Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 11 Sep 2025 17:59:54 +0530 Subject: [PATCH 63/69] updated tests --- tests/e2e/Adapter/Base.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 50b7b3fee..37ad7cce3 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -24,8 +24,8 @@ abstract class Base extends TestCase use IndexTests; use PermissionTests; use RelationshipTests; - use GeneralTests; use SpatialTests; + use GeneralTests; protected static string $namespace; From fb69bed8e108560da9592034f4a0755d25908a1f Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 11 Sep 2025 17:09:37 +0300 Subject: [PATCH 64/69] Update dependencies in composer.json and composer.lock, including upgrading utopia-php/mongo to version 0.6.0, brick/math to 0.14.0, and open-telemetry/api to 1.5.0. Added symfony/polyfill-php83 for compatibility with PHP 8.3 features. Updated phpunit to version 9.6.26 and utopia-php/framework to 0.33.27. --- composer.json | 2 +- composer.lock | 208 +++++++++++++++++++++++++++++++++----------------- 2 files changed, 141 insertions(+), 69 deletions(-) diff --git a/composer.json b/composer.json index 1824795de..4f3ff5b19 100755 --- a/composer.json +++ b/composer.json @@ -39,7 +39,7 @@ "utopia-php/framework": "0.33.*", "utopia-php/cache": "0.13.*", "utopia-php/pools": "0.8.*", - "utopia-php/mongo": "dev-feat-create-UUID as 0.5.3" + "utopia-php/mongo": "0.6.0" }, "require-dev": { "fakerphp/faker": "1.23.*", diff --git a/composer.lock b/composer.lock index 26bad1fd7..5b4b39a9c 100644 --- a/composer.lock +++ b/composer.lock @@ -4,29 +4,29 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "94af8bc26007c57d8f0ad1a76f38c16e", + "content-hash": "e23429f4a3f7e66afaa960e249ee7525", "packages": [ { "name": "brick/math", - "version": "0.13.1", + "version": "0.14.0", "source": { "type": "git", "url": "https://github.com/brick/math.git", - "reference": "fc7ed316430118cc7836bf45faff18d5dfc8de04" + "reference": "113a8ee2656b882d4c3164fa31aa6e12cbb7aaa2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/brick/math/zipball/fc7ed316430118cc7836bf45faff18d5dfc8de04", - "reference": "fc7ed316430118cc7836bf45faff18d5dfc8de04", + "url": "https://api.github.com/repos/brick/math/zipball/113a8ee2656b882d4c3164fa31aa6e12cbb7aaa2", + "reference": "113a8ee2656b882d4c3164fa31aa6e12cbb7aaa2", "shasum": "" }, "require": { - "php": "^8.1" + "php": "^8.2" }, "require-dev": { "php-coveralls/php-coveralls": "^2.2", - "phpunit/phpunit": "^10.1", - "vimeo/psalm": "6.8.8" + "phpstan/phpstan": "2.1.22", + "phpunit/phpunit": "^11.5" }, "type": "library", "autoload": { @@ -56,7 +56,7 @@ ], "support": { "issues": "https://github.com/brick/math/issues", - "source": "https://github.com/brick/math/tree/0.13.1" + "source": "https://github.com/brick/math/tree/0.14.0" }, "funding": [ { @@ -64,7 +64,7 @@ "type": "github" } ], - "time": "2025-03-29T13:50:30+00:00" + "time": "2025-08-29T12:40:03+00:00" }, { "name": "composer/semver", @@ -413,16 +413,16 @@ }, { "name": "open-telemetry/api", - "version": "1.4.0", + "version": "1.5.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "b3a9286f9c1c8247c83493c5b1fa475cd0cec7f7" + "reference": "7692075f486c14d8cfd37fba98a08a5667f089e5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/b3a9286f9c1c8247c83493c5b1fa475cd0cec7f7", - "reference": "b3a9286f9c1c8247c83493c5b1fa475cd0cec7f7", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/7692075f486c14d8cfd37fba98a08a5667f089e5", + "reference": "7692075f486c14d8cfd37fba98a08a5667f089e5", "shasum": "" }, "require": { @@ -479,7 +479,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-06-19T23:36:51+00:00" + "time": "2025-08-07T23:07:38+00:00" }, { "name": "open-telemetry/context", @@ -669,22 +669,22 @@ }, { "name": "open-telemetry/sdk", - "version": "1.7.0", + "version": "1.7.1", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "86287cf30fd6549444d7b8f7d8758d92e24086ac" + "reference": "52690d4b37ae4f091af773eef3c238ed2bc0aa06" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/86287cf30fd6549444d7b8f7d8758d92e24086ac", - "reference": "86287cf30fd6549444d7b8f7d8758d92e24086ac", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/52690d4b37ae4f091af773eef3c238ed2bc0aa06", + "reference": "52690d4b37ae4f091af773eef3c238ed2bc0aa06", "shasum": "" }, "require": { "ext-json": "*", "nyholm/psr7-server": "^1.1", - "open-telemetry/api": "~1.4.0", + "open-telemetry/api": "^1.4", "open-telemetry/context": "^1.0", "open-telemetry/sem-conv": "^1.0", "php": "^8.1", @@ -762,20 +762,20 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-08-06T03:07:06+00:00" + "time": "2025-09-05T07:17:06+00:00" }, { "name": "open-telemetry/sem-conv", - "version": "1.36.0", + "version": "1.37.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sem-conv.git", - "reference": "60dd18fd21d45e6f4234ecab89c14021b6e3de9a" + "reference": "8da7ec497c881e39afa6657d72586e27efbd29a1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sem-conv/zipball/60dd18fd21d45e6f4234ecab89c14021b6e3de9a", - "reference": "60dd18fd21d45e6f4234ecab89c14021b6e3de9a", + "url": "https://api.github.com/repos/opentelemetry-php/sem-conv/zipball/8da7ec497c881e39afa6657d72586e27efbd29a1", + "reference": "8da7ec497c881e39afa6657d72586e27efbd29a1", "shasum": "" }, "require": { @@ -819,7 +819,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-08-04T03:22:08+00:00" + "time": "2025-09-03T12:08:10+00:00" }, { "name": "php-http/discovery", @@ -1241,20 +1241,20 @@ }, { "name": "ramsey/uuid", - "version": "4.9.0", + "version": "4.9.1", "source": { "type": "git", "url": "https://github.com/ramsey/uuid.git", - "reference": "4e0e23cc785f0724a0e838279a9eb03f28b092a0" + "reference": "81f941f6f729b1e3ceea61d9d014f8b6c6800440" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ramsey/uuid/zipball/4e0e23cc785f0724a0e838279a9eb03f28b092a0", - "reference": "4e0e23cc785f0724a0e838279a9eb03f28b092a0", + "url": "https://api.github.com/repos/ramsey/uuid/zipball/81f941f6f729b1e3ceea61d9d014f8b6c6800440", + "reference": "81f941f6f729b1e3ceea61d9d014f8b6c6800440", "shasum": "" }, "require": { - "brick/math": "^0.8.8 || ^0.9 || ^0.10 || ^0.11 || ^0.12 || ^0.13", + "brick/math": "^0.8.8 || ^0.9 || ^0.10 || ^0.11 || ^0.12 || ^0.13 || ^0.14", "php": "^8.0", "ramsey/collection": "^1.2 || ^2.0" }, @@ -1313,9 +1313,9 @@ ], "support": { "issues": "https://github.com/ramsey/uuid/issues", - "source": "https://github.com/ramsey/uuid/tree/4.9.0" + "source": "https://github.com/ramsey/uuid/tree/4.9.1" }, - "time": "2025-06-25T14:20:11+00:00" + "time": "2025-09-04T20:59:21+00:00" }, { "name": "symfony/deprecation-contracts", @@ -1386,16 +1386,16 @@ }, { "name": "symfony/http-client", - "version": "v7.3.2", + "version": "v7.3.3", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "1c064a0c67749923483216b081066642751cc2c7" + "reference": "333b9bd7639cbdaecd25a3a48a9d2dcfaa86e019" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/1c064a0c67749923483216b081066642751cc2c7", - "reference": "1c064a0c67749923483216b081066642751cc2c7", + "url": "https://api.github.com/repos/symfony/http-client/zipball/333b9bd7639cbdaecd25a3a48a9d2dcfaa86e019", + "reference": "333b9bd7639cbdaecd25a3a48a9d2dcfaa86e019", "shasum": "" }, "require": { @@ -1403,6 +1403,7 @@ "psr/log": "^1|^2|^3", "symfony/deprecation-contracts": "^2.5|^3", "symfony/http-client-contracts": "~3.4.4|^3.5.2", + "symfony/polyfill-php83": "^1.29", "symfony/service-contracts": "^2.5|^3" }, "conflict": { @@ -1461,7 +1462,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.3.2" + "source": "https://github.com/symfony/http-client/tree/v7.3.3" }, "funding": [ { @@ -1481,7 +1482,7 @@ "type": "tidelift" } ], - "time": "2025-07-15T11:36:08+00:00" + "time": "2025-08-27T07:45:05+00:00" }, { "name": "symfony/http-client-contracts", @@ -1726,6 +1727,86 @@ ], "time": "2024-09-09T11:45:10+00:00" }, + { + "name": "symfony/polyfill-php83", + "version": "v1.33.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php83.git", + "reference": "17f6f9a6b1735c0f163024d959f700cfbc5155e5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php83/zipball/17f6f9a6b1735c0f163024d959f700cfbc5155e5", + "reference": "17f6f9a6b1735c0f163024d959f700cfbc5155e5", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php83\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 8.3+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php83/tree/v1.33.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2025-07-08T02:45:35+00:00" + }, { "name": "symfony/polyfill-php85", "version": "v1.33.0", @@ -2041,16 +2122,16 @@ }, { "name": "utopia-php/framework", - "version": "0.33.22", + "version": "0.33.27", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "c01a815cb976c9255e045fc3bcc3f5fcf477e0bc" + "reference": "d9d10a895e85c8c7675220347cc6109db9d3bd37" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/c01a815cb976c9255e045fc3bcc3f5fcf477e0bc", - "reference": "c01a815cb976c9255e045fc3bcc3f5fcf477e0bc", + "url": "https://api.github.com/repos/utopia-php/http/zipball/d9d10a895e85c8c7675220347cc6109db9d3bd37", + "reference": "d9d10a895e85c8c7675220347cc6109db9d3bd37", "shasum": "" }, "require": { @@ -2082,22 +2163,22 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.22" + "source": "https://github.com/utopia-php/http/tree/0.33.27" }, - "time": "2025-08-26T10:29:50+00:00" + "time": "2025-09-07T18:40:53+00:00" }, { "name": "utopia-php/mongo", - "version": "dev-feat-create-UUID", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "f25c14e4e3037093ad5679398da4805abb3dfec1" + "reference": "589e329a7fe4200e23ca87d65f3eb25a70ef0505" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/f25c14e4e3037093ad5679398da4805abb3dfec1", - "reference": "f25c14e4e3037093ad5679398da4805abb3dfec1", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/589e329a7fe4200e23ca87d65f3eb25a70ef0505", + "reference": "589e329a7fe4200e23ca87d65f3eb25a70ef0505", "shasum": "" }, "require": { @@ -2143,9 +2224,9 @@ ], "support": { "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/feat-create-UUID" + "source": "https://github.com/utopia-php/mongo/tree/0.6.0" }, - "time": "2025-08-18T14:00:43+00:00" + "time": "2025-09-11T13:26:21+00:00" }, { "name": "utopia-php/pools", @@ -3100,16 +3181,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.25", + "version": "9.6.26", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "049c011e01be805202d8eebedef49f769a8ec7b7" + "reference": "a0139ea157533454f611038326f3020b3051f129" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/049c011e01be805202d8eebedef49f769a8ec7b7", - "reference": "049c011e01be805202d8eebedef49f769a8ec7b7", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/a0139ea157533454f611038326f3020b3051f129", + "reference": "a0139ea157533454f611038326f3020b3051f129", "shasum": "" }, "require": { @@ -3183,7 +3264,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.25" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.26" }, "funding": [ { @@ -3207,7 +3288,7 @@ "type": "tidelift" } ], - "time": "2025-08-20T14:38:31+00:00" + "time": "2025-09-11T06:17:45+00:00" }, { "name": "rregeer/phpunit-coverage-check", @@ -4390,18 +4471,9 @@ "time": "2022-10-09T10:19:07+00:00" } ], - "aliases": [ - { - "package": "utopia-php/mongo", - "version": "dev-feat-create-UUID", - "alias": "0.5.3", - "alias_normalized": "0.5.3.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/mongo": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 1099eca19baf7dd447e7e973cb02675e52eb6da0 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 11 Sep 2025 19:53:54 +0530 Subject: [PATCH 65/69] get document fix --- src/Database/Adapter/SQL.php | 3 ++- tests/e2e/Adapter/Scopes/SpatialTests.php | 31 +++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 189f099c1..ee2163128 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -1902,7 +1902,8 @@ protected function getAttributeProjection(array $selections, string $prefix, arr foreach ($spatialAttributes as $spatialAttr) { $filteredAttr = $this->filter($spatialAttr); $quotedAttr = $this->quote($filteredAttr); - $projections[] = "ST_AsText({$this->quote($prefix)}.{$quotedAttr}) AS {$quotedAttr}"; + $axisOrder = $this->getSupportForSpatialAxisOrder() ? ', ' . $this->getSpatialAxisOrderSpec() : ''; + $projections[] = "ST_AsText({$this->quote($prefix)}.{$quotedAttr} {$axisOrder} ) AS {$quotedAttr}"; } diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 1ccd22a48..e55d41c6e 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -2626,4 +2626,35 @@ public function testSpatialIndexOnNonSpatial(): void $database->deleteCollection($collUpdateNull); } } + + public function testSpatialDocOrder(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + if (!$database->getAdapter()->getSupportForSpatialAttributes()) { + $this->markTestSkipped('Adapter does not support spatial attributes'); + } + + $collectionName = 'test_spatial_order_axis'; + // Create collection first + $database->createCollection($collectionName); + + // Create spatial attributes using createAttribute method + $this->assertEquals(true, $database->createAttribute($collectionName, 'pointAttr', Database::VAR_POINT, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); + + // Create test document + $doc1 = new Document( + [ + '$id' => 'doc1', + 'pointAttr' => [5.0, 5.5], + '$permissions' => [Permission::update(Role::any()), Permission::read(Role::any())] + ] + ); + $database->createDocument($collectionName, $doc1); + + $result = $database->getDocument($collectionName, 'doc1'); + $this->assertEquals($result->getAttribute('pointAttr')[0], 5.0); + $this->assertEquals($result->getAttribute('pointAttr')[1], 5.5); + $database->deleteCollection($collectionName); + } } From 51801aedae74792d0483af03d629405fdd7519ac Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 11 Sep 2025 20:46:04 +0530 Subject: [PATCH 66/69] added validator for wrong long/lat --- src/Database/Adapter/SQL.php | 3 +- src/Database/Validator/Spatial.php | 32 +++++++- tests/e2e/Adapter/Scopes/SpatialTests.php | 92 ++++++++++++++++++++++- tests/unit/Validator/SpatialTest.php | 28 +++++++ 4 files changed, 149 insertions(+), 6 deletions(-) diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index ee2163128..c51d525c9 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -1931,7 +1931,8 @@ protected function getAttributeProjection(array $selections, string $prefix, arr $quotedSelection = $this->quote($filteredSelection); if (in_array($selection, $spatialAttributes)) { - $projections[] = "ST_AsText({$this->quote($prefix)}.{$quotedSelection}) AS {$quotedSelection}"; + $axisOrder = $this->getSupportForSpatialAxisOrder() ? ', ' . $this->getSpatialAxisOrderSpec() : ''; + $projections[] = "ST_AsText({$this->quote($prefix)}.{$quotedSelection} {$axisOrder}) AS {$quotedSelection}"; } else { $projections[] = "{$this->quote($prefix)}.{$quotedSelection}"; } diff --git a/src/Database/Validator/Spatial.php b/src/Database/Validator/Spatial.php index 23886ffae..860f77532 100644 --- a/src/Database/Validator/Spatial.php +++ b/src/Database/Validator/Spatial.php @@ -33,7 +33,7 @@ protected function validatePoint(array $value): bool return false; } - return true; + return $this->isValidCoordinate($value[0], $value[1]); } /** @@ -49,7 +49,7 @@ protected function validateLineString(array $value): bool return false; } - foreach ($value as $point) { + foreach ($value as $pointIndex => $point) { if (!is_array($point) || count($point) !== 2) { $this->message = 'Each point in LineString must be an array of two values [x, y]'; return false; @@ -59,6 +59,11 @@ protected function validateLineString(array $value): bool $this->message = 'Each point in LineString must have numeric coordinates'; return false; } + + if (!$this->isValidCoordinate((float)$point[0], (float)$point[1])) { + $this->message = "Invalid coordinates at point #{$pointIndex}: {$this->message}"; + return false; + } } return true; @@ -77,14 +82,13 @@ protected function validatePolygon(array $value): bool return false; } - // Detect single-ring polygon: [[x, y], [x, y], ...] $isSingleRing = isset($value[0]) && is_array($value[0]) && count($value[0]) === 2 && is_numeric($value[0][0]) && is_numeric($value[0][1]); if ($isSingleRing) { - $value = [$value]; // wrap single ring + $value = [$value]; } foreach ($value as $ringIndex => $ring) { @@ -108,6 +112,11 @@ protected function validatePolygon(array $value): bool $this->message = "Coordinates of point #{$pointIndex} in ring #{$ringIndex} must be numeric"; return false; } + + if (!$this->isValidCoordinate((float)$point[0], (float)$point[1])) { + $this->message = "Invalid coordinates at point #{$pointIndex} in ring #{$ringIndex}: {$this->message}"; + return false; + } } // Check that the ring is closed (first point == last point) @@ -182,4 +191,19 @@ public function isValid($value): bool $this->message = 'Spatial value must be array or WKT string'; return false; } + + private function isValidCoordinate(int|float $x, int|float $y): bool + { + if ($x < -180 || $x > 180) { + $this->message = "Longitude (x) must be between -180 and 180, got {$x}"; + return false; + } + + if ($y < -90 || $y > 90) { + $this->message = "Latitude (y) must be between -90 and 90, got {$y}"; + return false; + } + + return true; + } } diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index e55d41c6e..4874f2ee6 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -12,7 +12,6 @@ use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; -use Utopia\Database\Validator\Index; trait SpatialTests { @@ -2657,4 +2656,95 @@ public function testSpatialDocOrder(): void $this->assertEquals($result->getAttribute('pointAttr')[1], 5.5); $database->deleteCollection($collectionName); } + + public function testInvalidCoordinateDocuments(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + if (!$database->getAdapter()->getSupportForSpatialAttributes()) { + $this->markTestSkipped('Adapter does not support spatial attributes'); + } + + $collectionName = 'test_invalid_coord_'; + try { + $database->createCollection($collectionName); + + $database->createAttribute($collectionName, 'pointAttr', Database::VAR_POINT, 0, true); + $database->createAttribute($collectionName, 'lineAttr', Database::VAR_LINESTRING, 0, true); + $database->createAttribute($collectionName, 'polyAttr', Database::VAR_POLYGON, 0, true); + + $invalidDocs = [ + // Invalid POINT (longitude > 180) + [ + '$id' => 'invalidDoc1', + 'pointAttr' => [200.0, 20.0], + 'lineAttr' => [[1.0, 2.0], [3.0, 4.0]], + 'polyAttr' => [ + [ + [0.0, 0.0], + [0.0, 10.0], + [10.0, 10.0], + [10.0, 0.0], + [0.0, 0.0] + ] + ] + ], + // Invalid POINT (latitude < -90) + [ + '$id' => 'invalidDoc2', + 'pointAttr' => [50.0, -100.0], + 'lineAttr' => [[1.0, 2.0], [3.0, 4.0]], + 'polyAttr' => [ + [ + [0.0, 0.0], + [0.0, 10.0], + [10.0, 10.0], + [10.0, 0.0], + [0.0, 0.0] + ] + ] + ], + // Invalid LINESTRING (point outside valid range) + [ + '$id' => 'invalidDoc3', + 'pointAttr' => [50.0, 20.0], + 'lineAttr' => [[1.0, 2.0], [300.0, 4.0]], // invalid longitude in line + 'polyAttr' => [ + [ + [0.0, 0.0], + [0.0, 10.0], + [10.0, 10.0], + [10.0, 0.0], + [0.0, 0.0] + ] + ] + ], + // Invalid POLYGON (point outside valid range) + [ + '$id' => 'invalidDoc4', + 'pointAttr' => [50.0, 20.0], + 'lineAttr' => [[1.0, 2.0], [3.0, 4.0]], + 'polyAttr' => [ + [ + [0.0, 0.0], + [0.0, 10.0], + [190.0, 10.0], // invalid longitude + [10.0, 0.0], + [0.0, 0.0] + ] + ] + ], + ]; + foreach ($invalidDocs as $docData) { + $this->expectException(StructureException::class); + $docData['$permissions'] = [Permission::update(Role::any()), Permission::read(Role::any())]; + $doc = new Document($docData); + $database->createDocument($collectionName, $doc); + } + + + } finally { + $database->deleteCollection($collectionName); + } + } } diff --git a/tests/unit/Validator/SpatialTest.php b/tests/unit/Validator/SpatialTest.php index 8fe83a870..e8df4d3d1 100644 --- a/tests/unit/Validator/SpatialTest.php +++ b/tests/unit/Validator/SpatialTest.php @@ -81,4 +81,32 @@ public function testWKTStrings(): void $this->assertFalse(Spatial::isWKTString('CIRCLE(0 0,1)')); $this->assertFalse(Spatial::isWKTString('POINT1(1 2)')); } + + public function testInvalidCoordinate(): void + { + // Point with invalid longitude + $validator = new Spatial(Database::VAR_POINT); + $this->assertFalse($validator->isValid([200, 10])); // longitude > 180 + $this->assertStringContainsString('Longitude', $validator->getDescription()); + + // Point with invalid latitude + $validator = new Spatial(Database::VAR_POINT); + $this->assertFalse($validator->isValid([10, -100])); // latitude < -90 + $this->assertStringContainsString('Latitude', $validator->getDescription()); + + // LineString with invalid coordinates + $validator = new Spatial(Database::VAR_LINESTRING); + $this->assertFalse($validator->isValid([ + [0, 0], + [181, 45] // invalid longitude + ])); + $this->assertStringContainsString('Invalid coordinates', $validator->getDescription()); + + // Polygon with invalid coordinates + $validator = new Spatial(Database::VAR_POLYGON); + $this->assertFalse($validator->isValid([ + [[0, 0], [1, 1], [190, 5], [0, 0]] // invalid longitude in ring + ])); + $this->assertStringContainsString('Invalid coordinates', $validator->getDescription()); + } } From 753800f9eb16bfd62b9647a0c54f39675dde728b Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 11 Sep 2025 20:51:57 +0530 Subject: [PATCH 67/69] linting --- src/Database/Validator/Spatial.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Validator/Spatial.php b/src/Database/Validator/Spatial.php index 860f77532..912f05b2b 100644 --- a/src/Database/Validator/Spatial.php +++ b/src/Database/Validator/Spatial.php @@ -33,7 +33,7 @@ protected function validatePoint(array $value): bool return false; } - return $this->isValidCoordinate($value[0], $value[1]); + return $this->isValidCoordinate((float)$value[0], (float) $value[1]); } /** From 23da2a8668ece0f292c53e3ebd7beb3bf3c971ff Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 14 Sep 2025 16:15:04 +0300 Subject: [PATCH 68/69] Enhance Mongo adapter: added 'required' parameter to updateAttribute method and implemented support checks for spatial axis order and distance calculations between multi-dimension geometries. --- src/Database/Adapter/Mongo.php | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index ad47a8185..9cb05a59c 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -1416,7 +1416,7 @@ public function deleteDocuments(string $collection, array $sequences, array $per * * @return bool */ - public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false, ?string $newKey = null): bool + public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false, ?string $newKey = null, bool $required = false): bool { if (!empty($newKey) && $newKey !== $id) { return $this->renameAttribute($collection, $id, $newKey); @@ -2450,6 +2450,27 @@ public function getSupportForSpatialIndexOrder(): bool } + /** + * Does the adapter support spatial axis order specification? + * + * @return bool + */ + public function getSupportForSpatialAxisOrder(): bool + { + return false; + } + +/** + * Does the adapter support calculating distance(in meters) between multidimension geometry(line, polygon,etc)? + * + * @return bool + */ + public function getSupportForDistanceBetweenMultiDimensionGeometryInMeters(): bool + { + return false; + } + + /** * Flattens the array. * From 15c04b9acf4eb39ad33fdd29073c235c80970959 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 14 Sep 2025 16:16:53 +0300 Subject: [PATCH 69/69] linter --- src/Database/Adapter/Mongo.php | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 9cb05a59c..dcb920144 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -2450,21 +2450,21 @@ public function getSupportForSpatialIndexOrder(): bool } - /** - * Does the adapter support spatial axis order specification? - * - * @return bool - */ + /** + * Does the adapter support spatial axis order specification? + * + * @return bool + */ public function getSupportForSpatialAxisOrder(): bool { return false; } -/** - * Does the adapter support calculating distance(in meters) between multidimension geometry(line, polygon,etc)? - * - * @return bool - */ + /** + * Does the adapter support calculating distance(in meters) between multidimension geometry(line, polygon,etc)? + * + * @return bool + */ public function getSupportForDistanceBetweenMultiDimensionGeometryInMeters(): bool { return false;