From d0c9823de0d8ae11ee5856f1477980f70a3cf3eb Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sat, 15 Feb 2025 00:37:55 +0100 Subject: [PATCH 1/6] type stability and alloc reduction --- src/SortTileRecursiveTree.jl | 51 ++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/src/SortTileRecursiveTree.jl b/src/SortTileRecursiveTree.jl index 4efecd2..4ae9231 100644 --- a/src/SortTileRecursiveTree.jl +++ b/src/SortTileRecursiveTree.jl @@ -45,21 +45,27 @@ end function leafnodes(geoms; nodecapacity=10) - extents_indices = [(GI.extent(geoms[i]), i) for i in eachindex(geoms)] - perm = sortperm(extents_indices; by=(v -> ((v[1][1][1] + v[1][1][2]) / 2))) # [extent/index][dim][min/max] sort by x - sorted_extents = extents_indices[perm] - r = length(sorted_extents) + ext1 = GI.extent(first(geoms)) + extents_indices = Tuple{typeof(ext1),Int}[(GI.extent(geoms[i]), i) for i in eachindex(geoms)] + # Use the same scratch space for all sorts + scratch = similar(extents_indices) + sort!(extents_indices; by=v -> (v[1][1][1] + v[1][1][2]) / 2, scratch) # [extent/index][dim][min/max] sort by x + r = length(extents_indices) P = ceil(Int, r / nodecapacity) S = ceil(Int, sqrt(P)) - x_splits = Iterators.partition(sorted_extents, S * nodecapacity) + x_splits = Iterators.partition(extents_indices, S * nodecapacity) nodes = STRLeafNode{Vector{typeof(extents_indices[1][1])}}[] for x_split in x_splits - perm = sortperm(x_split; by=(v -> ((v[1][2][1] + v[1][2][2]) / 2))) # [extent/index][dim][min/max] sort by y - sorted_split = x_split[perm] - y_splits = Iterators.partition(sorted_split, nodecapacity) + sort!(x_split; + by=v -> (v[1][2][1] + v[1][2][2]) / 2, + scratch=resize!(scratch, length(x_split)), + ) # [extent/index][dim][min/max] sort by y + y_splits = Iterators.partition(x_split, nodecapacity) for y_split in y_splits - push!(nodes, STRLeafNode(getindex.(y_split,1), getindex.(y_split,2))) + exts = first.(y_split)::Vector{typeof(ext1)} + inds = last.(y_split)::Vector{Int} + push!(nodes, STRLeafNode(exts, inds)) end end return nodes @@ -68,23 +74,28 @@ end # a bit of duplication... function parentnodes(nodes; nodecapacity=10) - extents_indices = [(GI.extent(node), node) for node in nodes] - perm = sortperm(extents_indices; by=(v -> ((v[1][1][1] + v[1][1][2]) / 2))) # [extent/node][dim][min/max] sort by x - sorted_extents = extents_indices[perm] - r = length(sorted_extents) + n1 = first(nodes) + ext1 = GI.extent(n1) + extent_indices = Tuple{typeof(ext1),typeof(n1)}[(GI.extent(node), node) for node in nodes] + sort!(extent_indices; by=v -> (v[1][1][1] + v[1][1][2]) / 2) # [extent/node][dim][min/max] sort by x + r = length(extent_indices) P = ceil(Int, r / nodecapacity) S = ceil(Int, sqrt(P)) - x_splits = Iterators.partition(sorted_extents, S * nodecapacity) + x_splits = Iterators.partition(extent_indices, S * nodecapacity) - T = typeof(extents_indices[1][1]) - N = Vector{typeof(extents_indices[1][2])} + T = typeof(extent_indices[1][1]) + N = Vector{typeof(extent_indices[1][2])} nodes = STRNode{T, N}[] for x_split in x_splits - perm = sortperm(x_split; by=(v -> ((v[1][2][1] + v[1][2][2]) / 2))) # [extent/index][dim][min/max] sort by y - sorted_split = x_split[perm] - y_splits = Iterators.partition(sorted_split, nodecapacity) + sort!(x_split; by=v -> (v[1][2][1] + v[1][2][2]) / 2) # [extent/index][dim][min/max] sort by y + y_splits = Iterators.partition(x_split, nodecapacity) for y_split in y_splits - push!(nodes, STRNode(foldl(Extents.union, getindex.(y_split,1)), getindex.(y_split,2))) + # Alloc free union over the extents + ext = foldl(y_split; init=y_split[1][1]) do u, (ext, _) + Extents.union(u, ext) + end + nodes = last.(y_split)::Vector{typeof(n1)} + push!(nodes, STRNode(ext, nodes)) end end return nodes From 639d5b0bac27db5d2b5fb9d15c6d906ef30dafff Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sat, 15 Feb 2025 00:46:36 +0100 Subject: [PATCH 2/6] more scratch --- src/SortTileRecursiveTree.jl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/SortTileRecursiveTree.jl b/src/SortTileRecursiveTree.jl index 4ae9231..9836f24 100644 --- a/src/SortTileRecursiveTree.jl +++ b/src/SortTileRecursiveTree.jl @@ -58,9 +58,9 @@ function leafnodes(geoms; nodecapacity=10) nodes = STRLeafNode{Vector{typeof(extents_indices[1][1])}}[] for x_split in x_splits sort!(x_split; - by=v -> (v[1][2][1] + v[1][2][2]) / 2, + by=v -> (v[1][2][1] + v[1][2][2]) / 2, # [extent/index][dim][min/max] sort by y scratch=resize!(scratch, length(x_split)), - ) # [extent/index][dim][min/max] sort by y + ) y_splits = Iterators.partition(x_split, nodecapacity) for y_split in y_splits exts = first.(y_split)::Vector{typeof(ext1)} @@ -77,7 +77,8 @@ function parentnodes(nodes; nodecapacity=10) n1 = first(nodes) ext1 = GI.extent(n1) extent_indices = Tuple{typeof(ext1),typeof(n1)}[(GI.extent(node), node) for node in nodes] - sort!(extent_indices; by=v -> (v[1][1][1] + v[1][1][2]) / 2) # [extent/node][dim][min/max] sort by x + scratch = similar(extents_indices) + sort!(extent_indices; by=v -> (v[1][1][1] + v[1][1][2]) / 2, scratch) # [extent/node][dim][min/max] sort by x r = length(extent_indices) P = ceil(Int, r / nodecapacity) S = ceil(Int, sqrt(P)) @@ -87,7 +88,10 @@ function parentnodes(nodes; nodecapacity=10) N = Vector{typeof(extent_indices[1][2])} nodes = STRNode{T, N}[] for x_split in x_splits - sort!(x_split; by=v -> (v[1][2][1] + v[1][2][2]) / 2) # [extent/index][dim][min/max] sort by y + sort!(x_split; + by=v -> (v[1][2][1] + v[1][2][2]) / 2 # [extent/index][dim][min/max] sort by y + scratch=resize!(scratch, length(x_split)), + ) y_splits = Iterators.partition(x_split, nodecapacity) for y_split in y_splits # Alloc free union over the extents From e9d90961b916669a172823cbcd7e86ecdb61fab5 Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sat, 15 Feb 2025 00:53:15 +0100 Subject: [PATCH 3/6] bugfix --- src/SortTileRecursiveTree.jl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/SortTileRecursiveTree.jl b/src/SortTileRecursiveTree.jl index 9836f24..a4ad261 100644 --- a/src/SortTileRecursiveTree.jl +++ b/src/SortTileRecursiveTree.jl @@ -76,20 +76,20 @@ end function parentnodes(nodes; nodecapacity=10) n1 = first(nodes) ext1 = GI.extent(n1) - extent_indices = Tuple{typeof(ext1),typeof(n1)}[(GI.extent(node), node) for node in nodes] + extents_indices = Tuple{typeof(ext1),typeof(n1)}[(GI.extent(node), node) for node in nodes] scratch = similar(extents_indices) - sort!(extent_indices; by=v -> (v[1][1][1] + v[1][1][2]) / 2, scratch) # [extent/node][dim][min/max] sort by x - r = length(extent_indices) + sort!(extents_indices; by=v -> (v[1][1][1] + v[1][1][2]) / 2, scratch) # [extent/node][dim][min/max] sort by x + r = length(extents_indices) P = ceil(Int, r / nodecapacity) S = ceil(Int, sqrt(P)) - x_splits = Iterators.partition(extent_indices, S * nodecapacity) + x_splits = Iterators.partition(extents_indices, S * nodecapacity) - T = typeof(extent_indices[1][1]) - N = Vector{typeof(extent_indices[1][2])} + T = typeof(extents_indices[1][1]) + N = Vector{typeof(extents_indices[1][2])} nodes = STRNode{T, N}[] for x_split in x_splits sort!(x_split; - by=v -> (v[1][2][1] + v[1][2][2]) / 2 # [extent/index][dim][min/max] sort by y + by=v -> (v[1][2][1] + v[1][2][2]) / 2, # [extent/index][dim][min/max] sort by y scratch=resize!(scratch, length(x_split)), ) y_splits = Iterators.partition(x_split, nodecapacity) @@ -98,8 +98,8 @@ function parentnodes(nodes; nodecapacity=10) ext = foldl(y_split; init=y_split[1][1]) do u, (ext, _) Extents.union(u, ext) end - nodes = last.(y_split)::Vector{typeof(n1)} - push!(nodes, STRNode(ext, nodes)) + ns = last.(y_split)::Vector{typeof(n1)} + push!(nodes, STRNode(ext, ns)) end end return nodes From db5fec9b999e9e7893ec3708c0d168505714b499 Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sat, 15 Feb 2025 01:26:06 +0100 Subject: [PATCH 4/6] tweak y_splitnodes --- src/SortTileRecursiveTree.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/SortTileRecursiveTree.jl b/src/SortTileRecursiveTree.jl index a4ad261..f1fd17c 100644 --- a/src/SortTileRecursiveTree.jl +++ b/src/SortTileRecursiveTree.jl @@ -86,7 +86,7 @@ function parentnodes(nodes; nodecapacity=10) T = typeof(extents_indices[1][1]) N = Vector{typeof(extents_indices[1][2])} - nodes = STRNode{T, N}[] + outnodes = STRNode{T, N}[] for x_split in x_splits sort!(x_split; by=v -> (v[1][2][1] + v[1][2][2]) / 2, # [extent/index][dim][min/max] sort by y @@ -98,11 +98,11 @@ function parentnodes(nodes; nodecapacity=10) ext = foldl(y_split; init=y_split[1][1]) do u, (ext, _) Extents.union(u, ext) end - ns = last.(y_split)::Vector{typeof(n1)} - push!(nodes, STRNode(ext, ns)) + y_splitnodes = last.(y_split)::Vector{eltype(nodes)} + push!(outnodes, STRNode(ext, y_splitnodes)) end end - return nodes + return outnodes end From 21e2334a54f2a8efe8cb6368359f0933941124df Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sat, 15 Feb 2025 15:45:05 +0100 Subject: [PATCH 5/6] drop 1.6 support and testing --- .github/workflows/CI.yml | 2 +- Project.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index aa63477..bc59ad2 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -22,7 +22,7 @@ jobs: matrix: version: - '1.10' - - '1.6' + - '1.11' - 'nightly' os: - ubuntu-latest diff --git a/Project.toml b/Project.toml index c5a3512..477a770 100644 --- a/Project.toml +++ b/Project.toml @@ -8,7 +8,7 @@ Extents = "411431e0-e8b7-467b-b5e0-f676ba4f2910" GeoInterface = "cf35fbd7-0cd7-5166-be24-54bfbe79505f" [compat] -julia = "1.6" +julia = "1.10" Extents = "0.1.0" GeoInterface = "1" From a86dd7a267e87a5833a5f84d9392cc60f7b8b912 Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sat, 15 Feb 2025 16:17:32 +0100 Subject: [PATCH 6/6] Update .github/workflows/CI.yml Co-authored-by: Anshul Singhvi --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index bc59ad2..dc2af87 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -22,7 +22,7 @@ jobs: matrix: version: - '1.10' - - '1.11' + - '1' - 'nightly' os: - ubuntu-latest