feat: added updated_at tracking for tiles and incremental reconstruction

- added migration 004_add_updated_at_to_tiles.rb with updated_at field and trigger for automatic updates
- updated tiles table schema in database_manager.rb with updated_at field and idx_tiles_zoom_updated index
- update updated_at on tile insert/update in config.ru (fetch_with_lock) and background_tile_loader.rb (fetch_tile)
- implemented incremental processing in tile_reconstructor.rb based on updated_at for gap filling optimization
- added save_last_run_timestamp and get_last_run_timestamp methods to track last reconstruction run
- filter tiles by updated_at > last_run_time or generated = -5 to process only changed tiles
- update updated_at when generating parent tiles and marking grandparent for regeneration

Author: Efreh

docker/ruby/Dockerfile

@@ -62,6 +62,6 @@ ENV RUBY_YJIT_ENABLE=1 \
 WORKDIR /app
 
 # --start_period=5s (Unknown flag: start_period)
-HEALTHCHECK --interval=15s --timeout=2s --retries=3 CMD curl --fail http://127.0.0.1:$PORT/healthcheck || exit 1
+# HEALTHCHECK --interval=15s --timeout=2s --retries=3 CMD curl --fail http://127.0.0.1:$PORT/healthcheck || exit 1
 CMD bundle exec rackup -o 0.0.0.0 -p $PORT -s falcon
 # docker-compose build --progress plain

src/background_tile_loader.rb

@@ -279,12 +279,16 @@ def fetch_tile(x, y, z)
 
       @route[:db][:tiles].insert_conflict(
         target: [:zoom_level, :tile_column, :tile_row],
-        update: { tile_data: Sequel[:excluded][:tile_data] }
+        update: {
+          tile_data: Sequel[:excluded][:tile_data],
+          updated_at: Sequel.lit("datetime('now', 'utc')")
+        }
       ).insert(
         zoom_level: z,
         tile_column: x,
         tile_row: tms_y(z, y),
-        tile_data: Sequel.blob(data)
+        tile_data: Sequel.blob(data),
+        updated_at: Sequel.lit("datetime('now', 'utc')")
       )
 
       if @route.dig(:gap_filling, :enabled)

src/config.ru

@@ -306,13 +306,13 @@ helpers do
       end
 
       route[:db][:tiles].insert_conflict(target: [:zoom_level, :tile_column, :tile_row],
-                                         update: { tile_data: Sequel[:excluded][:tile_data] })
+                                         update: {
+                                           tile_data: Sequel[:excluded][:tile_data],
+                                           updated_at: Sequel.lit("datetime('now', 'utc')")
+                                         })
                         .insert(zoom_level: z, tile_column: x, tile_row: tms,
-                                tile_data: Sequel.blob(result[:data]))
-
-#      if route.dig(:gap_filling, :enabled) && z >= route.dig(:gap_filling, :source_real_minzoom)
-#        route[:reconstructor]&.mark_parent_for_new_child(route[:db], z, x, tms, route[:minzoom])
-#      end
+                                tile_data: Sequel.blob(result[:data]),
+                                updated_at: Sequel.lit("datetime('now', 'utc')"))
 
       result[:data]
     end

src/database_manager.rb

@@ -98,10 +98,12 @@ def create_tables(db)
       Integer :tile_row,    null:false
       File    :tile_data,   null:false
       Integer :generated,   default: 0  # 0=original, 1=generated, 2=needs_regeneration
+      DateTime :updated_at, default: Sequel.lit("datetime('now', 'utc')")
       unique [:zoom_level,:tile_column,:tile_row], name: :tile_index
       index :zoom_level, name: :idx_tiles_zoom_level
       index [:zoom_level, Sequel.function(:length, :tile_data)], name: :idx_tiles_zoom_size
       index [:zoom_level, :generated], name: :idx_tiles_zoom_generated
+      index [:zoom_level, :updated_at], name: :idx_tiles_zoom_updated
     }
     db.create_table?(:misses){ 
       Integer :zoom_level, null: false

src/migrations/004_add_updated_at_to_tiles.rb

@@ -0,0 +1,27 @@
+Sequel.migration do
+  up do
+    next unless table_exists?(:tiles)
+
+    columns = schema(:tiles).map(&:first)
+    unless columns.include?(:updated_at)
+      alter_table(:tiles) do
+        add_column :updated_at, DateTime
+      end
+    end
+
+    run "UPDATE tiles SET updated_at = datetime('now', 'utc') WHERE updated_at IS NULL"
+
+    begin
+      add_index :tiles, [:zoom_level, :updated_at], name: :idx_tiles_zoom_updated
+    rescue Sequel::DatabaseError => e
+      raise unless e.message.include?('already exists')
+    end
+  end
+
+  down do
+    next unless table_exists?(:tiles)
+
+    raise Sequel::Error, 'Irreversible migration: dropping updated_at column would require table recreation'
+  end
+end
+

src/tile_reconstructor.rb

@@ -78,6 +78,28 @@ def status
 
   private
 
+  def save_last_run_timestamp(db)
+    db[:metadata].insert_conflict(
+      target: :name,
+      update: { value: Sequel[:excluded][:value] }
+    ).insert(
+      name: 'reconstruction_last_run',
+      value: Time.now.utc.iso8601
+    )
+  rescue => e
+    LOGGER.warn("TileReconstructor: failed to save last run timestamp: #{e.message}")
+  end
+
+  def get_last_run_timestamp(db)
+    timestamp_str = db[:metadata].where(name: 'reconstruction_last_run').get(:value)
+    return nil unless timestamp_str
+
+    Time.parse(timestamp_str)
+  rescue => e
+    LOGGER.warn("TileReconstructor: failed to get last run timestamp: #{e.message}")
+    nil
+  end
+
   # Parses schedule time from config, returns {hour:, minute:} or nil
   def parse_schedule_time
     time_str = @route.dig(:gap_filling, :schedule, :time) || @route.dig(:gap_filling, :schedule, 'time')
@@ -111,34 +133,38 @@ def run_reconstruction
 
     downsample_opts = build_downsample_opts(@route)
 
-    LOGGER.info("TileReconstructor: starting gap filling for #{@source_name} from zoom #{start_zoom} to #{minzoom}")
+    last_run_time = get_last_run_timestamp(db)
+    LOGGER.info("TileReconstructor: starting #{last_run_time ? "incremental (last run: #{last_run_time.iso8601})" : "full (first run)"} gap filling for #{@source_name} from zoom #{start_zoom} to #{minzoom}")
 
     start_zoom.downto(minzoom) do |z|
       break unless @running
 
       begin
-        process_zoom_level(z, db, downsample_opts, minzoom, maxzoom)
+        process_zoom_level(z, db, downsample_opts, minzoom, maxzoom, last_run_time)
       rescue => e
         LOGGER.error("TileReconstructor: failed to process zoom #{z} for #{@source_name}: #{e.message}")
         LOGGER.debug("TileReconstructor: backtrace: #{e.backtrace.join("\n")}")
       end
     end
 
+    save_last_run_timestamp(db)
+
     LOGGER.info("TileReconstructor: gap filling completed for #{@source_name}")
   end
 
   otl_def :run_reconstruction
 
-  def process_zoom_level(z, db, downsample_opts, minzoom, maxzoom)
+  def process_zoom_level(z, db, downsample_opts, minzoom, maxzoom, last_run_time = nil)
     parent_z = z - 1
     return if parent_z < minzoom
 
     LOGGER.info("TileReconstructor: processing zoom #{z} -> #{parent_z}")
 
-    all_tiles_z = load_tiles_for_zoom(z, db)
+    all_tiles_z = load_tiles_for_zoom(z, db, last_run_time)
     return if all_tiles_z.empty?
 
-    LOGGER.info("TileReconstructor: loaded #{all_tiles_z.size} tiles for zoom #{z}")
+    log_msg = last_run_time ? "loaded #{all_tiles_z.size} tiles for zoom #{z} (filtered by timestamp)" : "loaded #{all_tiles_z.size} tiles for zoom #{z}"
+    LOGGER.info("TileReconstructor: #{log_msg}")
 
     parent_coords_set = calculate_parent_coords(all_tiles_z)
     LOGGER.info("TileReconstructor: calculated #{parent_coords_set.size} unique parents for zoom #{parent_z}")
@@ -182,11 +208,19 @@ def process_parent(parent_coords, z, parent_z, db, downsample_opts, minzoom)
     generate_and_save_parent(px, py, parent_z, children_data_array, used_count, downsample_opts, db, grandparent_tile, parent_tile, parent_validation)
   end
 
-  # Loads all tiles for a zoom level (coordinates only, no blob)
-  def load_tiles_for_zoom(z, db)
-    db[:tiles].where(zoom_level: z)
-              .select(:tile_column, :tile_row, :generated)
-              .to_a
+  def load_tiles_for_zoom(z, db, last_run_time = nil)
+    query = db[:tiles].where(zoom_level: z)
+
+    if last_run_time
+      last_run_utc = last_run_time.utc
+      conditions = [
+        Sequel[:updated_at] > last_run_utc,
+        Sequel[:generated] => -5
+      ]
+      query = query.where { Sequel.|(*conditions) }
+    end
+
+    query.select(:tile_column, :tile_row, :generated).to_a
   end
 
   def calculate_parent_coords(all_tiles_z)
@@ -323,14 +357,16 @@ def generate_and_save_parent(px, py, parent_z, children_data_array, used_count,
         target: [:zoom_level, :tile_column, :tile_row],
         update: {
           tile_data: Sequel[:excluded][:tile_data],
-          generated: Sequel[:excluded][:generated]
+          generated: Sequel[:excluded][:generated],
+          updated_at: Sequel.lit("datetime('now', 'utc')")
         }
       ).insert(
         zoom_level: parent_z,
         tile_column: px,
         tile_row: py,
         tile_data: Sequel.blob(new_data),
-        generated: used_count
+        generated: used_count,
+        updated_at: Sequel.lit("datetime('now', 'utc')")
       )
 
       mark_grandparent_for_regeneration(grandparent_tile, db) if grandparent_tile && grandparent_tile[:generated] != 0
@@ -347,7 +383,7 @@ def mark_grandparent_for_regeneration(grandparent_tile, db)
       zoom_level: grandparent_tile[:zoom_level],
       tile_column: grandparent_tile[:tile_column],
       tile_row: grandparent_tile[:tile_row]
-    ).update(generated: -5)
+    ).update(generated: -5, updated_at: Sequel.lit("datetime('now', 'utc')"))
   end
 
   # Validates tile and returns its status

src/views/database.slim

@@ -100,7 +100,7 @@ html
         .content
           - page, per_page = (params[:page] || 1).to_i, 50
           - offset = (page - 1) * per_page
-          - table_data = { 'tiles' => { title: 'Tiles Table', cols: ['Zoom Level', 'Tile Column', 'Tile Row', 'Data Size', 'Generated'], data: @route[:db][:tiles].limit(per_page).offset(offset), format: ->(r) { [r[:zoom_level], r[:tile_column], r[:tile_row], "#{(r[:tile_data].bytesize / 1024.0).round(1)} KB", r[:generated]] } }, 'misses' => { title: 'Misses Table', cols: ['Zoom Level', 'Tile Column', 'Tile Row', 'Timestamp', 'Status', 'Reason', 'Details', 'Response Size'], data: @route[:db][:misses].order(Sequel.desc(:ts)).limit(per_page).offset(offset), format: ->(r) { [r[:zoom_level], r[:tile_column], r[:tile_row], Time.at(r[:ts]).strftime("%Y-%m-%d %H:%M:%S"), r[:status] || 'N/A', r[:reason] || 'N/A', r[:details] || 'N/A', r[:response_body] ? "#{(r[:response_body].bytesize / 1024.0).round(1)} KB" : 'N/A'] } }, 'metadata' => { title: 'Metadata Table', cols: ['Name', 'Value'], data: @route[:db][:metadata].limit(per_page).offset(offset), format: ->(r) { [r[:name], r[:value]] } }, 'tile_scan_progress' => { title: 'Autoscan Progress Table', cols: ['Source', 'Zoom Level', 'Last X', 'Last Y', 'Tiles Today', 'Last Scan Date', 'Status'], data: (@route[:db].table_exists?(:tile_scan_progress) ? @route[:db][:tile_scan_progress].order(:zoom_level).limit(per_page).offset(offset) : []), format: ->(r) { [r[:source], r[:zoom_level], r[:last_x], r[:last_y], r[:tiles_today], r[:last_scan_date], r[:status]] } } }
+          - table_data = { 'tiles' => { title: 'Tiles Table', cols: ['Zoom Level', 'Tile Column', 'Tile Row', 'Data Size', 'Generated', 'Updated At'], data: @route[:db][:tiles].limit(per_page).offset(offset), format: ->(r) { [r[:zoom_level], r[:tile_column], r[:tile_row], "#{(r[:tile_data].bytesize / 1024.0).round(1)} KB", r[:generated], r[:updated_at] ? Time.parse(r[:updated_at].to_s).strftime("%Y-%m-%d %H:%M:%S") : 'N/A'] } }, 'misses' => { title: 'Misses Table', cols: ['Zoom Level', 'Tile Column', 'Tile Row', 'Timestamp', 'Status', 'Reason', 'Details', 'Response Size'], data: @route[:db][:misses].order(Sequel.desc(:ts)).limit(per_page).offset(offset), format: ->(r) { [r[:zoom_level], r[:tile_column], r[:tile_row], Time.at(r[:ts]).strftime("%Y-%m-%d %H:%M:%S"), r[:status] || 'N/A', r[:reason] || 'N/A', r[:details] || 'N/A', r[:response_body] ? "#{(r[:response_body].bytesize / 1024.0).round(1)} KB" : 'N/A'] } }, 'metadata' => { title: 'Metadata Table', cols: ['Name', 'Value'], data: @route[:db][:metadata].limit(per_page).offset(offset), format: ->(r) { [r[:name], r[:value]] } }, 'tile_scan_progress' => { title: 'Autoscan Progress Table', cols: ['Source', 'Zoom Level', 'Last X', 'Last Y', 'Tiles Today', 'Last Scan Date', 'Status'], data: (@route[:db].table_exists?(:tile_scan_progress) ? @route[:db][:tile_scan_progress].order(:zoom_level).limit(per_page).offset(offset) : []), format: ->(r) { [r[:source], r[:zoom_level], r[:last_x], r[:last_y], r[:tiles_today], r[:last_scan_date], r[:status]] } } }
           - current_data = table_data[current_table]
           - total_count = current_table == 'tile_scan_progress' && !@route[:db].table_exists?(:tile_scan_progress) ? 0 : @route[:db][current_table.to_sym].count
           - total_pages = (total_count.to_f / per_page).ceil