Merge pull request #2471 from joto/timestamp

Add timestamp(tz) column types

Author: lonvia

src/flex-table-column.cpp

@@ -43,6 +43,8 @@ std::vector<column_type_lookup> const COLUMN_TYPES = {
      {"int8", table_column_type::int8},
      {"bigint", table_column_type::int8},
      {"real", table_column_type::real},
+     {"timestamp", table_column_type::timestamp},
+     {"timestamptz", table_column_type::timestamptz},
      {"hstore", table_column_type::hstore},
      {"json", table_column_type::json},
      {"jsonb", table_column_type::jsonb},
@@ -142,6 +144,10 @@ std::string flex_table_column_t::sql_type_name() const
         return "int8";
     case table_column_type::real:
         return "real";
+    case table_column_type::timestamp:
+        return "timestamp";
+    case table_column_type::timestamptz:
+        return "timestamptz";
     case table_column_type::hstore:
         return "hstore";
     case table_column_type::json:

src/flex-table-column.hpp

@@ -32,6 +32,9 @@ enum class table_column_type : uint8_t
 
     real,
 
+    timestamp,
+    timestamptz,
+
     hstore,
     json,
     jsonb,

src/flex-write.cpp

@@ -14,6 +14,8 @@
 #include "lua-utils.hpp"
 #include "wkb.hpp"
 
+#include <osmium/osm/timestamp.hpp>
+
 #include <algorithm>
 #include <cmath>
 #include <cstdint>
@@ -358,6 +360,26 @@ void flex_write_column(lua_State *lua_state,
             throw fmt_error("Invalid type '{}' for real column.",
                             lua_typename(lua_state, ltype));
         }
+    } else if (column.type() == table_column_type::timestamp) {
+        if (ltype == LUA_TNUMBER) {
+            auto const ts = osmium::Timestamp{lua_tointeger(lua_state, -1)};
+            copy_mgr->add_column(ts.to_iso());
+        } else if (ltype == LUA_TSTRING) {
+            copy_mgr->add_column(lua_tolstring(lua_state, -1, nullptr));
+        } else {
+            throw fmt_error("Invalid type '{}' for timestamp column.",
+                            lua_typename(lua_state, ltype));
+        }
+    } else if (column.type() == table_column_type::timestamptz) {
+        if (ltype == LUA_TNUMBER) {
+            auto const ts = osmium::Timestamp{lua_tointeger(lua_state, -1)};
+            copy_mgr->add_column(ts.to_iso());
+        } else if (ltype == LUA_TSTRING) {
+            copy_mgr->add_column(lua_tolstring(lua_state, -1, nullptr));
+        } else {
+            throw fmt_error("Invalid type '{}' for timestamptz column.",
+                            lua_typename(lua_state, ltype));
+        }
     } else if (column.type() == table_column_type::hstore) {
         if (ltype == LUA_TTABLE) {
             copy_mgr->new_hash();

tests/bdd/flex/extra-attributes.feature

@@ -12,6 +12,8 @@ Feature: Tests for including extra attributes
                     { column = 'version', type = 'int4' },
                     { column = 'changeset', type = 'int4' },
                     { column = 'timestamp', type = 'int4' },
+                    { column = 'timestamp_ts', type = 'timestamp' },
+                    { column = 'timestamp_ts_tz', type = 'timestamptz' },
                     { column = 'uid', type = 'int4' },
                     { column = 'user', type = 'text' },
                     { column = 'geom', type = 'linestring', not_null = true },
@@ -20,7 +22,10 @@ Feature: Tests for including extra attributes
 
             function osm2pgsql.process_way(object)
                 object.geom = object:as_linestring()
-                attr_table:insert(object)
+                a = object
+                a.timestamp_ts = a.timestamp
+                a.timestamp_ts_tz = a.timestamp
+                attr_table:insert(a)
             end
             """
 
@@ -36,8 +41,8 @@ Feature: Tests for including extra attributes
             | --slim |
 
         Then table osm2pgsql_test_attr contains
-            | type | way_id | tags->'highway' | tags->'osm_version' | version | changeset | timestamp  | uid  | "user" |
-            | way  | 20     | primary         | NULL                | 1       | 31        | 1578832496 | 17   | test   |
+            | type | way_id | tags->'highway' | tags->'osm_version' | version | changeset | timestamp  | timestamp_ts::text  | to_char(timestamp_ts_tz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') | uid  | "user" |
+            | way  | 20     | primary         | NULL                | 1       | 31        | 1578832496 | 2020-01-12 12:34:56 | 2020-01-12T12:34:56Z                                                      | 17   | test   |
 
         Given the grid
             |    |    |
@@ -46,8 +51,8 @@ Feature: Tests for including extra attributes
             | --slim | --append |
 
         Then table osm2pgsql_test_attr contains
-            | type |  way_id | tags->'highway' | tags->'osm_version' | version | changeset | timestamp | uid  | "user" |
-            | way  |  20     | primary         | NULL                | NULL    | NULL      | NULL      | NULL | NULL   |
+            | type |  way_id | tags->'highway' | tags->'osm_version' | version | changeset | timestamp | timestamp_ts | timestamp_ts_tz | uid  | "user" |
+            | way  |  20     | primary         | NULL                | NULL    | NULL      | NULL      | NULL         | NULL            | NULL | NULL   |
 
 
     Scenario: Importing data with extra attributes

tests/bdd/flex/timestamp.feature

@@ -0,0 +1,35 @@
+Feature: Handling of timestamps
+
+    Scenario: Write tags in different forms to table
+        Given the OSM data
+            """
+            n10 v1 dV t2020-12-12T11:22:33Z Tts=20260102T123456Z x10.0 y10.0
+            n11 v1 dV t2020-12-12T11:22:33Z Tts=2026-02-03T01:23:45Z x10.0 y10.0
+            """
+        And the lua style
+            """
+            local t = osm2pgsql.define_node_table('osm2pgsql_test', {
+                { column = 'ts', type = 'timestamp' },
+                { column = 'ts_tz', type = 'timestamptz' },
+            })
+
+            function osm2pgsql.process_node(object)
+                t:insert{
+                    ts = object.tags.ts,
+                    ts_tz = object.tags.ts,
+                }
+                t:insert{
+                    ts = object.timestamp,
+                    ts_tz = object.timestamp,
+                }
+            end
+            """
+        When running osm2pgsql flex
+
+        Then table osm2pgsql_test contains exactly
+            | node_id | ts::text            | to_char(ts_tz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') |
+            | 10      | 2026-01-02 12:34:56 | 2026-01-02T12:34:56Z                                            |
+            | 11      | 2026-02-03 01:23:45 | 2026-02-03T01:23:45Z                                            |
+            | 10      | 2020-12-12 11:22:33 | 2020-12-12T11:22:33Z                                            |
+            | 11      | 2020-12-12 11:22:33 | 2020-12-12T11:22:33Z                                            |
+