MDEV-15621 Auto add RANGE COLUMNS partitions by interval

Allow auto partitioning by interval in PARTITION BY RANGE COLUMNS

PARTITION BY RANGE COLUMNS (col_name)
INTERVAL interval [AUTO]
(
  PARTITION partition_name VALUES LESS THAN (value)
  [, PARTITION partition_name VALUES LESS THAN (value) ... ]
)

where

- col_name is the name of one column of type DATE or DATETIME or
  TIMESTAMP

- at least one partition is supplied, and the highest partition cannot
  have MAXVALUE range

- INTERVAL interval is a positive time interval. it can be mariadb
  format or oracle NUMTODSINTERVAL/NUMTOYMINTERVAL format. Like
  versioning, the smallest unit is second, i.e. no subsecond like
  microsecond.

- DATE column cannot have interval with values less than a day

When performing DML on such a table, it will first add partitions
by the specified interval until the partition covers the current time.

Partition addition will not cause an implicit commit like DDL normally
does.

The partitions are named pN.

Otherwise the table behaves exactly the same as a normal RANGE COLUMNS
partitioned table.

Note that TIMESTAMP is not allowed as a type for PARTITION BY RANGE
COLUMNS otherwise.

Author: mariadb-YuchenPei

mysql-test/main/partition_range_interval.result

@@ -778,6 +778,8 @@ SYMBOL sql_functions[] = {
   { "NOW",		SYM(NOW_SYM)},
   { "NTH_VALUE",        SYM(NTH_VALUE_SYM)},
   { "NTILE",            SYM(NTILE_SYM)},
+  { "NUMTODSINTERVAL",		SYM(NUMTODSINTERVAL_SYM)},
+  { "NUMTOYMINTERVAL",		SYM(NUMTOYMINTERVAL_SYM)},
   { "POSITION",		SYM(POSITION_SYM)},
   { "PERCENT_RANK",     SYM(PERCENT_RANK_SYM)},
   { "PERCENTILE_CONT",  SYM(PERCENTILE_CONT_SYM)},

mysql-test/main/partition_range_interval.test

@@ -400,7 +400,7 @@ bool partition_info::set_up_default_partitions(THD *thd, handler *file,
       num_parts= 2;
     use_default_num_partitions= false;
   }
-  else if (part_type != HASH_PARTITION)
+  else if (part_type != HASH_PARTITION && !is_range_interval())
   {
     const char *error_string;
     if (part_type == RANGE_PARTITION)
@@ -891,6 +891,39 @@ bool partition_info::vers_set_hist_part(THD *thd, uint *create_count)
   return false;
 }
 
+/*
+  Determine the number of range interval partitions to create, like
+  partition_info::vers_set_hist_part.
+
+  TODO(MDEV-15621): consider making this more efficient by using a
+  hash, possibly like Partition_share::partition_name_hash
+*/
+bool partition_info::range_interval_set_count(THD* thd, uint *create_count)
+{
+  /* At least one range partition is defined */
+  DBUG_ASSERT(partitions.elements > 0);
+  partition_element *el= partitions.elem(partitions.elements - 1);
+  Item *item= el->get_col_val(0).item_expression;
+  MYSQL_TIME cur_time, end_time;
+  thd->variables.time_zone->gmt_sec_to_TIME(&end_time, thd->query_start());
+  longlong cur= item->val_datetime_packed(thd);
+  unpack_time(cur, &cur_time, MYSQL_TIMESTAMP_DATETIME);
+  longlong end= pack_time(&end_time);
+  *create_count= 0;
+  while (cur <= end)
+  {
+    if (date_add_interval(thd, &cur_time, int_type, interval))
+      return true;
+    cur= pack_time(&cur_time);
+    ++*create_count;
+    if (partitions.elements + *create_count > MAX_PARTITIONS)
+    {
+      my_error(ER_TOO_MANY_PARTITIONS_ERROR, MYF(0));
+      return true;
+    }
+  }
+  return false;
+}
 
 /**
   @brief Run fast_alter_partition_table() to add new history partitions
@@ -1014,6 +1047,103 @@ bool vers_create_partitions(THD *thd, TABLE_LIST* tl, uint num_parts)
   return result;
 }
 
+/*
+  similar to vers_create_partitions, create range interval partitions
+*/
+bool range_interval_create_partitions(THD* thd, TABLE_LIST* tl, uint num_parts)
+{
+  bool result= true;
+  Table_specification_st create_info;
+  Alter_info alter_info;
+  TABLE *table= tl->table;
+  partition_info *save_part_info= thd->work_part_info;
+  /* TODO: this may still trigger MSAN unitialised? */
+  bool save_no_write_to_binlog= thd->lex->no_write_to_binlog;
+  thd->lex->no_write_to_binlog= true;
+
+  DBUG_ASSERT(!thd->is_error());
+  DBUG_ASSERT(num_parts);
+
+  {
+    alter_info.reset();
+    alter_info.partition_flags= ALTER_PARTITION_ADD;
+    create_info.init();
+    create_info.alter_info= &alter_info;
+    Alter_table_ctx alter_ctx(thd, tl, 1, &table->s->db, &table->s->table_name);
+
+    MDL_REQUEST_INIT(&tl->mdl_request, MDL_key::TABLE, tl->db.str,
+                    tl->table_name.str, MDL_SHARED_NO_WRITE, MDL_TRANSACTION);
+    if (thd->mdl_context.acquire_lock(&tl->mdl_request,
+                                      thd->variables.lock_wait_timeout))
+      goto exit;
+    table->mdl_ticket= tl->mdl_request.ticket;
+
+    create_info.db_type= table->s->db_type();
+    DBUG_ASSERT(create_info.db_type);
+
+    partition_info *part_info= new partition_info();
+    if (unlikely(!part_info))
+    {
+      my_error(ER_OUT_OF_RESOURCES, MYF(0));
+      goto exit;
+    }
+    part_info->use_default_num_partitions= false;
+    part_info->use_default_num_subpartitions= false;
+    part_info->num_parts= num_parts;
+    part_info->num_subparts= table->part_info->num_subparts;
+    part_info->subpart_type= table->part_info->subpart_type;
+    part_info->num_columns= table->part_info->num_columns;
+    part_info->part_type= RANGE_PARTITION;
+    /* for partition_info::fix_parser_data to exit early */
+    part_info->int_type= table->part_info->int_type;
+
+    thd->work_part_info= part_info;
+    bool partition_changed= false;
+    bool fast_alter_partition= false;
+    if (prep_alter_part_table(thd, table, &alter_info, &create_info,
+                              &partition_changed, &fast_alter_partition))
+    {
+      my_error(ER_INTERNAL_ERROR, MYF(ME_WARNING),
+               tl->db.str, tl->table_name.str);
+      goto exit;
+    }
+    if (!fast_alter_partition)
+    {
+      my_error(ER_INTERNAL_ERROR, MYF(ME_WARNING),
+               tl->db.str, tl->table_name.str);
+      goto exit;
+    }
+    DBUG_ASSERT(partition_changed);
+    if (mysql_prepare_alter_table(thd, table, &create_info, &alter_info,
+                                  &alter_ctx))
+    {
+      my_error(ER_INTERNAL_ERROR, MYF(ME_WARNING),
+               tl->db.str, tl->table_name.str);
+      goto exit;
+    }
+
+    alter_info.db= alter_ctx.db;
+    alter_info.table_name= alter_ctx.table_name;
+    if (fast_alter_partition_table(thd, table, &alter_info, &alter_ctx,
+                                   &create_info, tl))
+    {
+      my_error(ER_INTERNAL_ERROR, MYF(ME_WARNING),
+               tl->db.str, tl->table_name.str);
+      goto exit;
+    }
+  }
+
+  result= false;
+  // NOTE: we have to return DA_EMPTY for new command
+  DBUG_ASSERT(thd->get_stmt_da()->is_ok());
+  thd->get_stmt_da()->reset_diagnostics_area();
+  thd->variables.option_bits|= OPTION_BINLOG_THIS;
+
+exit:
+  thd->lex->no_write_to_binlog= save_no_write_to_binlog;
+  thd->work_part_info= save_part_info;
+  return result;
+}
 
 /**
   Warn at the end of DML command if the last history partition is out of LIMIT.
@@ -2320,6 +2450,7 @@ bool partition_info::fix_parser_data(THD *thd)
   partition_element *part_elem;
   uint num_elements;
   uint i= 0, j, k;
+  int sql_command= thd_sql_command(thd);
   DBUG_ENTER("partition_info::fix_parser_data");
 
   if (!(part_type == RANGE_PARTITION ||
@@ -2334,13 +2465,25 @@ bool partition_info::fix_parser_data(THD *thd)
         DBUG_RETURN(true);
       }
       /* If not set, use DEFAULT = 2 for CREATE and ALTER! */
-      if ((thd_sql_command(thd) == SQLCOM_CREATE_TABLE ||
-           thd_sql_command(thd) == SQLCOM_ALTER_TABLE) &&
+      if ((sql_command == SQLCOM_CREATE_TABLE ||
+           sql_command == SQLCOM_ALTER_TABLE) &&
           key_algorithm == KEY_ALGORITHM_NONE)
         key_algorithm= KEY_ALGORITHM_55;
     }
     DBUG_RETURN(FALSE);
   }
+  /*
+    TODO(MDEV-15621): we exit here for range interval partitions
+    because if this is called from prep_alter_part_table then
+    partition data is not calculated (in
+    check_range_interval_constants) yet. But maybe we shouldn't exit
+    here in user CREATE/ALTER TABLE statements for other checks.
+  */
+  if (sql_command != SQLCOM_CREATE_TABLE &&
+      sql_command != SQLCOM_ALTER_TABLE && is_range_interval())
+  {
+    DBUG_RETURN(FALSE);
+  }
   if (is_sub_partitioned() && list_of_subpart_fields)
   {
     /* KEY subpartitioning, check ALGORITHM = N. Should not pass the parser! */
@@ -2785,6 +2928,76 @@ bool partition_info::vers_init_info(THD * thd)
   return false;
 }
 
+bool partition_info::set_interval(THD* thd, Item* ival, interval_type type,
+                                  const char *table_name)
+{
+  bool error= get_interval_value(thd, ival, type, &interval) ||
+    interval.neg || interval.second_part ||
+    !(interval.year || interval.month || interval.day || interval.hour ||
+      interval.minute || interval.second);
+  if (error)
+  {
+    my_error(ER_PART_WRONG_VALUE, MYF(0), table_name, "INTERVAL");
+    return true;
+  }
+  int_type= type;
+  return false;
+}
+
+bool partition_info::set_interval(int num, LEX_CSTRING &type, bool is_ds,
+                                  const char *table_name)
+{
+  if (num < 0)
+    goto end;
+  if (is_ds)
+  {
+    if (type.length == 3 && !strncasecmp(type.str, "DAY", 3))
+    {
+      interval.day= num;
+      int_type= INTERVAL_DAY;
+      return false;
+    }
+    else if (type.length == 4 && !strncasecmp(type.str, "HOUR", 4))
+    {
+      interval.hour= num;
+      int_type= INTERVAL_HOUR;
+      return false;
+    }
+    else if (type.length == 6)
+    {
+      if (!strncasecmp(type.str, "MINUTE", 6))
+      {
+        interval.minute= num;
+        int_type= INTERVAL_MINUTE;
+        return false;
+      }
+      else if (!strncasecmp(type.str, "SECOND", 6))
+      {
+        interval.second= num;
+        int_type= INTERVAL_SECOND;
+        return false;
+      }
+    }
+  }
+  else
+  {
+    if (type.length == 4 && !strncasecmp(type.str, "YEAR", 4))
+    {
+      interval.year= num;
+      int_type= INTERVAL_YEAR;
+      return false;
+    }
+    else if (type.length == 5 && !strncasecmp(type.str, "MONTH", 5))
+    {
+      interval.month= num;
+      int_type= INTERVAL_MONTH;
+      return false;
+    }
+  }
+end:
+  my_error(ER_PART_WRONG_VALUE, MYF(0), table_name, "INTERVAL");
+  return true;
+}
 
 /**
   Assign INTERVAL and STARTS for SYSTEM_TIME partitions.

sql/lex.h

@@ -207,6 +207,10 @@ class partition_info : public DDL_LOG_STATE, public Sql_alloc
     part_column_list_val *list_col_array;
   };
 
+  /* TODO: change to pointer */
+  INTERVAL interval;
+  enum interval_type int_type;
+
   Vers_part_info *vers_info;
 
   /********************************************
@@ -319,7 +323,7 @@ class partition_info : public DDL_LOG_STATE, public Sql_alloc
     restore_part_field_ptrs(NULL), restore_subpart_field_ptrs(NULL),
     part_expr(NULL), subpart_expr(NULL), item_free_list(NULL),
     bitmaps_are_initialized(FALSE),
-    list_array(NULL), vers_info(NULL), err_value(0),
+    list_array(NULL), int_type(INTERVAL_LAST), vers_info(NULL), err_value(0),
     part_info_string(NULL),
     curr_part_elem(NULL), current_partition(NULL),
     curr_list_object(0), num_columns(0), table(NULL),
@@ -348,6 +352,9 @@ class partition_info : public DDL_LOG_STATE, public Sql_alloc
     temp_partitions.empty();
     part_field_list.empty();
     subpart_field_list.empty();
+    interval.second_part= interval.second= interval.minute=
+      interval.hour= interval.day= interval.month= interval.year= 0;
+    interval.neg= FALSE;
   }
   ~partition_info() = default;
 
@@ -417,11 +424,16 @@ class partition_info : public DDL_LOG_STATE, public Sql_alloc
   bool field_in_partition_expr(Field *field) const;
 
   bool vers_init_info(THD *thd);
+  bool set_interval(THD* thd, Item* ival, interval_type type,
+                    const char *table_name);
+  bool set_interval(int num, LEX_CSTRING& type, bool is_ds,
+                    const char *table_name);
   bool vers_set_interval(THD *thd, Item *interval,
                          interval_type int_type, Item *starts,
                          bool auto_part, const char *table_name);
   bool vers_set_limit(ulonglong limit, bool auto_part, const char *table_name);
   bool vers_set_hist_part(THD* thd, uint *create_count);
+  bool range_interval_set_count(THD* thd, uint *create_count);
   bool vers_require_hist_part(THD *thd) const
   {
     return part_type == VERSIONING_PARTITION &&
@@ -430,6 +442,8 @@ class partition_info : public DDL_LOG_STATE, public Sql_alloc
   void vers_check_limit(THD *thd);
   bool vers_fix_field_list(THD *thd);
   void vers_update_el_ids();
+  bool is_range_interval() const
+  { return int_type != INTERVAL_LAST && part_type == RANGE_PARTITION; }
   partition_element *get_partition(uint part_id)
   {
     List_iterator<partition_element> it(partitions);
@@ -452,6 +466,7 @@ void part_type_error(THD *thd, partition_info *work_part_info,
 uint32 get_next_partition_id_range(struct st_partition_iter* part_iter);
 bool check_partition_dirs(partition_info *part_info);
 bool vers_create_partitions(THD* thd, TABLE_LIST* tl, uint num_parts);
+bool range_interval_create_partitions(THD* thd, TABLE_LIST* tl, uint num_parts);
 
 /* Initialize the iterator to return a single partition with given part_id */
 
@@ -549,12 +564,12 @@ Lex_ident_partition make_partition_name(char *move_ptr, uint i)
 inline
 uint partition_info::next_part_no(uint new_parts) const
 {
-  if (part_type != VERSIONING_PARTITION)
+  if (part_type != VERSIONING_PARTITION && !is_range_interval())
     return num_parts;
   DBUG_ASSERT(new_parts > 0);
   /* Choose first non-occupied name suffix */
-  uint32 suffix= num_parts - 1;
-  DBUG_ASSERT(suffix > 0);
+  uint32 suffix= part_type == VERSIONING_PARTITION ? num_parts - 1 : 0;
+  DBUG_ASSERT(suffix > 0 || part_type != VERSIONING_PARTITION);
   char part_name[MAX_PART_NAME_SIZE + 1];
   List_iterator_fast<partition_element> it(table->part_info->partitions);
   for (uint cur_part= 0; cur_part < new_parts; ++cur_part, ++suffix)

sql/partition_info.cc

@@ -12404,3 +12404,9 @@ ER_WARN_QB_NAME_PATH_VIEW_NOT_FOUND
         eng "Hint %s is ignored. `%s` required at element #%u of the path is not found in the target query block."
 ER_WARN_QB_NAME_PATH_NOT_SUPPORTED_INSIDE_VIEW
         eng "Hint %s is ignored. QB_NAME hints with path are not supported inside view definitions."
+ER_PARTITION_INTERVAL_NOT_LIST
+        eng "LIST partition type does not support INTERVAL"
+ER_PARTITION_INTERVAL_FINER_THAN_DATE
+        eng "RANGE COLUMN partition by a DATE with INTERVAL smaller than date"
+ER_PARTITION_INTERVAL_MAXVALUE
+        eng "MAXVALUE is not allowed in range partitioning with interval"