tmp

Author: blathers-crl[bot]

build/tools/gen-cockroachdb-metrics/main.go

@@ -96,7 +96,7 @@ var (
 	skipPatterns = []*regexp.Regexp{
 		regexp.MustCompile(`^auth_`),
 		regexp.MustCompile(`^distsender_rpc_err_errordetailtype_`),
-		regexp.MustCompile(`^gossip_callbacks_[^_]+_`),
+		regexp.MustCompile(`^gossip_callbacks_`),
 		regexp.MustCompile(`^jobs_auto_config_env_runner_`),
 		regexp.MustCompile(`^jobs_update_table_`),
 		regexp.MustCompile(`^logical_replication_`),
@@ -304,9 +304,7 @@ func loadBaseMappings(path string) (*BaseMappingsYAML, error) {
 
 // collectAllCRDBMetrics collects all CRDB metrics from the YAML output and combines them
 // with any runtime conditional metrics that aren't documented in metrics.yaml
-func collectAllCRDBMetrics(
-	yamlOutput *YAMLOutput, runtimeConditionalMetrics []MetricInfo,
-) []MetricInfo {
+func collectAllCRDBMetrics(yamlOutput *YAMLOutput, runtimeConditionalMetrics []MetricInfo) []MetricInfo {
 	allMetrics := make([]MetricInfo, 0)
 
 	for _, layer := range yamlOutput.Layers {
@@ -330,9 +328,7 @@ func shouldSkipMetric(promName string) bool {
 }
 
 // mapMetricsToDatadog processes the CRDB metrics and maps them to Datadog names
-func mapMetricsToDatadog(
-	metrics []MetricInfo, datadogMappings map[string]string,
-) map[string]string {
+func mapMetricsToDatadog(metrics []MetricInfo, datadogMappings map[string]string) map[string]string {
 	result := make(map[string]string)
 
 	for _, metric := range metrics {
@@ -469,7 +465,7 @@ func main() {
 	flag.Parse()
 
 	args := flag.Args()
-
+	
 	// Check if running in datadog-only mode
 	if *datadogOnly {
 		if len(flag.Args()) < 2 {

docs/generated/settings/settings-for-tenants.txt

@@ -361,6 +361,7 @@ sql.stats.automatic_full_collection.enabled	boolean	true	automatic full statisti
 sql.stats.automatic_partial_collection.enabled	boolean	true	automatic partial statistics collection mode	application
 sql.stats.automatic_partial_collection.fraction_stale_rows	float	0.05	target fraction of stale rows per table that will trigger a partial statistics refresh	application
 sql.stats.automatic_partial_collection.min_stale_rows	integer	100	target minimum number of stale rows per table that will trigger a partial statistics refresh	application
+sql.stats.canary_fraction	float	0	probability that table statistics will use canary mode instead of stable mode for query planning [0.0-1.0]	application
 sql.stats.cleanup.recurrence	string	@hourly	cron-tab recurrence for SQL Stats cleanup job	application
 sql.stats.detailed_latency_metrics.enabled	boolean	false	label latency metrics with the statement fingerprint. Workloads with tens of thousands of distinct query fingerprints should leave this setting false. (experimental, affects performance for workloads with high fingerprint cardinality)	application
 sql.stats.error_on_concurrent_create_stats.enabled	boolean	false	set to true to error on concurrent CREATE STATISTICS jobs, instead of skipping them	application
@@ -427,4 +428,4 @@ trace.zipkin.collector	string		the address of a Zipkin instance to receive trace
 ui.database_locality_metadata.enabled	boolean	true	if enabled shows extended locality data about databases and tables in DB Console which can be expensive to compute	application
 ui.default_timezone	string		the default timezone used to format timestamps in the ui	application
 ui.display_timezone	enumeration	etc/utc	the timezone used to format timestamps in the ui. This setting is deprecatedand will be removed in a future version. Use the 'ui.default_timezone' setting instead. 'ui.default_timezone' takes precedence over this setting. [etc/utc = 0, america/new_york = 1]	application
-version	version	1000025.4-upgrading-to-1000026.1-step-006	set the active cluster version in the format '<major>.<minor>'	application
+version	version	25.4-upgrading-to-26.1-step-006	set the active cluster version in the format '<major>.<minor>'	application

docs/generated/settings/settings.html

@@ -316,6 +316,7 @@
 <tr><td><div id="setting-sql-stats-automatic-partial-collection-enabled" class="anchored"><code>sql.stats.automatic_partial_collection.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>automatic partial statistics collection mode</td><td>Basic/Standard/Advanced/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-stats-automatic-partial-collection-fraction-stale-rows" class="anchored"><code>sql.stats.automatic_partial_collection.fraction_stale_rows</code></div></td><td>float</td><td><code>0.05</code></td><td>target fraction of stale rows per table that will trigger a partial statistics refresh</td><td>Basic/Standard/Advanced/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-stats-automatic-partial-collection-min-stale-rows" class="anchored"><code>sql.stats.automatic_partial_collection.min_stale_rows</code></div></td><td>integer</td><td><code>100</code></td><td>target minimum number of stale rows per table that will trigger a partial statistics refresh</td><td>Basic/Standard/Advanced/Self-Hosted</td></tr>
+<tr><td><div id="setting-sql-stats-canary-fraction" class="anchored"><code>sql.stats.canary_fraction</code></div></td><td>float</td><td><code>0</code></td><td>probability that table statistics will use canary mode instead of stable mode for query planning [0.0-1.0]</td><td>Basic/Standard/Advanced/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-stats-cleanup-recurrence" class="anchored"><code>sql.stats.cleanup.recurrence</code></div></td><td>string</td><td><code>@hourly</code></td><td>cron-tab recurrence for SQL Stats cleanup job</td><td>Basic/Standard/Advanced/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-stats-detailed-latency-metrics-enabled" class="anchored"><code>sql.stats.detailed_latency_metrics.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>label latency metrics with the statement fingerprint. Workloads with tens of thousands of distinct query fingerprints should leave this setting false. (experimental, affects performance for workloads with high fingerprint cardinality)</td><td>Basic/Standard/Advanced/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-stats-error-on-concurrent-create-stats-enabled" class="anchored"><code>sql.stats.error_on_concurrent_create_stats.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>set to true to error on concurrent CREATE STATISTICS jobs, instead of skipping them</td><td>Basic/Standard/Advanced/Self-Hosted</td></tr>
@@ -386,6 +387,6 @@
 <tr><td><div id="setting-ui-database-locality-metadata-enabled" class="anchored"><code>ui.database_locality_metadata.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>if enabled shows extended locality data about databases and tables in DB Console which can be expensive to compute</td><td>Basic/Standard/Advanced/Self-Hosted</td></tr>
 <tr><td><div id="setting-ui-default-timezone" class="anchored"><code>ui.default_timezone</code></div></td><td>string</td><td><code></code></td><td>the default timezone used to format timestamps in the ui</td><td>Basic/Standard/Advanced/Self-Hosted</td></tr>
 <tr><td><div id="setting-ui-display-timezone" class="anchored"><code>ui.display_timezone</code></div></td><td>enumeration</td><td><code>etc/utc</code></td><td>the timezone used to format timestamps in the ui. This setting is deprecatedand will be removed in a future version. Use the &#39;ui.default_timezone&#39; setting instead. &#39;ui.default_timezone&#39; takes precedence over this setting. [etc/utc = 0, america/new_york = 1]</td><td>Basic/Standard/Advanced/Self-Hosted</td></tr>
-<tr><td><div id="setting-version" class="anchored"><code>version</code></div></td><td>version</td><td><code>1000025.4-upgrading-to-1000026.1-step-006</code></td><td>set the active cluster version in the format &#39;&lt;major&gt;.&lt;minor&gt;&#39;</td><td>Basic/Standard/Advanced/Self-Hosted</td></tr>
+<tr><td><div id="setting-version" class="anchored"><code>version</code></div></td><td>version</td><td><code>25.4-upgrading-to-26.1-step-006</code></td><td>set the active cluster version in the format &#39;&lt;major&gt;.&lt;minor&gt;&#39;</td><td>Basic/Standard/Advanced/Self-Hosted</td></tr>
 </tbody>
 </table>

pkg/BUILD.bazel

@@ -211,7 +211,6 @@ ALL_TESTS = [
     "//pkg/jobs/jobspb:jobspb_test",
     "//pkg/jobs/jobsprofiler:jobsprofiler_test",
     "//pkg/jobs/jobsprotectedts:jobsprotectedts_test",
-    "//pkg/jobs/metricspoller:metricspoller_test",
     "//pkg/jobs:jobs_test",
     "//pkg/keys:keys_test",
     "//pkg/keyvisualizer/spanstatsconsumer:spanstatsconsumer_test",
@@ -729,10 +728,8 @@ ALL_TESTS = [
     "//pkg/upgrade/upgrades:upgrades_test",
     "//pkg/util/admission/admissionpb:admissionpb_test",
     "//pkg/util/admission:admission_test",
-    "//pkg/util/allstacks:allstacks_test",
     "//pkg/util/assetbundle:assetbundle_test",
     "//pkg/util/base64:base64_test",
-    "//pkg/util/besteffort:besteffort_test",
     "//pkg/util/binfetcher:binfetcher_test",
     "//pkg/util/bitarray:bitarray_test",
     "//pkg/util/bitmap:bitmap_test",
@@ -1441,7 +1438,6 @@ GO_TARGETS = [
     "//pkg/jobs/jobsprotectedts:jobsprotectedts_test",
     "//pkg/jobs/jobstest:jobstest",
     "//pkg/jobs/metricspoller:metricspoller",
-    "//pkg/jobs/metricspoller:metricspoller_test",
     "//pkg/jobs:jobs",
     "//pkg/jobs:jobs_test",
     "//pkg/keys:keys",
@@ -2603,7 +2599,6 @@ GO_TARGETS = [
     "//pkg/util/admission:admission",
     "//pkg/util/admission:admission_test",
     "//pkg/util/allstacks:allstacks",
-    "//pkg/util/allstacks:allstacks_test",
     "//pkg/util/arith:arith",
     "//pkg/util/asciitsdb:asciitsdb",
     "//pkg/util/assetbundle/gen:gen",
@@ -2612,8 +2607,6 @@ GO_TARGETS = [
     "//pkg/util/assetbundle:assetbundle_test",
     "//pkg/util/base64:base64",
     "//pkg/util/base64:base64_test",
-    "//pkg/util/besteffort:besteffort",
-    "//pkg/util/besteffort:besteffort_test",
     "//pkg/util/binfetcher:binfetcher",
     "//pkg/util/binfetcher:binfetcher_test",
     "//pkg/util/bitarray:bitarray",

pkg/backup/BUILD.bazel

@@ -293,6 +293,7 @@ go_test(
         "//pkg/sql/sem/tree",
         "//pkg/sql/sessiondata",
         "//pkg/sql/sqlclustersettings",
+        "//pkg/sql/sqlliveness/slbase",
         "//pkg/sql/sqltestutils",
         "//pkg/sql/stats",
         "//pkg/storage",

pkg/backup/backup_tenant_test.go

@@ -9,16 +9,19 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/ccl/multiregionccl/multiregionccltestutils"
 	_ "github.com/cockroachdb/cockroach/pkg/cloud/impl" // register cloud storage providers
 	"github.com/cockroachdb/cockroach/pkg/jobs"
 	"github.com/cockroachdb/cockroach/pkg/jobs/jobspb"
+	"github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilitiespb"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/sql"
 	_ "github.com/cockroachdb/cockroach/pkg/sql/importer"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlclustersettings"
+	"github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slbase"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/jobutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
@@ -27,10 +30,78 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/testutils/testcluster"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/cockroachdb/errors"
 	"github.com/stretchr/testify/require"
 )
 
+func TestBackupSharedProcessTenantNodeDown(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	ctx := context.Background()
+
+	skip.UnderRace(t, "multi-node, multi-tenant test too slow under race")
+	skip.UnderDeadlock(t, "too slow under deadlock detector")
+	params := base.TestClusterArgs{
+		ServerArgs: base.TestServerArgs{
+			DefaultTestTenant: base.TestControlsTenantsExplicitly,
+		},
+	}
+	params.ServerArgs.Knobs.JobsTestingKnobs = jobs.NewTestingKnobsWithShortIntervals()
+	tc, hostDB, _, cleanup := backupRestoreTestSetupWithParams(t, multiNode, 0, /* numAccounts */
+		InitManualReplication, params)
+	defer cleanup()
+
+	hostDB.Exec(t, "ALTER TENANT ALL SET CLUSTER SETTING server.sqlliveness.ttl='2s'")
+	hostDB.Exec(t, "ALTER TENANT ALL SET CLUSTER SETTING server.sqlliveness.heartbeat='250ms'")
+
+	testTenantID := roachpb.MustMakeTenantID(11)
+	tenantApp, tenantDB, err := tc.Server(0).TenantController().StartSharedProcessTenant(ctx,
+		base.TestSharedProcessTenantArgs{
+			TenantID:   testTenantID,
+			TenantName: "test",
+		})
+	require.NoError(t, err)
+
+	tc.GrantTenantCapabilities(
+		ctx, t, testTenantID,
+		map[tenantcapabilitiespb.ID]string{tenantcapabilitiespb.CanUseNodelocalStorage: "true"})
+
+	tenantSQL := sqlutils.MakeSQLRunner(tenantDB)
+	tenantSQL.Exec(t, "CREATE TABLE foo AS SELECT generate_series(1, 4000)")
+	tenantSQL.Exec(t, "ALTER TABLE foo SPLIT AT VALUES (500), (1000), (1500), (2000), (2500), (3000)")
+	tenantSQL.Exec(t, "ALTER TABLE foo SCATTER")
+
+	t.Log("waiting for SQL instances")
+	waitStart := timeutil.Now()
+	for i := 1; i < multiNode; i++ {
+		testutils.SucceedsSoon(t, func() error {
+			t.Logf("waiting for server %d", i)
+			db, err := tc.Server(i).SystemLayer().SQLConnE(serverutils.DBName("cluster:test/defaultdb"))
+			if err != nil {
+				return err
+			}
+			return db.Ping()
+		})
+	}
+	t.Logf("all SQL instances (took %s)", timeutil.Since(waitStart))
+
+	// Shut down a node.
+	t.Log("shutting down server 2 (n3)")
+	tc.StopServer(2)
+
+	// We use succeeds soon here since it still takes some time
+	// for instance-based planning to recognize the downed node.
+	sv := &tenantApp.ClusterSettings().SV
+	padding := 10 * time.Second
+	timeout := slbase.DefaultTTL.Get(sv) + slbase.DefaultHeartBeat.Get(sv) + padding
+	testutils.SucceedsWithin(t, func() error {
+		_, err := tenantDB.Exec("BACKUP INTO 'nodelocal://1/worker-failure'")
+		return err
+	}, timeout)
+}
+
 func TestBackupTenantImportingTable(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)

pkg/backup/datadriven_test.go

@@ -27,7 +27,6 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/jobs/jobspb"
 	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/kv"
-	"github.com/cockroachdb/cockroach/pkg/kv/bulk"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvpb"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver"
@@ -46,7 +45,6 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
-	"github.com/cockroachdb/cockroach/pkg/util/admission"
 	"github.com/cockroachdb/cockroach/pkg/util/ctxgroup"
 	"github.com/cockroachdb/datadriven"
 	"github.com/cockroachdb/errors"
@@ -196,12 +194,6 @@ func (d *datadrivenTestState) addCluster(t *testing.T, cfg clusterCfg) error {
 	closedts.SideTransportCloseInterval.Override(context.Background(), &settings.SV, 10*time.Millisecond)
 	kvserver.RangeFeedRefreshInterval.Override(context.Background(), &settings.SV, 10*time.Millisecond)
 	sql.TempObjectWaitInterval.Override(context.Background(), &settings.SV, time.Millisecond)
-	// Disable AC yielding as these tests can run many in-process clusters at once
-	// and overload the host. Generally overload would mean bulk work, which only
-	// uses strictly spare capacitym gets starved, but these tests expect it to
-	// still run (just slowly, along with everything else).
-	bulk.YieldIfNoPacer.Override(context.Background(), &settings.SV, false)
-	admission.YieldInPacer.Override(context.Background(), &settings.SV, false)
 	params.ServerArgs.Settings = settings
 
 	clusterSize := cfg.nodes

pkg/backup/restore_schema_change_creation.go

@@ -15,7 +15,6 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/security/username"
-	"github.com/cockroachdb/cockroach/pkg/sql"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog"
 	descpb "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc"
@@ -196,9 +195,6 @@ func createSchemaChangeJobsFromMutations(
 				spanList[i] = jobspb.ResumeSpanList{ResumeSpans: []roachpb.Span{tableDesc.PrimaryIndexSpan(codec)}}
 			}
 		}
-		// Populate session data for schema changes that require it.
-		sd := sql.NewInternalSessionData(ctx, jr.ClusterSettings(), "schema-change-restore")
-		sd.UserProto = username.EncodeProto()
 		jobRecord := jobs.Record{
 			// We indicate that this schema change was triggered by a RESTORE since
 			// the job description may not have all the information to fully describe
@@ -210,7 +206,6 @@ func createSchemaChangeJobsFromMutations(
 				DescID:          tableDesc.ID,
 				TableMutationID: mutationID,
 				ResumeSpanList:  spanList,
-				SessionData:     &sd.SessionData,
 				// The version distinction for database jobs doesn't matter for a schema
 				// change on a single table.
 				FormatVersion: jobspb.DatabaseJobFormatVersion,

pkg/backup/testdata/backup-restore/materialized_view

@@ -8,7 +8,6 @@ CREATE DATABASE testdb;
 USE testdb;
 CREATE TABLE testdb.t (a int primary key, b int);
 CREATE MATERIALIZED VIEW testdb.mv AS SELECT a, b FROM testdb.t;
-CREATE MATERIALIZED VIEW testdb.mv2 AS SELECT a, b FROM testdb.t;
 INSERT INTO testdb.t (a, b) VALUES (1, 2);
 ----
 
@@ -24,18 +23,6 @@ exec-sql
 REFRESH MATERIALIZED VIEW mv;
 ----
 
-
-# Pause the schema change intentionally to refresh a view on mv2
-exec-sql
-SET CLUSTER SETTING jobs.debug.pausepoints = 'schemachanger.before.exec'
-----
-
-
-exec-sql expect-error-regex=(pq: job \d+ was paused before it completed with reason: pause point "schemachanger.before.exec" hit)
-REFRESH MATERIALIZED VIEW mv2;
-----
-regex matches error
-
 exec-sql
 BACKUP INTO 'nodelocal://1/test/'
 ----

pkg/build/version.txt

@@ -1 +1 @@
-v26.1.0-alpha.2
+v26.1.0-beta.1