Merge pull request #6 from duneanalytics/CUR2-1191

0xRobin · web-flow · commit 56b7869b7cbc · 2026-02-06T15:26:39.000+01:00
Add bigquery support
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.23 AS builder
+FROM public.ecr.aws/docker/library/golang:1.24.7 AS builder
 
 ENV USER=app
 ENV UID=10001
@@ -12,10 +12,10 @@ RUN adduser \
     --uid "${UID}" \
     "${USER}"
 
-ADD .   /go/src/github.com/a-monteiro/sql_exporter
-WORKDIR /go/src/github.com/a-monteiro/sql_exporter
+ADD .   /go/src/github.com/duneanalytics/sql_exporter
+WORKDIR /go/src/github.com/duneanalytics/sql_exporter
 
-#RUN make drivers-custom
+RUN make drivers-custom
 RUN make build
 RUN chmod +x ./sql_exporter
 
@@ -25,7 +25,7 @@ FROM scratch
 COPY --from=builder /etc/passwd /etc/passwd
 COPY --from=builder /etc/group /etc/group
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=builder /go/src/github.com/a-monteiro/sql_exporter/sql_exporter  /bin/sql_exporter
+COPY --from=builder /go/src/github.com/duneanalytics/sql_exporter/sql_exporter  /bin/sql_exporter
 
 EXPOSE 9100
 USER app:app
diff --git a/README.md b/README.md
@@ -11,6 +11,7 @@ monitoring system. Out of the box, it provides support for the following databas
 - Microsoft SQL Server
 - Oracle Database
 - Clickhouse
+- BigQuery
 - Snowflake
 - Vertica
 
@@ -197,6 +198,18 @@ For example, `p@$$w0rd#abc` then becomes `p%40%24%24w0rd%23abc`.
 
 For additional details please refer to [xo/dburl](https://github.com/xo/dburl) documentation.
 
+#### BigQuery
+
+Use a BigQuery DSN like `bigquery://<project>/<dataset>` (or `bigquery://<project>/<location>/<dataset>`). The driver
+uses `GOOGLE_APPLICATION_CREDENTIALS` to locate the service account JSON file. In Kubernetes, follow the existing
+pattern used for Snowflake/Trino: set `SQLEXPORTER_TARGET_DSN` from a Secret and mount the service account JSON from a
+Secret, then set `GOOGLE_APPLICATION_CREDENTIALS` to the mounted file path.
+
+To target a specific reservation per query, add a `reservation` query parameter to the DSN, for example:
+`bigquery://<project>/<dataset>?reservation=projects/<project>/locations/<location>/reservations/<reservation>`.
+The driver prefixes each query with `SET @@reservation='...'`, which uses BigQuery multi-statement scripting. See
+[BigQuery reservation assignments](https://docs.cloud.google.com/bigquery/docs/reservations-assignments#reservation).
+
 
 ## Miscellaneous
 
diff --git a/documentation/sql_exporter.yml b/documentation/sql_exporter.yml
@@ -35,6 +35,10 @@ target:
   name: mssql_database
   # Data source name always has a URI schema that matches the driver name. In some cases (e.g. MySQL)
   # the schema gets dropped or replaced to match the driver expected DSN format.
+  # BigQuery example:
+  # data_source_name: 'bigquery://<project>/<dataset>?reservation=projects/<project>/locations/<location>/reservations/<reservation>'
+  # BigQuery auth uses GOOGLE_APPLICATION_CREDENTIALS pointing at a service account JSON file.
+  # You can also reference environment variables, e.g. '$SQLEXPORTER_TARGET_DSN'.
   data_source_name: 'sqlserver://prom_user:prom_password@dbserver1.example.com:1433/dbname'
 
   # Collectors (referenced by name) to execute on the target.
diff --git a/drivers.go b/drivers.go
diff --git a/drivers_gen.go b/drivers_gen.go
@@ -29,6 +29,7 @@ var driverList = map[string][]string{
 	},
 	"custom": {
 		//"github.com/mithrandie/csvq-driver",
+		"gorm.io/driver/bigquery/driver",
 		"github.com/trinodb/trino-go-client/trino",
 		"github.com/databricks/databricks-sql-go",
 		"github.com/snowflakedb/gosnowflake",
diff --git a/errors/errors.go b/errors/errors.go
@@ -1,6 +1,7 @@
 package errors
 
 import (
+	"errors"
 	"fmt"
 )
 
@@ -24,7 +25,7 @@ type withContext struct {
 
 // New creates a new WithContext.
 func New(context string, err string) WithContext {
-	return &withContext{context, fmt.Errorf(err)}
+	return &withContext{context, errors.New(err)}
 }
 
 // Errorf formats according to a format specifier and returns a new WithContext.
diff --git a/examples/bigquery.collector.yml b/examples/bigquery.collector.yml
@@ -0,0 +1,10 @@
+collector_name: bigquery_ethereum_blocks
+
+metrics:
+  - metric_name: bigquery_ethereum_blocks_total
+    type: gauge
+    help: 'Total number of rows in ethereum_blocks.'
+    values: [count]
+    query: |
+      SELECT COUNT(*) AS count
+      FROM `PROD_DATASHARES_VIEWS_EU.ethereum_blocks`
diff --git a/examples/bigquery.yml b/examples/bigquery.yml
@@ -0,0 +1,48 @@
+# Example BigQuery target configuration (EU + US in one process).
+#
+# For local testing:
+#   export GOOGLE_APPLICATION_CREDENTIALS=/path/to/bigquery-monitoring-SA-key.json
+#   export SQLEXPORTER_TARGET_DSN_EU='bigquery://datashare-463211/EU/PROD_DATASHARES_VIEWS_EU?reservation=projects/datashare-463211/locations/EU/reservations/prod-datashare-monitoring-eu'
+#   export SQLEXPORTER_TARGET_DSN_US='bigquery://datashare-463211/US/PROD_DATASHARES_VIEWS_US?reservation=projects/datashare-463211/locations/US/reservations/prod-datashare-monitoring-us'
+#
+# For Kubernetes:
+#   - Set SQLEXPORTER_TARGET_DSN_EU/US from Secrets, as in existing Snowflake config.
+#   - Mount the SA JSON from a Secret and set GOOGLE_APPLICATION_CREDENTIALS
+#     to the mounted file path.
+
+global:
+  scrape_timeout: 30s
+
+jobs:
+  - job_name: bigquery_datashare_eu
+    collectors: [bigquery_ethereum_blocks_eu]
+    static_configs:
+      - targets:
+          datashare_eu: '$SQLEXPORTER_TARGET_DSN_EU'
+
+  - job_name: bigquery_datashare_us
+    collectors: [bigquery_ethereum_blocks_us]
+    static_configs:
+      - targets:
+          datashare_us: '$SQLEXPORTER_TARGET_DSN_US'
+
+collectors:
+  - collector_name: bigquery_ethereum_blocks_eu
+    metrics:
+      - metric_name: bigquery_ethereum_blocks_total
+        type: gauge
+        help: 'Total number of rows in ethereum_blocks.'
+        values: [count]
+        query: |
+          SELECT COUNT(*) AS count
+          FROM `PROD_DATASHARES_VIEWS_EU.ethereum_blocks`
+
+  - collector_name: bigquery_ethereum_blocks_us
+    metrics:
+      - metric_name: bigquery_ethereum_blocks_total
+        type: gauge
+        help: 'Total number of rows in ethereum_blocks.'
+        values: [count]
+        query: |
+          SELECT COUNT(*) AS count
+          FROM `PROD_DATASHARES_VIEWS_US.ethereum_blocks`
diff --git a/go.mod b/go.mod
@@ -1,91 +1,102 @@
 module github.com/burningalchemist/sql_exporter
 
-go 1.23.0
+go 1.24.7
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.38.1
-	github.com/aws/aws-sdk-go-v2/config v1.29.17
+	github.com/aws/aws-sdk-go-v2 v1.39.0
+	github.com/aws/aws-sdk-go-v2/config v1.31.8
 	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.35.7
-	github.com/databricks/databricks-sql-go v1.0.1-0.20230105210901-69b282787450
+	github.com/databricks/databricks-sql-go v1.9.0
 	github.com/kardianos/minwinsvc v1.0.2
 	github.com/prometheus/client_golang v1.22.0
 	github.com/prometheus/client_model v0.6.2
 	github.com/prometheus/common v0.65.0
 	github.com/prometheus/exporter-toolkit v0.14.0
 	github.com/sethvargo/go-envconfig v1.3.0
-	github.com/snowflakedb/gosnowflake v1.17.0
-	github.com/trinodb/trino-go-client v0.326.0
+	github.com/snowflakedb/gosnowflake v1.19.0
+	github.com/trinodb/trino-go-client v0.333.0
 	github.com/xo/dburl v0.23.8
 	google.golang.org/protobuf v1.36.6
 	gopkg.in/yaml.v3 v3.0.1
+	gorm.io/driver/bigquery v1.2.0
 )
 
 require (
-	github.com/apache/thrift v0.22.0 // indirect
-	github.com/dnephin/pflag v1.0.7 // indirect
-	github.com/docker/docker v28.2.2+incompatible // indirect
-	github.com/fatih/color v1.15.0 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
-	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.1 // indirect
-	github.com/hashicorp/go-uuid v1.0.3 // indirect
-	github.com/jcmturner/gofork v1.7.6 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/opencontainers/image-spec v1.1.1 // indirect
-	github.com/rs/zerolog v1.28.0 // indirect
-	gotest.tools/gotestsum v1.8.2 // indirect
-)
-
-require (
+	cloud.google.com/go v0.121.0 // indirect
+	cloud.google.com/go/auth v0.16.0 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
+	cloud.google.com/go/bigquery v1.66.2 // indirect
+	cloud.google.com/go/compute/metadata v0.6.0 // indirect
+	cloud.google.com/go/iam v1.5.0 // indirect
 	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
 	github.com/99designs/keyring v1.2.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect
 	github.com/BurntSushi/toml v1.4.0 // indirect
 	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/apache/arrow-go/v18 v18.4.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.70 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect
+	github.com/apache/arrow/go/v12 v12.0.1 // indirect
+	github.com/apache/arrow/go/v15 v15.0.2 // indirect
+	github.com/apache/thrift v0.22.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.18.12 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.7 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.79.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect
-	github.com/aws/smithy-go v1.22.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.88.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.29.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.38.4 // indirect
+	github.com/aws/smithy-go v1.23.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/coreos/go-oidc/v3 v3.5.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/danieljoos/wincred v1.2.2 // indirect
-	github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
+	github.com/dnephin/pflag v1.0.7 // indirect
+	github.com/dvsekhvalnov/jose2go v1.7.0 // indirect
+	github.com/fatih/color v1.16.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.7 // indirect
+	github.com/go-jose/go-jose/v3 v3.0.4 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
+	github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
 	github.com/golang/snappy v1.0.0 // indirect
 	github.com/google/flatbuffers v25.2.10+incompatible // indirect
+	github.com/google/s2a-go v0.1.9 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
+	github.com/googleapis/gax-go/v2 v2.14.1 // indirect
 	github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
+	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
+	github.com/jcmturner/gofork v1.7.6 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/klauspost/asmfmt v1.3.2 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/compress v1.18.1 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.11 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mdlayher/socket v0.4.1 // indirect
 	github.com/mdlayher/vsock v1.2.1 // indirect
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
@@ -95,26 +106,40 @@ require (
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
-	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
-	github.com/rogpeppe/go-internal v1.13.1 // indirect
+	github.com/rs/zerolog v1.28.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
 	go.opentelemetry.io/otel v1.37.0 // indirect
+	go.opentelemetry.io/otel/metric v1.37.0 // indirect
 	go.opentelemetry.io/otel/trace v1.37.0 // indirect
-	golang.org/x/crypto v0.41.0 // indirect
+	golang.org/x/crypto v0.45.0 // indirect
 	golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect
-	golang.org/x/mod v0.27.0 // indirect
-	golang.org/x/net v0.43.0 // indirect
+	golang.org/x/mod v0.29.0 // indirect
+	golang.org/x/net v0.47.0 // indirect
 	golang.org/x/oauth2 v0.30.0 // indirect
-	golang.org/x/sync v0.16.0 // indirect
-	golang.org/x/sys v0.35.0 // indirect
-	golang.org/x/term v0.34.0 // indirect
-	golang.org/x/text v0.28.0 // indirect
-	golang.org/x/tools v0.36.0 // indirect
+	golang.org/x/sync v0.18.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
+	golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect
+	golang.org/x/term v0.37.0 // indirect
+	golang.org/x/text v0.31.0 // indirect
+	golang.org/x/time v0.11.0 // indirect
+	golang.org/x/tools v0.38.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
+	google.golang.org/api v0.230.0 // indirect
+	google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197 // indirect
+	google.golang.org/grpc v1.73.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gotest.tools/gotestsum v1.8.2 // indirect
 )
 
 replace github.com/xo/dburl v0.23.8 => github.com/a-monteiro/dburl v0.23.8-fork
+
+replace gorm.io/driver/bigquery => github.com/duneanalytics/bigquery v1.2.0-dune.1
diff --git a/go.sum b/go.sum

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,7 @@`
`1`	`1`	`package errors`
`2`	`2`
`3`	`3`	`import (`
	`4`	`+ "errors"`
`4`	`5`	`"fmt"`
`5`	`6`	`)`
`6`	`7`
`@@ -24,7 +25,7 @@ type withContext struct {`
`24`	`25`
`25`	`26`	`// New creates a new WithContext.`
`26`	`27`	`func New(context string, err string) WithContext {`
`27`		`- return &withContext{context, fmt.Errorf(err)}`
	`28`	`+ return &withContext{context, errors.New(err)}`
`28`	`29`	`}`
`29`	`30`
`30`	`31`	`// Errorf formats according to a format specifier and returns a new WithContext.`