From 890d72dcee512b96509d3f516093abdb830b044a Mon Sep 17 00:00:00 2001 From: Manuel Dewald Date: Fri, 16 Jan 2026 16:46:59 +0100 Subject: [PATCH 1/3] feat(bootstrap): use local docker registry --- cli/cmd/bootstrap_gcp.go | 6 ++ internal/bootstrap/gcp.go | 173 ++++++++++++++++++++++++++++---------- 2 files changed, 136 insertions(+), 43 deletions(-) diff --git a/cli/cmd/bootstrap_gcp.go b/cli/cmd/bootstrap_gcp.go index 9e68669..9dd476b 100644 --- a/cli/cmd/bootstrap_gcp.go +++ b/cli/cmd/bootstrap_gcp.go @@ -22,6 +22,8 @@ type BootstrapGcpCmd struct { Opts *GlobalOptions Env env.Env CodesphereEnv *bootstrap.CodesphereEnvironment + + InputRegistryType string } func (c *BootstrapGcpCmd) RunE(_ *cobra.Command, args []string) error { @@ -70,6 +72,7 @@ func AddBootstrapGcpCmd(root *cobra.Command, opts *GlobalOptions) { flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.DNSProjectID, "dns-project-id", "", "GCP Project ID for Cloud DNS (optional)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.DNSZoneName, "dns-zone-name", "oms-testing", "Cloud DNS Zone Name (optional)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.InstallCodesphereVersion, "install-codesphere-version", "", "Codesphere version to install (default: none)") + flags.StringVar(&bootstrapGcpCmd.InputRegistryType, "registry-type", "local-container", "Container registry type to use (options: local-container, artifact-registry) (default: artifact-registry)") flags.BoolVar(&bootstrapGcpCmd.CodesphereEnv.WriteConfig, "write-config", true, "Write generated install config to file (default: true)") util.MarkFlagRequired(bootstrapGcpCmd.cmd, "project-name") @@ -81,11 +84,14 @@ func AddBootstrapGcpCmd(root *cobra.Command, opts *GlobalOptions) { } func (c *BootstrapGcpCmd) BootstrapGcp() error { + c.CodesphereEnv.RegistryType = bootstrap.RegistryType(c.InputRegistryType) + gcpClient := bootstrap.NewGCPClient(os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")) bootstrapper, err := bootstrap.NewGCPBootstrapper(c.Env, c.CodesphereEnv, gcpClient) if err != nil { return err } + env, err := bootstrapper.Bootstrap() envBytes, err2 := json.MarshalIndent(env, "", " ") envString := string(envBytes) diff --git a/internal/bootstrap/gcp.go b/internal/bootstrap/gcp.go index 6b0601b..3dd5c46 100644 --- a/internal/bootstrap/gcp.go +++ b/internal/bootstrap/gcp.go @@ -27,6 +27,13 @@ import ( "google.golang.org/grpc/status" ) +type RegistryType string + +const ( + RegistryTypeLocalContainer RegistryType = "local-container" + RegistryTypeArtifactRegistry RegistryType = "artifact-registry" +) + type GCPBootstrapper struct { ctx context.Context env *CodesphereEnvironment @@ -38,20 +45,21 @@ type GCPBootstrapper struct { } type CodesphereEnvironment struct { - ProjectID string `json:"project_id"` - ProjectName string `json:"project_name"` - DNSProjectID string `json:"dns_project_id"` - PostgreSQLNode node.Node `json:"postgresql_node"` - ControlPlaneNodes []node.Node `json:"control_plane_nodes"` - CephNodes []node.Node `json:"ceph_nodes"` - Jumpbox node.Node `json:"jumpbox"` - ContainerRegistryURL string `json:"container_registry_url"` - ExistingConfigUsed bool `json:"existing_config_used"` - InstallCodesphereVersion string `json:"install_codesphere_version"` - Preemptible bool `json:"preemptible"` - WriteConfig bool `json:"write_config"` - GatewayIP string `json:"gateway_ip"` - PublicGatewayIP string `json:"public_gateway_ip"` + ProjectID string `json:"project_id"` + ProjectName string `json:"project_name"` + DNSProjectID string `json:"dns_project_id"` + PostgreSQLNode node.Node `json:"postgresql_node"` + ControlPlaneNodes []node.Node `json:"control_plane_nodes"` + CephNodes []node.Node `json:"ceph_nodes"` + Jumpbox node.Node `json:"jumpbox"` + ContainerRegistryURL string `json:"container_registry_url"` + ExistingConfigUsed bool `json:"existing_config_used"` + InstallCodesphereVersion string `json:"install_codesphere_version"` + Preemptible bool `json:"preemptible"` + WriteConfig bool `json:"write_config"` + GatewayIP string `json:"gateway_ip"` + PublicGatewayIP string `json:"public_gateway_ip"` + RegistryType RegistryType `json:"registry_type"` ProjectDisplayName string BillingAccount string @@ -136,9 +144,11 @@ func (b *GCPBootstrapper) Bootstrap() (*CodesphereEnvironment, error) { return b.env, fmt.Errorf("failed to enable required APIs: %w", err) } - err = b.EnsureArtifactRegistry() - if err != nil { - return b.env, fmt.Errorf("failed to ensure artifact registry: %w", err) + if b.env.RegistryType == RegistryTypeArtifactRegistry { + err = b.EnsureArtifactRegistry() + if err != nil { + return b.env, fmt.Errorf("failed to ensure artifact registry: %w", err) + } } err = b.EnsureServiceAccounts() @@ -186,6 +196,13 @@ func (b *GCPBootstrapper) Bootstrap() (*CodesphereEnvironment, error) { return b.env, fmt.Errorf("failed to ensure hosts are configured: %w", err) } + if b.env.RegistryType == RegistryTypeLocalContainer { + err = b.EnsureLocalContainerRegistry() + if err != nil { + return b.env, fmt.Errorf("failed to ensure local container registry: %w", err) + } + } + if b.env.WriteConfig { err = b.UpdateInstallConfig() if err != nil { @@ -309,39 +326,104 @@ func (b *GCPBootstrapper) EnsureArtifactRegistry() error { return nil } -func (b *GCPBootstrapper) EnsureServiceAccounts() error { - _, _, err := b.EnsureServiceAccount("cloud-controller") - if err != nil { - return err +// Installs a docker registry on the postgres node to speed up image loading time +func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { + localRegistryServer := b.env.PostgreSQLNode.InternalIP + ":5000" + + // Figure out if registry is already running + checkCommand := `test "$(docker ps --filter 'name=registry' --format '{{.Names}}' | wc -l)" -eq "1"` + err := b.env.PostgreSQLNode.RunSSHCommand(&b.env.Jumpbox, b.NodeManager, "root", checkCommand) + if err == nil && b.InstallConfig.Registry != nil && b.InstallConfig.Registry.Server == localRegistryServer && + b.InstallConfig.Registry.Username != "" && b.InstallConfig.Registry.Password != "" { + log.Println("Local container registry already running on postgres node") + return nil } - sa, newSa, err := b.EnsureServiceAccount("artifact-registry-writer") - if err != nil { - return err + + log.Println("Installing registry") + + b.InstallConfig.Registry.Server = localRegistryServer + b.InstallConfig.Registry.Username = "custom-registry" + b.InstallConfig.Registry.Password = shortuuid.New() + + commands := []string{ + "apt-get update", + "apt-get install -y podman apache2-utils", + "htpasswd -bBc /root/registry.password " + b.InstallConfig.Registry.Username + " " + b.InstallConfig.Registry.Password, + "openssl req -newkey rsa:4096 -nodes -sha256 -keyout /root/registry.key -x509 -days 365 -out /root/registry.crt -subj \"/C=DE/ST=BW/L=Karlsruhe/O=Codesphere/CN=" + b.env.PostgreSQLNode.InternalIP + "\" -addext \"subjectAltName = DNS:postgres,IP:" + b.env.PostgreSQLNode.InternalIP + "\"", + "podman rm -f registry || true", + `podman run -d \ + --restart=always --name registry --net=host\ + --env REGISTRY_HTTP_ADDR=0.0.0.0:5000 \ + --env REGISTRY_AUTH=htpasswd \ + --env REGISTRY_AUTH_HTPASSWD_REALM='Registry Realm' \ + --env REGISTRY_AUTH_HTPASSWD_PATH=/auth/registry.password \ + -v /root/registry.password:/auth/registry.password \ + --env REGISTRY_HTTP_TLS_CERTIFICATE=/certs/registry.crt \ + --env REGISTRY_HTTP_TLS_KEY=/certs/registry.key \ + -v /root/registry.crt:/certs/registry.crt \ + -v /root/registry.key:/certs/registry.key \ + registry:2`, + `mkdir -p /etc/docker/certs.d/` + b.InstallConfig.Registry.Server, + `cp /root/registry.crt /etc/docker/certs.d/` + b.InstallConfig.Registry.Server + `/ca.crt`, + } + for _, cmd := range commands { + err := b.env.PostgreSQLNode.RunSSHCommand(&b.env.Jumpbox, b.NodeManager, "root", cmd) + if err != nil { + return fmt.Errorf("failed to run command on postgres node: %w", err) + } } - if !newSa && b.InstallConfig.Registry.Password != "" { - return nil + allNodes := append(b.env.ControlPlaneNodes, b.env.CephNodes...) + for _, node := range allNodes { + err := b.env.PostgreSQLNode.RunSSHCommand(&b.env.Jumpbox, b.NodeManager, "root", "scp -o StrictHostKeyChecking=no /root/registry.crt root@"+node.InternalIP+":/usr/local/share/ca-certificates/registry.crt") + if err != nil { + return fmt.Errorf("failed to copy registry certificate to node %s: %w", node.InternalIP, err) + } + err = node.RunSSHCommand(&b.env.Jumpbox, b.NodeManager, "root", "update-ca-certificates") + if err != nil { + return fmt.Errorf("failed to update CA certificates on node %s: %w", node.InternalIP, err) + } + err = node.RunSSHCommand(&b.env.Jumpbox, b.NodeManager, "root", "systemctl restart docker.service || true") // docker is probably not yet installed + if err != nil { + return fmt.Errorf("failed to restart docker service on node %s: %w", node.InternalIP, err) + } } - for retries := range 5 { - privateKey, err := b.GCPClient.CreateServiceAccountKey(b.ctx, b.env.ProjectID, sa) + return nil +} - if err != nil && status.Code(err) != codes.AlreadyExists { - if retries > 3 { - return fmt.Errorf("failed to create service account key: %w", err) - } +func (b *GCPBootstrapper) EnsureServiceAccounts() error { + _, _, err := b.EnsureServiceAccount("cloud-controller") + if err != nil { + return err + } - log.Printf("got response %d trying to create service account key for %s, retrying...", status.Code(err), sa) + if b.env.RegistryType == RegistryTypeArtifactRegistry { + sa, newSa, err := b.EnsureServiceAccount("artifact-registry-writer") + if err != nil { + return err + } - time.Sleep(5 * time.Second) - continue + if !newSa && b.InstallConfig.Registry.Password != "" { + return nil } - log.Printf("Service account key for %s ensured", sa) - b.InstallConfig.Registry.Password = string(privateKey) - b.InstallConfig.Registry.Username = "_json_key_base64" + for retries := range 5 { + privateKey, err := b.GCPClient.CreateServiceAccountKey(b.ctx, b.env.ProjectID, sa) - break + if err != nil && status.Code(err) != codes.AlreadyExists { + if retries > 3 { + return fmt.Errorf("failed to create service account key: %w", err) + } + log.Printf("got response %d trying to create service account key for %s, retrying...", status.Code(err), sa) + time.Sleep(5 * time.Second) + continue + } + log.Printf("Service account key for %s ensured", sa) + b.InstallConfig.Registry.Password = string(privateKey) + b.InstallConfig.Registry.Username = "_json_key_base64" + break + } } return nil @@ -352,11 +434,16 @@ func (b *GCPBootstrapper) EnsureServiceAccount(name string) (string, bool, error } func (b *GCPBootstrapper) EnsureIAMRoles() error { - err := b.GCPClient.AssignIAMRole(b.ctx, b.env.ProjectID, "artifact-registry-writer", "roles/artifactregistry.writer") + err := b.GCPClient.AssignIAMRole(b.ctx, b.env.ProjectID, "cloud-controller", "roles/compute.admin") if err != nil { return err } - err = b.GCPClient.AssignIAMRole(b.ctx, b.env.ProjectID, "cloud-controller", "roles/compute.admin") + + if b.env.RegistryType != RegistryTypeArtifactRegistry { + return nil + } + + err = b.GCPClient.AssignIAMRole(b.ctx, b.env.ProjectID, "artifact-registry-writer", "roles/artifactregistry.writer") return err } @@ -498,7 +585,7 @@ func (b *GCPBootstrapper) EnsureComputeInstances() error { vmDefs := []VMDef{ {"jumpbox", "e2-medium", []string{"jumpbox", "ssh"}, []int64{}, true}, - {"postgres", "e2-medium", []string{"postgres"}, []int64{50}, true}, + {"postgres", "e2-standard-8", []string{"postgres"}, []int64{}, true}, {"ceph-1", "e2-standard-8", []string{"ceph"}, []int64{20, 200}, false}, {"ceph-2", "e2-standard-8", []string{"ceph"}, []int64{20, 200}, false}, {"ceph-3", "e2-standard-8", []string{"ceph"}, []int64{20, 200}, false}, @@ -741,7 +828,7 @@ func (b *GCPBootstrapper) EnsureRootLoginEnabled() error { if err != nil { return fmt.Errorf("timed out waiting for SSH service to start on jumpbox: %w", err) } - fmt.Printf("SSH service available on jumpbox '%s'\n", b.env.Jumpbox.Name) + log.Printf("SSH service available on jumpbox '%s'", b.env.Jumpbox.Name) hasRootLogin := b.env.Jumpbox.HasRootLoginEnabled(nil, b.NodeManager) if !hasRootLogin { From 31a4a3734932ab3a816728b80b2db9e84ab0bd01 Mon Sep 17 00:00:00 2001 From: Manuel Dewald Date: Thu, 22 Jan 2026 14:05:07 +0100 Subject: [PATCH 2/3] podman to test for existing registry --- internal/bootstrap/gcp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/bootstrap/gcp.go b/internal/bootstrap/gcp.go index 3dd5c46..802a60e 100644 --- a/internal/bootstrap/gcp.go +++ b/internal/bootstrap/gcp.go @@ -331,7 +331,7 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { localRegistryServer := b.env.PostgreSQLNode.InternalIP + ":5000" // Figure out if registry is already running - checkCommand := `test "$(docker ps --filter 'name=registry' --format '{{.Names}}' | wc -l)" -eq "1"` + checkCommand := `test "$(podman ps --filter 'name=registry' --format '{{.Names}}' | wc -l)" -eq "1"` err := b.env.PostgreSQLNode.RunSSHCommand(&b.env.Jumpbox, b.NodeManager, "root", checkCommand) if err == nil && b.InstallConfig.Registry != nil && b.InstallConfig.Registry.Server == localRegistryServer && b.InstallConfig.Registry.Username != "" && b.InstallConfig.Registry.Password != "" { From 1e165852a5489fa15de2b94926261b14e3ba9c8a Mon Sep 17 00:00:00 2001 From: NautiluX <2600004+NautiluX@users.noreply.github.com> Date: Fri, 23 Jan 2026 08:43:00 +0000 Subject: [PATCH 3/3] chore(docs): Auto-update docs and licenses Signed-off-by: NautiluX <2600004+NautiluX@users.noreply.github.com> --- docs/oms-cli_beta_bootstrap-gcp.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/oms-cli_beta_bootstrap-gcp.md b/docs/oms-cli_beta_bootstrap-gcp.md index 108cbb3..1984b14 100644 --- a/docs/oms-cli_beta_bootstrap-gcp.md +++ b/docs/oms-cli_beta_bootstrap-gcp.md @@ -32,6 +32,7 @@ oms-cli beta bootstrap-gcp [flags] --preemptible Use preemptible VMs for Codesphere infrastructure (default: false) --project-name string Unique GCP Project Name (required) --region string GCP Region (default: europe-west4) (default "europe-west4") + --registry-type string Container registry type to use (options: local-container, artifact-registry) (default: artifact-registry) (default "local-container") --secrets-dir string Directory for secrets (default: /etc/codesphere/secrets) (default "/etc/codesphere/secrets") --secrets-file string Path to secrets files (optional) (default "prod.vault.yaml") --ssh-private-key-path string SSH Private Key Path (default: ~/.ssh/id_rsa) (default "~/.ssh/id_rsa")