diff --git a/charts/supply-chain/templates/pipeline-qtodo.yaml b/charts/supply-chain/templates/pipeline-qtodo.yaml index 6b77d973..a5d02f9b 100644 --- a/charts/supply-chain/templates/pipeline-qtodo.yaml +++ b/charts/supply-chain/templates/pipeline-qtodo.yaml @@ -110,6 +110,10 @@ spec: - name: registry-auth-config - name: git-auth optional: true +{{- if and .Values.git.sslCABundle.enabled (ne (default "https" .Values.git.credentials.authType) "ssh") }} + - name: ssl-ca-directory + optional: true +{{- end }} results: - name: CHAINS-GIT_URL @@ -160,6 +164,10 @@ spec: value: $(params.git-url) - name: REVISION value: $(params.git-revision) +{{- if and .Values.git.sslCABundle.enabled (ne (default "https" .Values.git.credentials.authType) "ssh") }} + - name: CRT_FILENAME + value: tls-ca-bundle.pem +{{- end }} workspaces: - name: output workspace: qtodo-source @@ -170,6 +178,10 @@ spec: - name: basic-auth workspace: git-auth {{- end }} +{{- if and .Values.git.sslCABundle.enabled (ne (default "https" .Values.git.credentials.authType) "ssh") }} + - name: ssl-ca-directory + workspace: ssl-ca-directory +{{- end }} - name: qtodo-build-artifact runAfter: diff --git a/charts/supply-chain/templates/pipelinerun-qtodo.yaml b/charts/supply-chain/templates/pipelinerun-qtodo.yaml index e87cdaf4..3d6fe54d 100644 --- a/charts/supply-chain/templates/pipelinerun-qtodo.yaml +++ b/charts/supply-chain/templates/pipelinerun-qtodo.yaml @@ -107,6 +107,11 @@ spec: - name: registry-auth-config secret: secretName: {{ .Values.registry.authSecretName }} +{{- if and .Values.git.sslCABundle.enabled (ne (default "https" .Values.git.credentials.authType) "ssh") }} + - name: ssl-ca-directory + configMap: + name: {{ .Values.git.sslCABundle.configMapName }} +{{- end }} MANIFEST echo "PipelineRun created successfully." {{- end }} diff --git a/charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml b/charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml index ff1a0367..60bfc5e3 100644 --- a/charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml +++ b/charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml @@ -2,7 +2,7 @@ {{- $authType := .Values.git.credentials.authType | default "https" }} {{- $host := .Values.git.credentials.host }} --- -apiVersion: "external-secrets.io/v1beta1" +apiVersion: "external-secrets.io/v1" kind: ExternalSecret metadata: name: qtodo-git-credentials @@ -44,7 +44,7 @@ spec: .gitconfig: | [credential "{{ $host }}"] helper = store - .git-credentials: {{ printf "https://{{ .%s }}:{{ .%s | trim }}@%s" $userKey $passKey $hostBare | quote }} + .git-credentials: {{ printf "https://{{ .%s | trim }}:{{ .%s | trim }}@%s" $userKey $passKey $hostBare | quote }} data: - secretKey: {{ $userKey }} remoteRef: diff --git a/charts/supply-chain/templates/secrets/rhtas-client-secret.yaml b/charts/supply-chain/templates/secrets/rhtas-client-secret.yaml index 3cc78bb8..317e4c16 100644 --- a/charts/supply-chain/templates/secrets/rhtas-client-secret.yaml +++ b/charts/supply-chain/templates/secrets/rhtas-client-secret.yaml @@ -1,6 +1,6 @@ {{- if and .Values.rhtas.oidc.enabled (ne .Values.rhtas.oidc.clientSecretName "") }} --- -apiVersion: "external-secrets.io/v1beta1" +apiVersion: "external-secrets.io/v1" kind: ExternalSecret metadata: name: {{ .Values.rhtas.oidc.clientSecretName }} diff --git a/charts/supply-chain/values.yaml b/charts/supply-chain/values.yaml index 8cfb6260..d7e7cbc8 100644 --- a/charts/supply-chain/values.yaml +++ b/charts/supply-chain/values.yaml @@ -51,6 +51,13 @@ git: passwordKey: "password" sshPrivateKeyKey: "ssh-privatekey" knownHostsKey: "known_hosts" + # Corporate/custom CA bundle for HTTPS git clones from internal hosts. + # When enabled, the git-clone task mounts the CA ConfigMap as the + # ssl-ca-directory workspace so TLS verification succeeds against + # internal Git servers (e.g. GitLab behind a corporate CA). + sslCABundle: + enabled: false + configMapName: "ztvp-trusted-ca" # qtodo repository configuration qtodo: diff --git a/charts/ztvp-certificates/files/extract-certificates.sh.tpl b/charts/ztvp-certificates/files/extract-certificates.sh.tpl index c0edb8f8..d271ea09 100644 --- a/charts/ztvp-certificates/files/extract-certificates.sh.tpl +++ b/charts/ztvp-certificates/files/extract-certificates.sh.tpl @@ -22,6 +22,7 @@ log "ZTVP CA Certificate Extraction" log "===========================================" log "Auto-detect: {{ .Values.autoDetect }}" log "Custom CA: {{ .Values.customCA.secretRef.enabled }}" +log "Remote hosts: {{ len .Values.customCA.remoteHosts }}" log "Namespace: {{ .Values.global.namespace }}" log "ConfigMap: {{ .Values.configMapName }}" @@ -46,6 +47,30 @@ else fi {{- end }} +# =================================================================== +# PHASE 1.5: Extract CA chains from remote hosts (if configured) +# No authentication required -- CAs are part of the public TLS handshake. +# =================================================================== + +{{- if .Values.customCA.remoteHosts }} +REMOTE_HOST_COUNT=0 +{{- range $host := .Values.customCA.remoteHosts }} +log "Extracting CA chain from remote host: {{ $host }}:443" +REMOTE_CERTS=$(openssl s_client -connect {{ $host }}:443 -showcerts /dev/null \ + | awk '/BEGIN CERTIFICATE/,/END CERTIFICATE/') +if [[ -n "$REMOTE_CERTS" ]]; then + SAFE_NAME=$(echo "{{ $host }}" | tr '.:' '-') + echo "$REMOTE_CERTS" > "${TEMP_DIR}/remote-${SAFE_NAME}.crt" + REMOTE_HOST_COUNT=$((REMOTE_HOST_COUNT + 1)) + CUSTOM_CA_FOUND=true + log "OK: Extracted CA chain from {{ $host }}" +else + error "Failed to extract CA chain from {{ $host }}:443 (is the host reachable?)" +fi +{{- end }} +log "Extracted CA chains from $REMOTE_HOST_COUNT remote host(s)" +{{- end }} + # =================================================================== # PHASE 2: Extract Ingress CA (if auto-detect enabled) # =================================================================== @@ -298,6 +323,7 @@ metadata: ztvp.io/auto-detect: "{{ .Values.autoDetect }}" ztvp.io/custom-ca-enabled: "{{ .Values.customCA.secretRef.enabled }}" ztvp.io/custom-ca-found: "${CUSTOM_CA_FOUND}" + ztvp.io/remote-hosts: "{{ len .Values.customCA.remoteHosts }}" ztvp.io/ingress-ca-found: "${INGRESS_CA_FOUND}" ztvp.io/service-ca-found: "${SERVICE_CA_FOUND}" ztvp.io/cluster-ca-found: "${CLUSTER_CA_FOUND}" diff --git a/charts/ztvp-certificates/values.yaml b/charts/ztvp-certificates/values.yaml index cd3d57c0..fcb5e9be 100644 --- a/charts/ztvp-certificates/values.yaml +++ b/charts/ztvp-certificates/values.yaml @@ -38,6 +38,18 @@ customCA: # oc create secret generic --from-file=ca.crt=/path/to/cert.crt -n openshift-config # Configure via overrides/values-ztvp-certificates.yaml (using extraValueFiles) additionalCertificates: [] + + # Remote host CA extraction: fetch TLS CA chains directly from remote hosts. + # No authentication needed -- CA certificates are part of the public TLS handshake. + # The extraction Job runs openssl s_client against each host on port 443 and + # saves the full certificate chain. Useful for internal Git servers, registries, + # or any service behind a corporate CA. + # The CronJob keeps the extracted CAs fresh automatically. + remoteHosts: [] + # Example: + # remoteHosts: + # - gitlab.cee.redhat.com + # - registry.internal.example.com # Example: # additionalCertificates: # - name: corporate-root-ca diff --git a/docs/private-repos.md b/docs/private-repos.md index aabe35ca..553614a8 100644 --- a/docs/private-repos.md +++ b/docs/private-repos.md @@ -246,9 +246,10 @@ Expected output: `Synced` (or `OutOfSync` if you have uncommitted changes). in a container without your Git host's fingerprint in known_hosts. * **HTTPS: "x509: certificate signed by unknown authority"** -- This - affects internal/self-hosted GitLab instances whose TLS certificates are - signed by a corporate CA. GitHub and public GitLab (`gitlab.com`) use - publicly trusted CAs and do not require this step. + affects internal/self-hosted Git servers (e.g. Gitea, GitLab) whose TLS + certificates are signed by a custom or corporate CA. GitHub and public + GitLab (`gitlab.com`) use publicly trusted CAs and do not require this + step. The corporate CA must be in the cluster trust store **before** install because the VP operator needs it to clone the repository. Add the internal CA @@ -263,6 +264,18 @@ oc patch proxy/cluster --type=merge \ Wait a few minutes for operator pods to restart with the updated bundle. + If the custom CA is added **after** the pattern is already deployed, the + `trusted-ca-bundle` ConfigMap will be updated by the cluster CA injector, + but the ArgoCD repo-server will **not** pick it up automatically. The + repo-server uses an init container (`fetch-ca`) that copies the CA bundle + into an `emptyDir` volume at pod startup; this only runs once. Restart + the repo-server to load the updated bundle: + +```shell +oc rollout restart deployment/vp-gitops-repo-server -n vp-gitops +oc rollout status deployment/vp-gitops-repo-server -n vp-gitops +``` + > [!NOTE] > After the pattern deploys, the `ztvp-certificates` chart automatically > merges your `custom-ca` content into its managed `ztvp-proxy-ca` diff --git a/docs/supply-chain.md b/docs/supply-chain.md index 4e653893..3afad556 100644 --- a/docs/supply-chain.md +++ b/docs/supply-chain.md @@ -18,6 +18,11 @@ In this project, we used the [qtodo](https://github.com/validatedpatterns-demos/ > * `applications.noobaa-mcg` — NooBaa MCG object storage (required by Quay and RHTPA) > * `subscriptions.odf` and `subscriptions.quay-operator` and their namespace entries > +> Additionally, uncomment the following Vault JWT roles in `overrides/values-vault-jwt.yaml` so that RHTPA and the pipeline ServiceAccount can authenticate to Vault via SPIFFE: +> +> * `rhtpa` role — allows RHTPA to read its OIDC credentials from Vault +> * `supply-chain` role — allows the Tekton pipeline ServiceAccount to read git credentials, registry credentials, and RHTPA OIDC secrets from Vault +> > If you prefer to use an external image registry instead of Quay, skip the Quay and NooBaa sections and set the registry parameters in the `supply-chain` application overrides accordingly. ## Components @@ -267,6 +272,7 @@ Once the supply-chain application has synced in ArgoCD, start the pipeline using * For **git-auth**, the binding depends on the authentication mode (see [How it works](#how-it-works) for details): * **HTTPS mode**: select `Secret` and the name of the secret is `qtodo-git-credentials`. The `git-clone` ClusterTask's `basic-auth` workspace requires the secret to be provided explicitly; ServiceAccount-level credential injection alone is not sufficient for HTTPS. * **SSH mode**: leave **git-auth** unbound (empty). SSH credentials are injected automatically via the `pipeline` ServiceAccount. Binding the workspace directly causes the `git-clone` ClusterTask's `prepare.sh` to run a recursive `chmod` on the copied secret volume, which fails on the read-only Kubernetes projected volume symlinks. + * For **ssl-ca-directory** (HTTPS mode with internal Git hosts only): if `git.sslCABundle.enabled` is `true`, select `ConfigMap` and the name is `ztvp-trusted-ca`. This is only needed when cloning over HTTPS from a Git server behind a corporate or self-signed CA (see [Corporate CA Trust for Internal Git Hosts](#corporate-ca-trust-for-internal-git-hosts)). 5. Press **Start** to finish and run the pipeline. @@ -299,6 +305,10 @@ spec: - name: git-auth secret: secretName: qtodo-git-credentials + # Add this workspace when git.sslCABundle.enabled is true (internal Git hosts): + # - name: ssl-ca-directory + # configMap: + # name: ztvp-trusted-ca ``` **SSH mode** (leave `git-auth` unbound): @@ -476,6 +486,41 @@ When `git.credentials.enabled` is `true`: * **SSH mode**: the `git-auth` workspace must be left **unbound**. SSH credentials are injected automatically via the ServiceAccount. Binding the workspace triggers the `git-clone` ClusterTask's `prepare.sh`, which runs a recursive `chmod` on the copied secret volume; this fails on the read-only Kubernetes projected volume symlinks and aborts the step. * The Vault policy `hub-supply-chain-jwt-secret` grants read access to `secret/data/hub/supply-chain/*` for the pipeline's SPIFFE identity. +> [!NOTE] +> If your internal Git server also uses a corporate or self-signed CA, see [Corporate CA Trust for Internal Git Hosts](#corporate-ca-trust-for-internal-git-hosts) to configure TLS trust. + +### Corporate CA Trust for Internal Git Hosts + +This section applies whenever the pipeline clones from a Git server whose TLS certificate is signed by a corporate or self-signed CA, regardless of whether the repository is private. It is only relevant for HTTPS clones; SSH connections do not use TLS certificate verification. + +> [!NOTE] +> Public Git hosts (github.com, gitlab.com) use publicly trusted certificates and do not require this. If the repository is also private, combine these settings with the [Protected Repositories](#protected-repositories) configuration above. + +When a repository is hosted on an internal Git server (e.g. GitLab behind a corporate CA), the `git-clone` task will fail with `SSL certificate problem: self-signed certificate in certificate chain` because the pod does not trust the corporate CA. + +The `ztvp-certificates` chart already extracts and distributes the cluster's CA bundle (ingress, service, and any custom/corporate CAs). When the `supply-chain` feature is enabled, the `ztvp-trusted-ca` ConfigMap is automatically distributed to the pipeline namespace (`layered-zero-trust-hub`) via ACM policy. + +To make the `git-clone` task use this CA bundle, enable the SSL CA bundle mount in the `supply-chain` application overrides: + +```yaml +- name: git.sslCABundle.enabled + value: "true" +``` + +This binds the `ztvp-trusted-ca` ConfigMap as the `ssl-ca-directory` workspace on the `git-clone` task and sets the `CRT_FILENAME` parameter to `tls-ca-bundle.pem` (matching the key in the ConfigMap). The upstream `git-clone` ClusterTask uses this file to set `GIT_SSL_CAPATH`, so TLS verification succeeds against internal Git servers. + +The corporate CA must be included in the `ztvp-trusted-ca` bundle. The easiest way is to use **automatic remote host extraction** -- add the Git host to `customCA.remoteHosts` in the `ztvp-certificates` overrides: + +```yaml +# ztvp-certificates overrides in values-hub.yaml +- name: customCA.remoteHosts[0] + value: "gitlab.internal.example.com" +``` + +The `ztvp-certificates` extraction Job will connect to the host on port 443, extract the full CA chain from the TLS handshake (no authentication needed), and merge it into the CA bundle. The CronJob keeps it fresh automatically. + +Alternatively, you can provide the CA certificate manually via `customCA.secretRef` or `customCA.additionalCertificates`. See the [ztvp-certificates documentation](./ztvp-certificates.md) for details. + ### Init task (pre-flight image check) The pipeline includes an `init` task that runs before `git-clone`. It uses `skopeo inspect` to check whether the target image already exists in the registry. If the image exists (and `rebuild` is not set to `"true"`), the pipeline skips the build. This avoids unnecessary rebuilds and is modeled after the [RHTAP sample pipelines](https://github.com/konflux-ci/build-definitions). diff --git a/scripts/features/protected-repos.yaml b/scripts/features/protected-repos.yaml index 8674e382..de238f2a 100644 --- a/scripts/features/protected-repos.yaml +++ b/scripts/features/protected-repos.yaml @@ -6,6 +6,10 @@ # https - basic-auth via .git-credentials (username + PAT) # ssh - SSH key pair (ssh-privatekey + known_hosts) # +# For internal Git hosts (corporate CA), the generator auto-enables +# customCA.remoteHosts and git.sslCABundle so the pipeline trusts the +# server's TLS certificate without manual CA provisioning. +# # Requires the git-credentials secret in values-secret.yaml.template # to be uncommented and populated with the appropriate credentials. clusterGroup: @@ -20,5 +24,11 @@ clusterGroup: value: "REPLACE_WITH_GIT_HOST" - name: git.credentials.vaultPath value: "secret/data/hub/supply-chain/git-credentials" + - name: git.sslCABundle.enabled + value: "REPLACE_WITH_SSL_CA_ENABLED" - name: qtodo.repository value: "REPLACE_WITH_GIT_REPO_URL" + ztvp-certificates: + overrides: + - name: "customCA.remoteHosts[0]" + value: "REPLACE_WITH_GIT_HOSTNAME" diff --git a/scripts/features/supply-chain.yaml b/scripts/features/supply-chain.yaml index fdc7dbe8..e5743dc3 100644 --- a/scripts/features/supply-chain.yaml +++ b/scripts/features/supply-chain.yaml @@ -26,6 +26,12 @@ clusterGroup: - /spec/tasks merge_into_applications: + ztvp-certificates: + overrides: + - name: "distribution.targetNamespaces[0]" + value: "qtodo" + - name: "distribution.targetNamespaces[1]" + value: "{{ $.Values.global.pattern }}-hub" vault: jwt: roles: diff --git a/scripts/gen-feature-variants.py b/scripts/gen-feature-variants.py index ed6bd320..58d98485 100755 --- a/scripts/gen-feature-variants.py +++ b/scripts/gen-feature-variants.py @@ -344,39 +344,87 @@ def _substitute_repository_placeholders(base, org=None, image_name=None): GIT_REPO_PLACEHOLDER = "REPLACE_WITH_GIT_REPO_URL" GIT_HOST_PLACEHOLDER = "REPLACE_WITH_GIT_HOST" GIT_AUTH_TYPE_PLACEHOLDER = "REPLACE_WITH_GIT_AUTH_TYPE" +SSL_CA_ENABLED_PLACEHOLDER = "REPLACE_WITH_SSL_CA_ENABLED" +GIT_HOSTNAME_PLACEHOLDER = "REPLACE_WITH_GIT_HOSTNAME" + +PUBLIC_GIT_HOSTS = {"github.com", "gitlab.com", "bitbucket.org"} SSH_URL_RE = re.compile(r"^[\w.-]+@([\w.-]+):") def _parse_git_repo_url(git_repo_url): - """Derive (host, auth_type) from a Git repository URL. + """Derive (host, auth_type, hostname) from a Git repository URL. - HTTPS URLs -> host = "https://github.com", auth_type = "https" - SSH URLs -> host = "github.com", auth_type = "ssh" + HTTPS URLs -> host = "https://github.com", auth_type = "https", hostname = "github.com" + SSH URLs -> host = "github.com", auth_type = "ssh", hostname = "github.com" """ m = SSH_URL_RE.match(git_repo_url) if m: - return m.group(1), "ssh" + hostname = m.group(1) + if not hostname: + raise ValueError(f"Invalid SSH URL: {git_repo_url}") + return hostname, "ssh", hostname parsed = urlparse(git_repo_url) + if not parsed.hostname: + raise ValueError(f"Invalid git URL (no hostname): {git_repo_url}") scheme = parsed.scheme or "https" hostname = parsed.hostname or "" - return f"{scheme}://{hostname}", "https" + return f"{scheme}://{hostname}", "https", hostname -def _substitute_git_overrides(base, git_repo_url, git_host, git_auth_type): - """Replace git-related placeholders in supply-chain overrides.""" +def _substitute_git_overrides( + base, git_repo_url, git_host, git_auth_type, git_hostname +): + """Replace git-related placeholders in supply-chain and ztvp-certificates overrides.""" apps = base.get("clusterGroup", {}).get("applications", {}) + is_internal = git_hostname not in PUBLIC_GIT_HOSTS + sc = apps.get("supply-chain", {}) - placeholder_map = { + sc_placeholder_map = { "qtodo.repository": (GIT_REPO_PLACEHOLDER, git_repo_url), "git.credentials.host": (GIT_HOST_PLACEHOLDER, git_host), "git.credentials.authType": (GIT_AUTH_TYPE_PLACEHOLDER, git_auth_type), + "git.sslCABundle.enabled": ( + SSL_CA_ENABLED_PLACEHOLDER, + "true" if is_internal else "false", + ), } - for override in sc.get("overrides", []): - entry = placeholder_map.get(override.get("name")) + sc_overrides = sc.get("overrides", []) + for override in sc_overrides: + entry = sc_placeholder_map.get(override.get("name")) if entry and str(override.get("value")) == entry[0]: override["value"] = entry[1] + # Remove git.sslCABundle.enabled override when false (public hosts) + if not is_internal: + sc_overrides[:] = [ + o + for o in sc_overrides + if not ( + o.get("name") == "git.sslCABundle.enabled" and o.get("value") == "false" + ) + ] + + certs = apps.get("ztvp-certificates", {}) + certs_overrides = certs.get("overrides", []) + if is_internal: + for override in certs_overrides: + if ( + override.get("name") == "customCA.remoteHosts[0]" + and str(override.get("value")) == GIT_HOSTNAME_PLACEHOLDER + ): + override["value"] = git_hostname + else: + # Remove the remoteHosts placeholder for public hosts + certs_overrides[:] = [ + o + for o in certs_overrides + if not ( + o.get("name") == "customCA.remoteHosts[0]" + and str(o.get("value")) == GIT_HOSTNAME_PLACEHOLDER + ) + ] + def _update_vault_jwt_override_file(override_file_path, new_roles): """Update the vault JWT override file with new roles from feature fragments. @@ -475,8 +523,10 @@ def generate_variant( _substitute_repository_placeholders(base, org=org, image_name=image_name) if git_repo_url: - git_host, git_auth_type = _parse_git_repo_url(git_repo_url) - _substitute_git_overrides(base, git_repo_url, git_host, git_auth_type) + git_host, git_auth_type, git_hostname = _parse_git_repo_url(git_repo_url) + _substitute_git_overrides( + base, git_repo_url, git_host, git_auth_type, git_hostname + ) validate_output(base) cg = base.get("clusterGroup") diff --git a/values-hub.yaml b/values-hub.yaml index 240cafa2..c346a7d6 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -561,12 +561,17 @@ clusterGroup: # # Protected repository credentials (Vault + ExternalSecret) # # - name: git.credentials.enabled # # value: "true" + # # - name: git.credentials.authType + # # value: "https" # or "ssh" # # - name: git.credentials.host # # value: "https://github.com" # # - name: git.credentials.vaultPath # # value: "secret/data/hub/supply-chain/git-credentials" # # - name: qtodo.repository # # value: "https://github.com/your-org/qtodo.git" + # # Corporate CA trust for internal Git hosts (HTTPS only) + # # - name: git.sslCABundle.enabled + # # value: "true" # # ACS Central Services acs-central: