From ca4ab0ec92d5df0e68541708014eb865d018b463 Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Wed, 3 Dec 2025 15:03:20 -0500 Subject: [PATCH 1/5] coco: initial integration with ztvp This adds initial integration for Confidential Containers and Trustee Operators as a separated clustergroup. Co-authored-by: Chris Butler Signed-off-by: Beraldo Leal --- ansible/azure-nat-gateway.yaml | 88 ++++++ ansible/azure-requirements.txt | 5 + ansible/init-data-gzipper.yaml | 70 +++++ ansible/initdata-default.toml.tpl | 78 +++++ ansible/install-deps.yaml | 18 ++ overrides/values-Azure.yaml | 8 + overrides/values-sandbox.yaml | 8 + overrides/values-trustee.yaml | 24 ++ values-coco-dev.yaml | 462 ++++++++++++++++++++++++++++++ values-global.yaml | 2 +- 10 files changed, 762 insertions(+), 1 deletion(-) create mode 100644 ansible/azure-nat-gateway.yaml create mode 100644 ansible/azure-requirements.txt create mode 100644 ansible/init-data-gzipper.yaml create mode 100644 ansible/initdata-default.toml.tpl create mode 100644 ansible/install-deps.yaml create mode 100644 overrides/values-Azure.yaml create mode 100644 overrides/values-sandbox.yaml create mode 100644 overrides/values-trustee.yaml create mode 100644 values-coco-dev.yaml diff --git a/ansible/azure-nat-gateway.yaml b/ansible/azure-nat-gateway.yaml new file mode 100644 index 00000000..128c4e50 --- /dev/null +++ b/ansible/azure-nat-gateway.yaml @@ -0,0 +1,88 @@ +--- + +- name: Configure Azure NAT Gateway + become: false + connection: local + hosts: localhost + gather_facts: false + vars: + kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" + resource_prefix: "coco" + tasks: + - name: Get Azure credentials + kubernetes.core.k8s_info: + kind: Secret + namespace: openshift-cloud-controller-manager + name: azure-cloud-credentials + register: azure_credentials + retries: 20 + delay: 5 + + - name: Get Azure configuration + kubernetes.core.k8s_info: + kind: ConfigMap + namespace: openshift-cloud-controller-manager + name: cloud-conf + register: azure_cloud_conf + retries: 20 + delay: 5 + + - name: Set facts + ansible.builtin.set_fact: + azure_subscription_id: "{{ (azure_cloud_conf.resources[0]['data']['cloud.conf'] | from_json)['subscriptionId'] }}" + azure_tenant_id: "{{ (azure_cloud_conf.resources[0]['data']['cloud.conf'] | from_json)['tenantId'] }}" + azure_resource_group: "{{ (azure_cloud_conf.resources[0]['data']['cloud.conf'] | from_json)['vnetResourceGroup'] }}" + azure_client_id: "{{ azure_credentials.resources[0]['data']['azure_client_id'] | b64decode }}" + azure_client_secret: "{{ azure_credentials.resources[0]['data']['azure_client_secret'] | b64decode }}" + azure_vnet: "{{ (azure_cloud_conf.resources[0]['data']['cloud.conf'] | from_json)['vnetName'] }}" + azure_subnet: "{{ (azure_cloud_conf.resources[0]['data']['cloud.conf'] | from_json)['subnetName'] }}" + coco_public_ip_name: "{{ resource_prefix }}-pip" + coco_nat_gateway_name: "{{ resource_prefix }}-nat-gateway" + no_log: true + + - name: Create Public IP for NAT Gateway + azure.azcollection.azure_rm_publicipaddress: + subscription_id: "{{ azure_subscription_id }}" + tenant: "{{ azure_tenant_id }}" + client_id: "{{ azure_client_id }}" + secret: "{{ azure_client_secret }}" + resource_group: "{{ azure_resource_group }}" + name: "{{ coco_public_ip_name }}" + sku: "standard" + allocation_method: "static" + + - name: Retrieve Public IP for NAT Gateway + azure.azcollection.azure_rm_publicipaddress_info: + subscription_id: "{{ azure_subscription_id }}" + tenant: "{{ azure_tenant_id }}" + client_id: "{{ azure_client_id }}" + secret: "{{ azure_client_secret }}" + resource_group: "{{ azure_resource_group }}" + name: "{{ coco_public_ip_name }}" + register: coco_gw_public_ip + + - name: Create NAT Gateway + azure.azcollection.azure_rm_natgateway: + subscription_id: "{{ azure_subscription_id }}" + tenant: "{{ azure_tenant_id }}" + client_id: "{{ azure_client_id }}" + secret: "{{ azure_client_secret }}" + resource_group: "{{ azure_resource_group }}" + name: "{{ coco_nat_gateway_name }}" + idle_timeout_in_minutes: 10 + sku: + name: standard + public_ip_addresses: + - "{{ coco_gw_public_ip.publicipaddresses[0].id }}" + register: coco_natgw + + - name: Update the worker subnet to associate NAT gateway + azure.azcollection.azure_rm_subnet: + subscription_id: "{{ azure_subscription_id }}" + tenant: "{{ azure_tenant_id }}" + client_id: "{{ azure_client_id }}" + secret: "{{ azure_client_secret }}" + resource_group: "{{ azure_resource_group }}" + name: "{{ azure_subnet }}" + virtual_network_name: "{{ azure_vnet }}" + nat_gateway: "{{ coco_nat_gateway_name }}" diff --git a/ansible/azure-requirements.txt b/ansible/azure-requirements.txt new file mode 100644 index 00000000..8560fafd --- /dev/null +++ b/ansible/azure-requirements.txt @@ -0,0 +1,5 @@ +azure-identity>=1.19.0 +azure-mgmt-core>=1.4.0 +azure-mgmt-managementgroups>=1.0.0 +azure-mgmt-network>=25.0.0 +azure-mgmt-resource>=23.0.0 diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml new file mode 100644 index 00000000..209a226f --- /dev/null +++ b/ansible/init-data-gzipper.yaml @@ -0,0 +1,70 @@ +- name: Gzip initdata + become: false + connection: local + hosts: localhost + gather_facts: false + vars: + kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" + cluster_platform: "{{ global.clusterPlatform | default('none') | lower }}" + hub_domain: "{{ global.hubClusterDomain | default('none') | lower}}" + image_security_policy: "{{ coco.imageSecurityPolicy | default('insecure') }}" + template_src: "initdata-default.toml.tpl" + tasks: + - name: Create temporary working directory + ansible.builtin.tempfile: + state: directory + suffix: initdata + register: tmpdir + - name: Read KBS TLS secret from Kubernetes + kubernetes.core.k8s_info: + kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" + api_version: v1 + kind: Secret + name: kbs-tls-self-signed + namespace: imperative + register: kbs_secret_result + + - name: Extract and decode certificate from secret + ansible.builtin.set_fact: + trustee_cert: "{{ kbs_secret_result.resources[0].data['tls.crt'] | b64decode }}" + when: kbs_secret_result.resources | length > 0 + + - name: Fail if certificate not found + ansible.builtin.fail: + msg: "KBS TLS certificate not found in secret 'kbs-tls-self-signed' in namespace 'imperative'" + when: kbs_secret_result.resources | length == 0 + + - name: Define temp file paths + ansible.builtin.set_fact: + rendered_path: "{{ tmpdir.path }}/rendered.toml" + gz_path: "{{ tmpdir.path }}/rendered.toml.gz" + + - name: Render template to temp file + ansible.builtin.template: + src: "{{ template_src }}" + dest: "{{ rendered_path }}" + mode: "0600" + + + - name: Gzip the rendered content + ansible.builtin.shell: | + gzip -c "{{ rendered_path }}" > "{{ gz_path }}" + changed_when: true + + - name: Read gzip as base64 + ansible.builtin.slurp: + path: "{{ gz_path }}" + register: gz_slurped + + - name: Create/update ConfigMap with gzipped+base64 content + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig | default(omit) }}" + state: present + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: "initdata" + namespace: "imperative" + data: + INITDATA: "{{ gz_slurped.content }}" diff --git a/ansible/initdata-default.toml.tpl b/ansible/initdata-default.toml.tpl new file mode 100644 index 00000000..dd2b9977 --- /dev/null +++ b/ansible/initdata-default.toml.tpl @@ -0,0 +1,78 @@ +# NOTE: PodVMs run in separate VMs outside the cluster network, so they cannot +# resolve cluster-internal service DNS (*.svc.cluster.local). Therefore, we must +# use the external KBS route even for same-cluster deployments. +# For multi-cluster deployments, this also points to the trusted cluster's KBS. + +algorithm = "sha384" +version = "0.1.0" + +[data] +"aa.toml" = ''' +[token_configs] +[token_configs.coco_as] +url = "https://kbs.{{ hub_domain }}" + +[token_configs.kbs] +url = "https://kbs.{{ hub_domain }}" +cert = """ +{{ trustee_cert }} +""" +''' + +"cdh.toml" = ''' +socket = 'unix:///run/confidential-containers/cdh.sock' +credentials = [] + +[kbc] +name = "cc_kbc" +url = "https://kbs.{{ hub_domain }}" +kbs_cert = """ +{{ trustee_cert }} +""" + +[image] +# Container image signature verification policy +# Options: insecure, reject, signed (configured via coco.imageSecurityPolicy in values) +image_security_policy_uri = "kbs:///default/security-policy/{{ image_security_policy }}" +''' + +"policy.rego" = ''' +package agent_policy + +default AddARPNeighborsRequest := true +default AddSwapRequest := true +default CloseStdinRequest := true +default CopyFileRequest := true +default CreateContainerRequest := true +default CreateSandboxRequest := true +default DestroySandboxRequest := true +default ExecProcessRequest := true # TEMPORARY: enabled for debugging sealed secrets issue +default GetMetricsRequest := true +default GetOOMEventRequest := true +default GuestDetailsRequest := true +default ListInterfacesRequest := true +default ListRoutesRequest := true +default MemHotplugByProbeRequest := true +default OnlineCPUMemRequest := true +default PauseContainerRequest := true +default PullImageRequest := true +default ReadStreamRequest := true # TEMPORARY: enabled for debugging sealed secrets issue +default RemoveContainerRequest := true +default RemoveStaleVirtiofsShareMountsRequest := true +default ReseedRandomDevRequest := true +default ResumeContainerRequest := true +default SetGuestDateTimeRequest := true +default SetPolicyRequest := true +default SignalProcessRequest := true +default StartContainerRequest := true +default StartTracingRequest := true +default StatsContainerRequest := true +default StopTracingRequest := true +default TtyWinResizeRequest := true +default UpdateContainerRequest := true +default UpdateEphemeralMountsRequest := true +default UpdateInterfaceRequest := true +default UpdateRoutesRequest := true +default WaitProcessRequest := true +default WriteStreamRequest := true +''' \ No newline at end of file diff --git a/ansible/install-deps.yaml b/ansible/install-deps.yaml new file mode 100644 index 00000000..f3fd5417 --- /dev/null +++ b/ansible/install-deps.yaml @@ -0,0 +1,18 @@ +- name: Retrieve Credentials for AAP on OpenShift + become: false + connection: local + hosts: localhost + gather_facts: false + tasks: + - name: Ensure collection is installed + community.general.ansible_galaxy_install: + type: collection + name: azure.azcollection + - name: Ensure community.crypto collection is installed + community.general.ansible_galaxy_install: + type: collection + name: community.crypto + - name: Install Azure SDK + ansible.builtin.pip: + requirements: "~/.ansible/collections/ansible_collections/azure/azcollection/requirements.txt" + extra_args: --user diff --git a/overrides/values-Azure.yaml b/overrides/values-Azure.yaml new file mode 100644 index 00000000..b2a15e5e --- /dev/null +++ b/overrides/values-Azure.yaml @@ -0,0 +1,8 @@ +# Azure platform-specific configuration + +# CoCo confidential computing configuration for Azure +global: + coco: + azure: + defaultVMFlavour: "Standard_DC2eds_v5" + VMFlavours: "Standard_DC2eds_v5,Standard_DC4eds_v5,Standard_DC8eds_v5,Standard_DC16eds_v5" diff --git a/overrides/values-sandbox.yaml b/overrides/values-sandbox.yaml new file mode 100644 index 00000000..cf7cf984 --- /dev/null +++ b/overrides/values-sandbox.yaml @@ -0,0 +1,8 @@ +# Override the default values for the sandboxed-containers chart +# Configures External Secrets Operator integration for Azure SSH keys + +# Secret store configuration for External Secrets Operator +# Points to the ClusterSecretStore that knows how to connect to Vault +secretStore: + name: vault-backend + kind: ClusterSecretStore diff --git a/overrides/values-trustee.yaml b/overrides/values-trustee.yaml new file mode 100644 index 00000000..706ba898 --- /dev/null +++ b/overrides/values-trustee.yaml @@ -0,0 +1,24 @@ +# Override the default values for the trustee chart +# This lists the secret resources that are uploaded to your chosen ESO backend (default: Vault). +# It does not contain the secrets themselves, only references to Vault paths. +# +# NOTE: When adding new CoCo workloads to coco.workloads, you must also add +# corresponding spire-cert-{workload} and spire-key-{workload} entries here + +# Secret store configuration for External Secrets Operator +# Points to the ClusterSecretStore that knows how to connect to Vault +secretStore: + name: vault-backend + kind: ClusterSecretStore + +kbs: + secretResources: + - name: "kbsres1" + key: "secret/data/hub/kbsres1" + - name: "passphrase" + key: "secret/data/hub/passphrase" + # SPIRE x509pop certificates per workload type + - name: "spire-cert-qtodo" + key: "secret/data/pushsecrets/spire-cert-qtodo" + - name: "spire-key-qtodo" + key: "secret/data/pushsecrets/spire-key-qtodo" diff --git a/values-coco-dev.yaml b/values-coco-dev.yaml new file mode 100644 index 00000000..d6a2d974 --- /dev/null +++ b/values-coco-dev.yaml @@ -0,0 +1,462 @@ +# CoCo Development Configuration (Single Cluster) +# Combines ZTVP (SPIRE/Keycloak/Vault) with CoCo (Trustee/Sandboxed Containers) +# All components deployed on single cluster for development/testing +# +# WARNING: NOT RECOMMENDED FOR PRODUCTION +# This configuration runs Trustee/KBS on the same untrusted cluster as worker nodes. +# Production deployments should use multi-cluster setup with Trustee on a trusted cluster. + +# This spire config is required to fix a bug in the zero-trust-workload-identity-manager operator +spire: + oidcDiscoveryProvider: + ingress: + enabled: true + annotations: + route.openshift.io/termination: reencrypt + route.openshift.io/destination-ca-certificate-secret: spire-bundle + +# Moved outside of clusterGroup to avoid validation errors +# CoCo workload types for x509pop certificate generation +# One SPIRE agent certificate is generated per workload type +# All pods of the same workload type (e.g., all qtodo pods) share the same agent certificate +# NOTE: KBS resource policy should be added to enforce workload-specific certificate access +coco: + # Container image signature verification policy + # Options: insecure (accept all), reject (reject all), signed (require cosign signature) + # See values-secret.yaml.template for policy definitions + imageSecurityPolicy: insecure + workloads: + - name: "qtodo" + namespace: "qtodo" + +clusterGroup: + name: coco-dev + isHubCluster: true + namespaces: + - open-cluster-management + - vault + - qtodo + - golang-external-secrets + - keycloak-system: + operatorGroup: true + targetNamespace: keycloak-system + - cert-manager + - cert-manager-operator: + operatorGroup: true + targetNamespace: cert-manager-operator + # Layer 1: Quay Registry (for container image storage and signing) + # COMMENTED OUT: Uncomment to enable integrated Quay registry + # - openshift-storage: + # operatorGroup: true + # targetNamespace: openshift-storage + # annotations: + # openshift.io/cluster-monitoring: "true" + # argocd.argoproj.io/sync-wave: "-5" # Propagated to OperatorGroup by framework + # - quay-enterprise: + # annotations: + # argocd.argoproj.io/sync-wave: "1" # Create before NooBaa and all Quay components + # labels: + # openshift.io/cluster-monitoring: "true" + # RHTAS namespace (required when RHTAS application is enabled) + # COMMENTED OUT: Uncomment to enable RHTAS with SPIFFE signing + # - trusted-artifact-signer: + # annotations: + # argocd.argoproj.io/sync-wave: "1" # Auto-created by RHTAS operator + # labels: + # openshift.io/cluster-monitoring: "true" + - zero-trust-workload-identity-manager: + operatorGroup: true + targetNamespace: zero-trust-workload-identity-manager + - openshift-compliance: + operatorGroup: true + targetNamespace: openshift-compliance + annotations: + openshift.io/cluster-monitoring: "true" + # CoCo namespaces + - openshift-sandboxed-containers-operator + - trustee-operator-system + subscriptions: + acm: + name: advanced-cluster-management + namespace: open-cluster-management + channel: release-2.14 + catalogSource: redhat-operators + cert-manager: + name: openshift-cert-manager-operator + namespace: cert-manager-operator + channel: stable-v1 + catalogSource: redhat-marketplace + rhbk: + name: rhbk-operator + namespace: keycloak-system + channel: stable-v26.2 + catalogSource: redhat-marketplace + zero-trust-workload-identity-manager: + name: openshift-zero-trust-workload-identity-manager + namespace: zero-trust-workload-identity-manager + channel: tech-preview-v0.2 + catalogSource: redhat-marketplace + compliance-operator: + name: compliance-operator + namespace: openshift-compliance + channel: stable + catalogSource: redhat-marketplace + config: + nodeSelector: + node-role.kubernetes.io/worker: "" + # CoCo subscriptions + sandboxed: + name: sandboxed-containers-operator + namespace: openshift-sandboxed-containers-operator + source: redhat-operators + channel: stable + installPlanApproval: Manual + csv: sandboxed-containers-operator.v1.10.3 + trustee: + name: trustee-operator + namespace: trustee-operator-system + source: redhat-operators + channel: stable + installPlanApproval: Manual + csv: trustee-operator.v0.4.2 + # Storage and Registry operator subscriptions + # COMMENTED OUT: Uncomment to enable integrated Quay registry + # ODF provides object storage backend (NooBaa) for Quay and RHTPA + # odf: + # name: odf-operator + # namespace: openshift-storage + # channel: stable-4.19 + # annotations: + # argocd.argoproj.io/sync-wave: "-4" # Install after OperatorGroup (-5) + # quay-operator: + # name: quay-operator + # namespace: openshift-operators + # channel: stable-3.15 + # annotations: + # argocd.argoproj.io/sync-wave: "-3" # Install after ODF operator + # RHTAS operator subscription (required when RHTAS application is enabled) + # COMMENTED OUT: Uncomment to enable RHTAS with SPIFFE integration + # rhtas-operator: + # name: rhtas-operator + # namespace: openshift-operators + # channel: stable + # annotations: + # argocd.argoproj.io/sync-wave: "-2" # Install after Quay operator, before applications + # catalogSource: redhat-operators + projects: + - hub + # Explicitly mention the cluster-state based overrides we plan to use for this pattern. + # We can use self-referential variables because the chart calls the tpl function with these variables defined + sharedValueFiles: + - '/overrides/values-{{ $.Values.global.clusterPlatform }}.yaml' + # sharedValueFiles is a flexible mechanism that will add the listed valuefiles to every app defined in the + # applications section. We intend this to supplement and possibly even replace previous "magic" mechanisms, though + # we do not at present have a target date for removal. + # + # To replicate the "classic" magic include structure, the clusterGroup would need all of these + # sharedValueFiles, in this order: + # - '/overrides/values-{{ $.Values.global.clusterPlatform }}.yaml' + # - '/overrides/values-{{ $.Values.global.clusterPlatform }}-{{ $.Values.global.clusterVersion }}.yaml' + # - '/overrides/values-{{ $.Values.global.clusterPlatform }}-{{ $.Values.clusterGroup.name }}.yaml' + # - '/overrides/values-{{ $.Values.global.clusterVersion }}-{{ $.Values.clusterGroup.name }}.yaml" + # - '/overrides/values-{{ $.Values.global.localClusterName }}.yaml' + + # This kind of variable substitution will work with any of the variables the Validated Patterns operator knows + # about and sets, so this is also possible, for example: + # - '/overrides/values-{{ $.Values.global.hubClusterDomain }}.yaml' + # - '/overrides/values-{{ $.Values.global.localClusterDomain }}.yaml' + applications: + acm: + name: acm + namespace: open-cluster-management + project: hub + chart: acm + chartVersion: 0.1.* + ignoreDifferences: + - group: internal.open-cluster-management.io + kind: ManagedClusterInfo + jsonPointers: + - /spec/loggingCA + # We override the secret store because we are not provisioning clusters + overrides: + - name: global.secretStore.backend + value: none + acm-managed-clusters: + name: acm-managed-clusters + project: hub + path: charts/acm-managed-clusters + ignoreDifferences: + - group: cluster.open-cluster-management.io + kind: ManagedCluster + jsonPointers: + - /metadata/labels/cloud + - /metadata/labels/vendor + compliance-scanning: + name: compliance-scanning + namespace: openshift-compliance + annotations: + argocd.argoproj.io/sync-wave: '-30' + project: hub + path: charts/compliance-scanning + vault: + name: vault + namespace: vault + project: hub + chart: hashicorp-vault + chartVersion: 0.1.* + jwt: + enabled: true + oidcDiscoveryUrl: https://spire-spiffe-oidc-discovery-provider.zero-trust-workload-identity-manager.svc.cluster.local + oidcDiscoveryCa: /run/secrets/kubernetes.io/serviceaccount/service-ca.crt + defaultRole: qtodo + roles: + - name: qtodo + audience: qtodo + subject: spiffe://apps.{{ $.Values.global.clusterDomain }}/ns/qtodo/sa/qtodo + policies: + - global-secret + # Shared Object Storage Backend + # COMMENTED OUT: Uncomment to enable integrated Quay registry + # NooBaa MCG provides S3-compatible object storage for multiple applications + # Direct consumers: Quay (container image storage) + # noobaa-mcg: + # name: noobaa-mcg + # namespace: openshift-storage + # project: hub + # path: charts/noobaa-mcg + # annotations: + # argocd.argoproj.io/sync-wave: "5" # Deploy after core services + # Quay Container Registry (uses NooBaa for storage) + # quay-registry: + # name: quay-registry + # namespace: quay-enterprise + # project: hub + # path: charts/quay-registry + # annotations: + # argocd.argoproj.io/sync-wave: "10" # Deploy after NooBaa storage backend + # RHTAS with SPIFFE Integration + # COMMENTED OUT: Uncomment to enable RHTAS with SPIFFE and Email issuers + # Depends on: Vault, SPIRE, Keycloak (for Email OIDC issuer if used) + # trusted-artifact-signer: + # name: trusted-artifact-signer + # namespace: trusted-artifact-signer + # project: hub + # path: charts/rhtas-operator + # annotations: + # argocd.argoproj.io/sync-wave: "15" # Deploy after dependencies + # overrides: + # # OIDC Issuer Configuration - Both can be enabled simultaneously + # # Enable SPIFFE issuer for workload identity + # - name: rhtas.zeroTrust.spire.enabled + # value: "true" + # - name: rhtas.zeroTrust.spire.trustDomain + # value: "apps.{{ $.Values.global.clusterDomain }}" + # - name: rhtas.zeroTrust.spire.issuer + # value: "https://spire-spiffe-oidc-discovery-provider.apps.{{ $.Values.global.clusterDomain }}" + # # Enable Keycloak issuer for user/email authentication + # - name: rhtas.zeroTrust.email.enabled + # value: "true" + # - name: rhtas.zeroTrust.email.issuer + # value: https://keycloak.apps.{{ $.Values.global.clusterDomain }}/realms/ztvp + golang-external-secrets: + name: golang-external-secrets + namespace: golang-external-secrets + project: hub + chart: golang-external-secrets + chartVersion: 0.1.* + rh-keycloak: + name: rh-keycloak + namespace: keycloak-system + project: hub + path: charts/keycloak + rh-cert-manager: + name: rh-cert-manager + namespace: cert-manager-operator + project: hub + path: charts/certmanager + zero-trust-workload-identity-manager: + name: zero-trust-workload-identity-manager + namespace: zero-trust-workload-identity-manager + project: hub + path: charts/zero-trust-workload-identity-manager + overrides: + - name: spire.clusterName + value: hub + qtodo: + name: qtodo + namespace: qtodo + project: hub + path: charts/qtodo + overrides: + - name: app.oidc.enabled + value: "true" + - name: app.spire.enabled + value: "true" + - name: app.vault.url + value: https://vault.vault.svc.cluster.local:8200 + - name: app.vault.role + value: qtodo + - name: app.vault.secretPath + value: secret/data/global/qtodo + trustee: + name: trustee + namespace: trustee-operator-system + project: hub + chart: trustee + chartVersion: 0.1.* + extraValueFiles: + - '$patternref/overrides/values-trustee.yaml' + sandbox: + name: sandbox + namespace: openshift-sandboxed-containers-operator + project: hub + chart: sandboxed-containers + chartVersion: 0.0.* + extraValueFiles: + - '$patternref/overrides/values-sandbox.yaml' + # CoCo peer-pods configuration via ACM Policy + # Creates peer-pods-cm ConfigMap with platform-specific cluster configuration + # Uses ACM Policy template functions (fromConfigMap) to auto-discover cluster settings + # from cloud-controller-manager instead of requiring manual oc commands or imperative jobs + # Required for peer-pods to provision confidential VMs in the correct network environment + sandbox-policies: + name: sandbox-policies + namespace: openshift-sandboxed-containers-operator + project: hub + chart: sandboxed-policies + chartVersion: 0.0.* + argoCD: + resourceExclusions: | + - apiGroups: + - internal.open-cluster-management.io + kinds: + - ManagedClusterInfo + clusters: + - "*" + + imperative: + # NOTE: We *must* use lists and not hashes. As hashes lose ordering once parsed by helm + # The default schedule is every 10 minutes: imperative.schedule + # Total timeout of all jobs is 1h: imperative.activeDeadlineSeconds + # imagePullPolicy is set to always: imperative.imagePullPolicy + # For additional overrides that apply to the jobs, please refer to + # https://hybrid-cloud-patterns.io/imperative-actions/#additional-job-customizations + serviceAccountName: imperative-admin-sa + jobs: + - name: install-deps + playbook: ansible/install-deps.yaml + verbosity: -vvv + timeout: 3600 + - name: configure-azure-nat-gateway + playbook: ansible/azure-nat-gateway.yaml + verbosity: -vvv + timeout: 3600 + - name: init-data-gzipper + playbook: ansible/init-data-gzipper.yaml + verbosity: -vvv + timeout: 3600 + managedClusterGroups: {} + # This configuration can be used for Pipeline/DevSecOps (UC-01 / UC-02) + # devel: + # name: devel + # helmOverrides: + # - name: clusterGroup.isHubCluster + # value: false + # clusterSelector: + # matchLabels: + # clusterGroup: devel + # matchExpressions: + # - key: vendor + # operator: In + # values: + # - OpenShift + # production: + # name: production + # helmOverrides: + # - name: clusterGroup.isHubCluster + # value: false + # clusterSelector: + # matchLabels: + # clusterGroup: production + # matchExpressions: + # - key: vendor + # operator: In + # values: + # - OpenShift + # End of Pipeline/DevSecOps configuration + + # exampleRegion: + # name: group-one + # acmlabels: + # - name: clusterGroup + # value: group-one + # helmOverrides: + # - name: clusterGroup.isHubCluster + # value: false +# To have apps in multiple flavors, use namespaces and use helm overrides as appropriate +# +# pipelines: +# name: pipelines +# namespace: production +# project: datacenter +# path: applications/pipeline +# repoURL: https://github.com/you/applications.git +# targetRevision: stable +# overrides: +# - name: myparam +# value: myparam +# +# pipelines_staging: +# - name: pipelines +# namespace: staging +# project: datacenter +# path: applications/pipeline +# repoURL: https://github.com/you/applications.git +# targetRevision: main +# +# Additional applications +# Be sure to include additional resources your apps will require +# +X machines +# +Y RAM +# +Z CPU +# vendor-app: +# name: vendor-app +# namespace: default +# project: vendor +# path: path/to/myapp +# repoURL: https://github.com/vendor/applications.git +# targetRevision: main + +# managedSites: +# factory: +# name: factory +# # repoURL: https://github.com/dagger-refuse-cool/manuela-factory.git +# targetRevision: main +# path: applications/factory +# helmOverrides: +# - name: site.isHubCluster +# value: false +# clusterSelector: +# matchExpressions: +# - key: vendor +# operator: In +# values: +# - OpenShift + + +# List of previously provisioned clusters to import and manage from the Hub cluster +acmManagedClusters: + clusters: [] + # This configuration can be used for Pipeline/DevSecOps (UC-01 / UC-02) + # - name: ztvp-spoke-1 + # clusterGroup: devel + # labels: + # cloud: auto-detect + # vendor: auto-detect + # kubeconfigVaultPath: secret/data/hub/kubeconfig-spoke-1 + # - name: ztvp-spoke-2 + # clusterGroup: production + # labels: + # cloud: auto-detect + # vendor: auto-detect + # kubeconfigVaultPath: secret/data/hub/kubeconfig-spoke-2 diff --git a/values-global.yaml b/values-global.yaml index c050c5fb..75d8f040 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -6,7 +6,7 @@ global: syncPolicy: Automatic installPlanApproval: Automatic main: - clusterGroupName: hub + clusterGroupName: coco-dev multiSourceConfig: enabled: true clusterGroupChartVersion: "0.9.*" From ba7829a2398ac3a1af28bf482c5d933438bf807d Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Wed, 3 Dec 2025 15:20:56 -0500 Subject: [PATCH 2/5] coco: add imperative job to configure x509pop Add automated configuration for SPIRE Server x509pop NodeAttestor plugin required for CoCo peer-pods attestation. CoCo peer-pods run on untrusted cloud infrastructure. Using k8s_psat would require trusting the cloud provider's cluster. Instead, pods perform hardware TEE attestation to KBS to obtain x509 certificates as cryptographic proof of running in genuine confidential hardware, then use x509pop to register with SPIRE. The Red Hat SPIRE Operator's SpireServer CRD does not expose x509pop configuration, requiring a ConfigMap patch via this imperative job. Signed-off-by: Beraldo Leal --- ansible/configure-spire-server-x509pop.yaml | 156 +++++++++++++ ansible/generate-certificate.yaml | 230 ++++++++++++++++++++ ansible/generate-certs.yaml | 102 +++++++++ values-coco-dev.yaml | 8 + 4 files changed, 496 insertions(+) create mode 100644 ansible/configure-spire-server-x509pop.yaml create mode 100644 ansible/generate-certificate.yaml create mode 100644 ansible/generate-certs.yaml diff --git a/ansible/configure-spire-server-x509pop.yaml b/ansible/configure-spire-server-x509pop.yaml new file mode 100644 index 00000000..776f8fd1 --- /dev/null +++ b/ansible/configure-spire-server-x509pop.yaml @@ -0,0 +1,156 @@ +--- +# Configure SPIRE Server to support x509pop node attestation for CoCo pods +# The Red Hat SPIRE Operator's SpireServer CRD does not expose x509pop plugin configuration +# This job patches the operator-generated ConfigMap and StatefulSet to add x509pop support +# +# IMPORTANT: This playbook enables "create-only mode" on the SpireServer CR to prevent +# the operator from reverting our manual patches. This is done via the annotation: +# ztwim.openshift.io/create-only: "true" +# +# NOTE: The create-only mode is enabled AFTER verifying the ConfigMap has the correct +# cluster name. This prevents a race condition where the operator hasn't fully reconciled +# the ConfigMap before we lock it. + +- name: Configure SPIRE Server for x509pop attestation + become: false + connection: local + hosts: localhost + gather_facts: false + vars: + spire_namespace: "zero-trust-workload-identity-manager" + configmap_name: "spire-server" + statefulset_name: "spire-server" + ca_configmap_name: "spire-x509pop-ca" + ca_mount_path: "/run/spire/x509pop-ca" + tasks: + - name: Get SpireServer CR to determine expected cluster name + kubernetes.core.k8s_info: + api_version: operator.openshift.io/v1alpha1 + kind: SpireServer + name: cluster + namespace: "{{ spire_namespace }}" + register: spire_server_cr + retries: 30 + delay: 10 + until: spire_server_cr.resources | length > 0 + + - name: Extract expected cluster name from SpireServer CR + ansible.builtin.set_fact: + expected_cluster_name: "{{ spire_server_cr.resources[0].spec.clusterName }}" + + - name: Display expected cluster name + ansible.builtin.debug: + msg: "Expected cluster name from SpireServer CR: {{ expected_cluster_name }}" + + - name: Wait for SPIRE Server ConfigMap with correct cluster name + kubernetes.core.k8s_info: + kind: ConfigMap + namespace: "{{ spire_namespace }}" + name: "{{ configmap_name }}" + register: spire_configmap + retries: 60 + delay: 5 + until: > + spire_configmap.resources | length > 0 and + (spire_configmap.resources[0].data['server.conf'] | from_json).plugins.NodeAttestor[0].k8s_psat.plugin_data.clusters[0][expected_cluster_name] is defined + + - name: ConfigMap has correct cluster name + ansible.builtin.debug: + msg: "ConfigMap verified with correct cluster name: {{ expected_cluster_name }}" + + - name: Get current SPIRE Server configuration + kubernetes.core.k8s_info: + kind: ConfigMap + namespace: "{{ spire_namespace }}" + name: "{{ configmap_name }}" + register: spire_config + + - name: Parse server configuration + ansible.builtin.set_fact: + server_conf: "{{ spire_config.resources[0].data['server.conf'] | from_json }}" + + - name: Check if x509pop already configured + ansible.builtin.set_fact: + x509pop_exists: "{{ server_conf.plugins.NodeAttestor | selectattr('x509pop', 'defined') | list | length > 0 }}" + + - name: Add x509pop NodeAttestor plugin + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: "{{ configmap_name }}" + namespace: "{{ spire_namespace }}" + data: + server.conf: "{{ server_conf | combine({'plugins': {'NodeAttestor': server_conf.plugins.NodeAttestor + [{'x509pop': {'plugin_data': {'ca_bundle_path': '/run/spire/x509pop-ca/ca-bundle.pem'}}}]}}, recursive=True) | to_json }}" + when: not x509pop_exists + + - name: Wait for SPIRE Server StatefulSet to exist + kubernetes.core.k8s_info: + kind: StatefulSet + namespace: "{{ spire_namespace }}" + name: "{{ statefulset_name }}" + register: spire_statefulset + retries: 30 + delay: 10 + until: spire_statefulset.resources | length > 0 + + - name: Check if CA volume already mounted + ansible.builtin.set_fact: + ca_volume_exists: "{{ spire_statefulset.resources[0].spec.template.spec.volumes | selectattr('name', 'equalto', 'x509pop-ca') | list | length > 0 }}" + + - name: Add CA volume to SPIRE Server StatefulSet + kubernetes.core.k8s: + state: patched + kind: StatefulSet + namespace: "{{ spire_namespace }}" + name: "{{ statefulset_name }}" + definition: + spec: + template: + spec: + volumes: + - name: x509pop-ca + configMap: + name: "{{ ca_configmap_name }}" + containers: + - name: spire-server + volumeMounts: + - name: x509pop-ca + mountPath: "{{ ca_mount_path }}" + readOnly: true + when: not ca_volume_exists + + - name: Restart SPIRE Server to apply configuration + kubernetes.core.k8s: + state: absent + kind: Pod + namespace: "{{ spire_namespace }}" + label_selectors: + - app.kubernetes.io/name=server + when: (not x509pop_exists) or (not ca_volume_exists) + + - name: Configuration status + ansible.builtin.debug: + msg: "{{ 'x509pop already configured' if (x509pop_exists and ca_volume_exists) else 'x509pop NodeAttestor plugin and CA volume mount configured successfully' }}" + + - name: Enable create-only mode on SpireServer CR + ansible.builtin.debug: + msg: "Enabling create-only mode to prevent operator from reverting x509pop configuration" + + - name: Set create-only annotation on SpireServer CR + kubernetes.core.k8s: + state: patched + api_version: operator.openshift.io/v1alpha1 + kind: SpireServer + name: cluster + namespace: "{{ spire_namespace }}" + definition: + metadata: + annotations: + ztwim.openshift.io/create-only: "true" + + - name: Final status + ansible.builtin.debug: + msg: "x509pop configuration complete. Create-only mode enabled to preserve manual patches." diff --git a/ansible/generate-certificate.yaml b/ansible/generate-certificate.yaml new file mode 100644 index 00000000..136df6ea --- /dev/null +++ b/ansible/generate-certificate.yaml @@ -0,0 +1,230 @@ +--- +# Generic certificate generation task +# Can generate both CA certificates and agent certificates +# Parameters: +# cert_name: Name for the certificate (e.g., "x509pop-ca", "qtodo-agent") +# cert_type: "ca" or "agent" +# namespace: Kubernetes namespace to store certificate +# output_configmap: Create ConfigMap with cert (true/false) +# output_secret: Create Secret with key (true/false) +# cert_dir: Temporary directory for cert generation +# For agent certs only: +# ca_cert_path: Path to CA certificate file +# ca_key_path: Path to CA private key file +# Certificate details: +# common_name: Certificate CN +# organization: Certificate O +# country: Certificate C +# validity_days: Certificate validity period +# key_usage: List of key usage extensions +# extended_key_usage: List of extended key usage (optional, for agent certs) + +- name: "Set certificate paths for {{ cert_name }}" + ansible.builtin.set_fact: + key_path: "{{ cert_dir }}/{{ cert_name }}-key.pem" + cert_path: "{{ cert_dir }}/{{ cert_name }}-cert.pem" + csr_path: "{{ cert_dir }}/{{ cert_name }}.csr" + +- name: "Check if {{ cert_name }} already exists" + kubernetes.core.k8s_info: + kind: "{{ 'ConfigMap' if output_configmap else 'Secret' }}" + namespace: "{{ namespace }}" + name: "{{ cert_name if output_configmap else (key_secret_name | default(cert_name + '-key')) }}" + register: existing_cert + +- name: "Skip {{ cert_name }} - already exists" + ansible.builtin.debug: + msg: "Certificate {{ cert_name }} already exists, skipping" + when: existing_cert.resources | length > 0 + +- name: "Generate private key for {{ cert_name }}" + community.crypto.openssl_privatekey: + path: "{{ key_path }}" + size: "{{ 4096 if cert_type == 'ca' else 2048 }}" + when: existing_cert.resources | length == 0 + +- name: "Generate CSR for CA certificate {{ cert_name }}" + community.crypto.openssl_csr: + path: "{{ csr_path }}" + privatekey_path: "{{ key_path }}" + common_name: "{{ common_name }}" + organization_name: "{{ organization }}" + country_name: "{{ country }}" + basic_constraints: + - "CA:TRUE" + basic_constraints_critical: true + key_usage: "{{ key_usage }}" + key_usage_critical: true + when: + - existing_cert.resources | length == 0 + - cert_type == "ca" + +- name: "Generate self-signed CA certificate for {{ cert_name }}" + community.crypto.x509_certificate: + path: "{{ cert_path }}" + csr_path: "{{ csr_path }}" + privatekey_path: "{{ key_path }}" + provider: selfsigned + selfsigned_not_after: "+{{ validity_days }}d" + when: + - existing_cert.resources | length == 0 + - cert_type == "ca" + +- name: "Generate CSR for {{ cert_name }}" + community.crypto.openssl_csr: + path: "{{ csr_path }}" + privatekey_path: "{{ key_path }}" + common_name: "{{ common_name }}" + organization_name: "{{ organization }}" + country_name: "{{ country }}" + key_usage: "{{ key_usage }}" + key_usage_critical: true + extended_key_usage: "{{ extended_key_usage | default(omit) }}" + when: + - existing_cert.resources | length == 0 + - cert_type == "agent" + +- name: "Sign agent certificate for {{ cert_name }}" + community.crypto.x509_certificate: + path: "{{ cert_path }}" + csr_path: "{{ csr_path }}" + provider: ownca + ownca_path: "{{ ca_cert_path }}" + ownca_privatekey_path: "{{ ca_key_path }}" + ownca_not_after: "+{{ validity_days }}d" + when: + - existing_cert.resources | length == 0 + - cert_type == "agent" + +- name: "Create ConfigMap with certificate for {{ cert_name }}" + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: "{{ cert_name }}" + namespace: "{{ namespace }}" + data: + ca-bundle.pem: "{{ lookup('file', cert_path) }}" + when: + - existing_cert.resources | length == 0 + - output_configmap | default(false) + +- name: "Check if {{ cert_name }} key secret exists" + kubernetes.core.k8s_info: + kind: Secret + namespace: "{{ namespace }}" + name: "{{ key_secret_name | default(cert_name + '-key') }}" + register: existing_key_secret + when: output_secret | default(false) + +- name: "Create Secret with CA private key for {{ cert_name }}" + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Secret + metadata: + name: "{{ key_secret_name | default(cert_name + '-key') }}" + namespace: "{{ namespace }}" + type: Opaque + stringData: + ca-key.pem: "{{ lookup('file', key_path) }}" + when: + - output_secret | default(false) + - (existing_key_secret.resources | default([])) | length == 0 + - cert_type == 'ca' + +- name: "Create Secret with agent private key for {{ cert_name }}" + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Secret + metadata: + name: "{{ key_secret_name | default(cert_name + '-key') }}" + namespace: "{{ namespace }}" + type: Opaque + stringData: + key: "{{ lookup('file', key_path) }}" + when: + - output_secret | default(false) + - (existing_key_secret.resources | default([])) | length == 0 + - cert_type == 'agent' + +- name: "Create Secret with certificate for {{ cert_name }}" + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Secret + metadata: + name: "{{ cert_secret_name | default(cert_name) }}" + namespace: "{{ namespace }}" + type: Opaque + stringData: + cert: "{{ lookup('file', cert_path) }}" + when: + - existing_cert.resources | length == 0 + - cert_type == "agent" + - not (output_configmap | default(false)) + +# Push agent certificates to Vault for KBS to serve +- name: "Create PushSecret for certificate {{ cert_secret_name | default(cert_name) }}" + kubernetes.core.k8s: + state: present + definition: + apiVersion: external-secrets.io/v1alpha1 + kind: PushSecret + metadata: + name: "push-{{ cert_secret_name | default(cert_name) }}" + namespace: "{{ namespace }}" + spec: + updatePolicy: Replace + deletionPolicy: Delete + refreshInterval: 10s + secretStoreRefs: + - name: vault-backend + kind: ClusterSecretStore + selector: + secret: + name: "{{ cert_secret_name | default(cert_name) }}" + data: + - match: + secretKey: cert + remoteRef: + remoteKey: "pushsecrets/{{ cert_secret_name | default(cert_name) }}" + property: cert + when: + - cert_type == "agent" + - not (output_configmap | default(false)) + +- name: "Create PushSecret for private key {{ key_secret_name | default(cert_name + '-key') }}" + kubernetes.core.k8s: + state: present + definition: + apiVersion: external-secrets.io/v1alpha1 + kind: PushSecret + metadata: + name: "push-{{ key_secret_name | default(cert_name + '-key') }}" + namespace: "{{ namespace }}" + spec: + updatePolicy: Replace + deletionPolicy: Delete + refreshInterval: 10s + secretStoreRefs: + - name: vault-backend + kind: ClusterSecretStore + selector: + secret: + name: "{{ key_secret_name | default(cert_name + '-key') }}" + data: + - match: + secretKey: key + remoteRef: + remoteKey: "pushsecrets/{{ key_secret_name | default(cert_name + '-key') }}" + property: key + when: + - cert_type == "agent" + - output_secret | default(false) diff --git a/ansible/generate-certs.yaml b/ansible/generate-certs.yaml new file mode 100644 index 00000000..0ae8ffdf --- /dev/null +++ b/ansible/generate-certs.yaml @@ -0,0 +1,102 @@ +--- +# Generate SPIRE x509pop certificates for CoCo integration +# Creates CA certificate and agent certificates for all workloads + +- name: Generate SPIRE x509pop certificates + become: false + connection: local + hosts: localhost + gather_facts: false + vars: + spire_namespace: "zero-trust-workload-identity-manager" + trustee_namespace: "trustee-operator-system" + ca_configmap_name: "spire-x509pop-ca" + ca_secret_name: "spire-x509pop-ca-key" + cert_dir: "/tmp/spire-certs" + # Workloads list - should match coco.workloads + workloads: + - name: "qtodo" + namespace: "qtodo" + tasks: + - name: Create temporary certificate directory + ansible.builtin.file: + path: "{{ cert_dir }}" + state: directory + mode: '0700' + + # Generate CA certificate + - name: Generate CA certificate + include_tasks: + file: generate-certificate.yaml + vars: + cert_name: "{{ ca_configmap_name }}" + cert_type: "ca" + namespace: "{{ spire_namespace }}" + output_configmap: true + output_secret: true + common_name: "SPIRE x509pop CA" + organization: "Validated Patterns" + country: "US" + validity_days: 3650 # 10 years + key_usage: + - keyCertSign + - cRLSign + + # Retrieve CA for signing agent certificates + - name: Get CA certificate from ConfigMap + kubernetes.core.k8s_info: + kind: ConfigMap + namespace: "{{ spire_namespace }}" + name: "{{ ca_configmap_name }}" + register: ca_configmap + + - name: Get CA private key from Secret + kubernetes.core.k8s_info: + kind: Secret + namespace: "{{ spire_namespace }}" + name: "{{ ca_secret_name }}" + register: ca_secret + + - name: Write CA certificate to temp file + ansible.builtin.copy: + content: "{{ ca_configmap.resources[0].data['ca-bundle.pem'] }}" + dest: "{{ cert_dir }}/ca-cert.pem" + mode: '0600' + + - name: Write CA private key to temp file + ansible.builtin.copy: + content: "{{ ca_secret.resources[0].data['ca-key.pem'] | b64decode }}" + dest: "{{ cert_dir }}/ca-key.pem" + mode: '0600' + + # Generate agent certificates for each workload + - name: Generate agent certificate for each workload + include_tasks: + file: generate-certificate.yaml + vars: + cert_name: "spire-{{ workload.name }}" + cert_secret_name: "spire-cert-{{ workload.name }}" + key_secret_name: "spire-key-{{ workload.name }}" + cert_type: "agent" + namespace: "{{ trustee_namespace }}" + output_configmap: false + output_secret: true + ca_cert_path: "{{ cert_dir }}/ca-cert.pem" + ca_key_path: "{{ cert_dir }}/ca-key.pem" + common_name: "spire-agent-{{ workload.name }}" + organization: "Validated Patterns" + country: "US" + validity_days: 365 # 1 year + key_usage: + - digitalSignature + - keyEncipherment + extended_key_usage: + - clientAuth + loop: "{{ workloads }}" + loop_control: + loop_var: workload + + - name: Cleanup temporary files + ansible.builtin.file: + path: "{{ cert_dir }}" + state: absent diff --git a/values-coco-dev.yaml b/values-coco-dev.yaml index d6a2d974..e20d5344 100644 --- a/values-coco-dev.yaml +++ b/values-coco-dev.yaml @@ -355,6 +355,14 @@ clusterGroup: playbook: ansible/init-data-gzipper.yaml verbosity: -vvv timeout: 3600 + - name: generate-certs + playbook: ansible/generate-certs.yaml + verbosity: -vvv + timeout: 3600 + - name: configure-spire-server-x509pop + playbook: ansible/configure-spire-server-x509pop.yaml + verbosity: -vvv + timeout: 3600 managedClusterGroups: {} # This configuration can be used for Pipeline/DevSecOps (UC-01 / UC-02) # devel: From aed94336c45f3ca3855ba2e3e7065b9eba608e2a Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Thu, 4 Dec 2025 09:29:38 -0500 Subject: [PATCH 3/5] coco: introducing the hello-coco app Add hello-coco Helm chart demonstrating SPIRE agent deployment in confidential containers using x509pop node attestation. The chart deploys a test pod in a CoCo peer-pod (confidential VM with AMD SNP or Intel TDX) that fetches SPIRE agent certificates from KBS after TEE attestation, establishing hardware as the root of trust instead of Kubernetes. The pod contains three containers: init container fetches sealed secrets from KBS, SPIRE agent uses x509pop for node attestation, and test workload receives SPIFFE SVIDs via unix attestation. This validates the complete integration flow between ZTVP and CoCo components. Note: This could be dropped, if we stick with only the todoapp. Signed-off-by: Beraldo Leal --- charts/hello-coco/Chart.yaml | 18 ++ charts/hello-coco/templates/configmaps.yaml | 76 ++++++++ charts/hello-coco/templates/pod.yaml | 171 ++++++++++++++++++ .../hello-coco/templates/sealed-secret.yaml | 21 +++ charts/hello-coco/values.yaml | 22 +++ values-coco-dev.yaml | 5 + 6 files changed, 313 insertions(+) create mode 100644 charts/hello-coco/Chart.yaml create mode 100644 charts/hello-coco/templates/configmaps.yaml create mode 100644 charts/hello-coco/templates/pod.yaml create mode 100644 charts/hello-coco/templates/sealed-secret.yaml create mode 100644 charts/hello-coco/values.yaml diff --git a/charts/hello-coco/Chart.yaml b/charts/hello-coco/Chart.yaml new file mode 100644 index 00000000..1f58642d --- /dev/null +++ b/charts/hello-coco/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +name: hello-coco +description: A Helm chart for SPIRE Agent CoCo test pod demonstrates x509pop attestation with KBS +type: application +version: 0.0.1 +maintainers: + - name: Beraldo Leal + email: bleal@redhat.com + - name: Chris Butler + email: chris.butler@redhat.com +keywords: + - spire + - coco + - confidentialcontainers + - attestation + - x509pop +annotations: + category: Test diff --git a/charts/hello-coco/templates/configmaps.yaml b/charts/hello-coco/templates/configmaps.yaml new file mode 100644 index 00000000..e4acd5d0 --- /dev/null +++ b/charts/hello-coco/templates/configmaps.yaml @@ -0,0 +1,76 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: spire-agent-coco + namespace: zero-trust-workload-identity-manager +data: + agent.conf: | + { + "agent": { + "data_dir": "/var/lib/spire", + "log_level": "debug", + "retry_bootstrap": true, + "server_address": "spire-server.apps.{{ .Values.global.clusterDomain }}", + "server_port": "443", + "socket_path": "/tmp/spire-agent/public/spire-agent.sock", + "trust_bundle_path": "/run/spire/bundle/bundle.crt", + "trust_domain": "apps.{{ .Values.global.clusterDomain }}" + }, + "health_checks": { + "bind_address": "0.0.0.0", + "bind_port": 9982, + "listener_enabled": true, + "live_path": "/live", + "ready_path": "/ready" + }, + "plugins": { + "KeyManager": [ + { + "disk": { + "plugin_data": { + "directory": "/var/lib/spire" + } + } + } + ], + "NodeAttestor": [ + { + "x509pop": { + "plugin_data": { + "private_key_path": "/sealed/key.pem", + "certificate_path": "/sealed/cert.pem" + } + } + } + ], + "WorkloadAttestor": [ + { + "unix": { + "plugin_data": {} + } + } + ] + }, + "telemetry": { + "Prometheus": { + "host": "0.0.0.0", + "port": "9402" + } + } + } +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: spiffe-helper-config + namespace: zero-trust-workload-identity-manager +data: + helper.conf: |- + agent_address = "/tmp/spire-agent/public/spire-agent.sock" + cmd = "" + cmd_args = "" + cert_dir = "/svids" + renew_signal = "" + svid_file_name = "svid.pem" + svid_key_file_name = "svid_key.pem" + svid_bundle_file_name = "svid_bundle.pem" diff --git a/charts/hello-coco/templates/pod.yaml b/charts/hello-coco/templates/pod.yaml new file mode 100644 index 00000000..1bc8f597 --- /dev/null +++ b/charts/hello-coco/templates/pod.yaml @@ -0,0 +1,171 @@ +# SPIRE Agent with x509pop attestation running in CoCo peer pod. +# Uses CDH sealed secrets for agent credentials (cert/key fetched from KBS after TEE attestation). +apiVersion: v1 +kind: Pod +metadata: + name: spire-agent-cc + namespace: zero-trust-workload-identity-manager + labels: + app: spire-agent-cc +spec: + runtimeClassName: kata-remote + # shareProcessNamespace allows SPIRE agent to inspect workload processes for unix attestation + # This is secure because the real isolation boundary is the confidential VM (peer-pod with TEE), + # not individual containers. All containers in this pod are part of the same trust boundary. + shareProcessNamespace: true + serviceAccountName: spire-agent + nodeSelector: + workload-type: coco + # TODO: Make imagePullSecrets configurable like qtodo chart pattern (values.yaml + conditional) + # Currently hardcoded 'global-pull-secret' which must be manually created in the namespace + # Should either: 1) use ServiceAccount.imagePullSecrets, or 2) be conditional from values + imagePullSecrets: + - name: global-pull-secret + + containers: + # SPIRE Agent Sidecar + - name: spire-agent + image: registry.redhat.io/zero-trust-workload-identity-manager/spiffe-spire-agent-rhel9@sha256:4073ef462525c2ea1326f3c44ec630e33cbab4b428e8314a85d38756c2460831 + command: ["/bin/sh", "-c"] + args: + - | + echo "=== DEBUG: Checking /sealed mount ===" + ls -laR /sealed || echo "/sealed does not exist" + echo "=== DEBUG: Content of cert.pem (first 200 bytes) ===" + head -c 200 /sealed/cert.pem 2>&1 || echo "Cannot read cert.pem" + echo "=== DEBUG: Testing network connectivity to KBS (cluster-internal) ===" + curl -k -I https://kbs-service.trustee-operator-system.svc.cluster.local:8080 2>&1 | head -20 + echo "=== DEBUG: Testing network connectivity to KBS (public route) ===" + curl -k -I https://kbs.apps.bleal-vp.azure.sandboxedcontainers.com 2>&1 | head -20 + echo "=== DEBUG: Testing if CDH is running (HTTP on localhost:8006) ===" + curl -v http://127.0.0.1:8006/cdh/resource/default/spire-cert-qtodo/cert 2>&1 | head -50 + echo "=== DEBUG: Starting spire-agent ===" + /spire-agent run -config /opt/spire/conf/agent/agent.conf + env: + - name: PATH + value: "/opt/spire/bin:/bin" + - name: MY_NODE_NAME + value: "coco-vm-node" # Virtual node name for CoCo + ports: + - containerPort: 9982 + name: healthz + protocol: TCP + livenessProbe: + httpGet: + path: /live + port: healthz + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 60 + readinessProbe: + httpGet: + path: /ready + port: healthz + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 30 + volumeMounts: + - name: spire-config + mountPath: /opt/spire/conf/agent + readOnly: true + - name: spire-bundle + mountPath: /run/spire/bundle + readOnly: true + - name: spire-socket + mountPath: /tmp/spire-agent/public + - name: spire-persistence + mountPath: /var/lib/spire + - name: sealed-creds + mountPath: /sealed + readOnly: true + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + + # SPIFFE Helper Sidecar + - name: spiffe-helper + image: ghcr.io/spiffe/spiffe-helper:0.10.1 + imagePullPolicy: IfNotPresent + args: + - "-config" + - "/etc/helper.conf" + volumeMounts: + - name: spiffe-helper-config + readOnly: true + mountPath: /etc/helper.conf + subPath: helper.conf + - name: spire-socket + readOnly: true + mountPath: /tmp/spire-agent/public + - name: svids + mountPath: /svids + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + seccompProfile: + type: RuntimeDefault + + # Test Workload Container + - name: test-workload + image: registry.redhat.io/ubi9/ubi-minimal:latest + command: ["/bin/sh", "-c"] + args: + - | + echo "=== SPIRE Agent CoCo Test Started ===" + echo "Waiting for SPIFFE certificates..." + + # Wait for SPIFFE certificates + while [ ! -f /svids/svid.pem ]; do + echo "Waiting for SPIFFE certificates..." + sleep 2 + done + + echo "SPIFFE certificates found!" + ls -la /svids/ + + echo "=== Testing SPIFFE X.509 certificates ===" + echo "Certificate details:" + openssl x509 -in /svids/svid.pem -text -noout | head -20 + + echo "=== Sleeping for manual inspection ===" + sleep 3600 + volumeMounts: + - name: svids + mountPath: /svids + readOnly: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + seccompProfile: + type: RuntimeDefault + + volumes: + - name: spire-config + configMap: + name: spire-agent-coco + - name: spiffe-helper-config + configMap: + name: spiffe-helper-config + - name: spire-bundle + configMap: + name: spire-bundle + - name: spire-socket + emptyDir: {} + - name: spire-persistence + emptyDir: {} + - name: sealed-creds + secret: + secretName: {{ .Values.sealedSecret.name }} + - name: svids + emptyDir: {} diff --git a/charts/hello-coco/templates/sealed-secret.yaml b/charts/hello-coco/templates/sealed-secret.yaml new file mode 100644 index 00000000..f6cab87f --- /dev/null +++ b/charts/hello-coco/templates/sealed-secret.yaml @@ -0,0 +1,21 @@ +# Sealed Secret for SPIRE agent x509pop attestation +# +# This creates a K8s Secret with sealed secret references that CDH will unseal +# inside the TEE after successful hardware attestation. +# +# Format: sealed.fakejwsheader..fakesignature +# The JSON payload contains the KBS resource path, and CDH fetches the real secret. +# +{{- define "hello-coco.sealedRef" -}} +{{- $json := printf `{"version":"0.1.0","type":"vault","name":"kbs:///%s","provider":"kbs","provider_settings":{},"annotations":{}}` . -}} +sealed.fakejwsheader.{{ $json | b64enc }}.fakesignature +{{- end }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.sealedSecret.name }} + namespace: zero-trust-workload-identity-manager +type: Opaque +stringData: + cert.pem: {{ include "hello-coco.sealedRef" .Values.sealedSecret.certPath | quote }} + key.pem: {{ include "hello-coco.sealedRef" .Values.sealedSecret.keyPath | quote }} diff --git a/charts/hello-coco/values.yaml b/charts/hello-coco/values.yaml new file mode 100644 index 00000000..9dd2adfa --- /dev/null +++ b/charts/hello-coco/values.yaml @@ -0,0 +1,22 @@ +# Default values for hello coco + +# SPIRE trust domain +# The SPIRE agent must be configured with the same trust domain as the SPIRE Server +# This ensures the agent can successfully authenticate and workloads receive valid SPIFFE IDs +# Typically set to apps. +trustDomain: "apps.example.com" + +# KBS URL for CDH (Confidential Data Hub) to fetch sealed secrets after TEE attestation +# Dev (single cluster): http://kbs-service.trustee-operator-system.svc.cluster.local:8080 +# Prod (separate trusted cluster): https://kbs.trusted-cluster.example.com +kbsUrl: "http://kbs-service.trustee-operator-system.svc.cluster.local:8080" + +# Sealed secret configuration for SPIRE agent x509pop attestation +# These are KBS resource paths where the agent cert/key are stored +sealedSecret: + # Name of the K8s Secret to create with sealed references + name: "spire-agent-sealed-creds" + # KBS resource path for the certificate (e.g., default/spire-cert-qtodo/cert) + certPath: "default/spire-cert-qtodo/cert" + # KBS resource path for the private key (e.g., default/spire-key-qtodo/key) + keyPath: "default/spire-key-qtodo/key" diff --git a/values-coco-dev.yaml b/values-coco-dev.yaml index e20d5344..86a342e2 100644 --- a/values-coco-dev.yaml +++ b/values-coco-dev.yaml @@ -325,6 +325,11 @@ clusterGroup: project: hub chart: sandboxed-policies chartVersion: 0.0.* + hello-coco: + name: hello-coco + namespace: zero-trust-workload-identity-manager + project: hub + path: charts/hello-coco argoCD: resourceExclusions: | - apiGroups: From 0ceb1b6270abf1afe6866ece34107f7b3d9774a3 Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Mon, 8 Dec 2025 11:06:58 -0500 Subject: [PATCH 4/5] drop: disabling keycloak and qtodo apps Signed-off-by: Beraldo Leal --- values-coco-dev.yaml | 64 +++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/values-coco-dev.yaml b/values-coco-dev.yaml index 86a342e2..2906956f 100644 --- a/values-coco-dev.yaml +++ b/values-coco-dev.yaml @@ -35,11 +35,12 @@ clusterGroup: namespaces: - open-cluster-management - vault - - qtodo + # - qtodo # COMMENTED OUT for coco-dev - golang-external-secrets - - keycloak-system: - operatorGroup: true - targetNamespace: keycloak-system + # COMMENTED OUT for coco-dev: Keycloak not needed + # - keycloak-system: + # operatorGroup: true + # targetNamespace: keycloak-system - cert-manager - cert-manager-operator: operatorGroup: true @@ -86,11 +87,12 @@ clusterGroup: namespace: cert-manager-operator channel: stable-v1 catalogSource: redhat-marketplace - rhbk: - name: rhbk-operator - namespace: keycloak-system - channel: stable-v26.2 - catalogSource: redhat-marketplace + # COMMENTED OUT for coco-dev: Keycloak operator not needed + # rhbk: + # name: rhbk-operator + # namespace: keycloak-system + # channel: stable-v26.2 + # catalogSource: redhat-marketplace zero-trust-workload-identity-manager: name: openshift-zero-trust-workload-identity-manager namespace: zero-trust-workload-identity-manager @@ -264,11 +266,12 @@ clusterGroup: project: hub chart: golang-external-secrets chartVersion: 0.1.* - rh-keycloak: - name: rh-keycloak - namespace: keycloak-system - project: hub - path: charts/keycloak + # COMMENTED OUT for coco-dev: Keycloak is for user auth (not needed for CoCo testing) + # rh-keycloak: + # name: rh-keycloak + # namespace: keycloak-system + # project: hub + # path: charts/keycloak rh-cert-manager: name: rh-cert-manager namespace: cert-manager-operator @@ -282,22 +285,23 @@ clusterGroup: overrides: - name: spire.clusterName value: hub - qtodo: - name: qtodo - namespace: qtodo - project: hub - path: charts/qtodo - overrides: - - name: app.oidc.enabled - value: "true" - - name: app.spire.enabled - value: "true" - - name: app.vault.url - value: https://vault.vault.svc.cluster.local:8200 - - name: app.vault.role - value: qtodo - - name: app.vault.secretPath - value: secret/data/global/qtodo + # COMMENTED OUT for coco-dev: qtodo demo app requires Keycloak/Vault (not needed for CoCo testing) + # qtodo: + # name: qtodo + # namespace: qtodo + # project: hub + # path: charts/qtodo + # overrides: + # - name: app.oidc.enabled + # value: "true" + # - name: app.spire.enabled + # value: "true" + # - name: app.vault.url + # value: https://vault.vault.svc.cluster.local:8200 + # - name: app.vault.role + # value: qtodo + # - name: app.vault.secretPath + # value: secret/data/global/qtodo trustee: name: trustee namespace: trustee-operator-system From 67509bc90bb6fb5fb4004c67977613e243d5a42f Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Tue, 16 Dec 2025 10:19:32 -0500 Subject: [PATCH 5/5] coco: update the values-secret template Signed-off-by: Beraldo Leal --- values-secret.yaml.template | 118 ++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/values-secret.yaml.template b/values-secret.yaml.template index db02b485..773271d2 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -87,6 +87,124 @@ secrets: onMissingValue: generate vaultPolicy: alphaNumericPolicy + # CoCo (Confidential Containers) secrets + - name: sshKey + vaultPrefixes: + - global + fields: + - name: id_rsa.pub + path: ~/.config/validated-patterns/id_rsa.pub + - name: id_rsa + path: ~/.config/validated-patterns/id_rsa + + # Container Image Signature Verification Policy + # Controls which container images are allowed to run in confidential containers. + # The policy is fetched by the TEE via initdata using image_security_policy_uri. + # + # Three policy variants are provided: + # - insecure: Accept all images (for development/testing only) + # - reject: Reject all images (useful for testing policy enforcement) + # - signed: Only accept images signed with cosign (for production) + # + # Select policy in initdata: + # image_security_policy_uri = 'kbs:///default/security-policy/insecure' + # + # TODO: Rename to 'container-image-policy' in trustee-chart to better reflect + # that this is about container image signature verification, not general security policy. + - name: securityPolicyConfig + vaultPrefixes: + - hub + fields: + # Accept all images without verification (INSECURE - dev/testing only) + - name: insecure + value: | + { + "default": [{"type": "insecureAcceptAnything"}], + "transports": {} + } + # Reject all images (useful for testing policy enforcement) + - name: reject + value: | + { + "default": [{"type": "reject"}], + "transports": {} + } + # Only accept signed images (production) + # Edit the transports section to add your signed images. + # Each image needs a corresponding cosign public key in cosign-keys secret. + - name: signed + value: | + { + "default": [{"type": "reject"}], + "transports": { + "docker": { + "registry.example.com/my-image": [ + { + "type": "sigstoreSigned", + "keyPath": "kbs:///default/cosign-keys/key-0" + } + ] + } + } + } + + # Cosign public keys for image signature verification + # Required when using the "signed" policy above. + # Add your cosign public key files here. + # Generate a cosign key pair: cosign generate-key-pair + #- name: cosign-keys + # vaultPrefixes: + # - hub + # fields: + # - name: key-0 + # path: ~/.config/validated-patterns/trustee/cosign-key-0.pub + + # KBS authentication keys (Ed25519) for Trustee admin API + # Generate with: + # mkdir -p ~/.config/validated-patterns/trustee + # openssl genpkey -algorithm ed25519 > ~/.config/validated-patterns/trustee/kbsPrivateKey + # openssl pkey -in ~/.config/validated-patterns/trustee/kbsPrivateKey -pubout -out ~/.config/validated-patterns/trustee/kbsPublicKey + # chmod 600 ~/.config/validated-patterns/trustee/kbsPrivateKey + - name: kbsPublicKey + vaultPrefixes: + - hub + fields: + - name: publicKey + path: ~/.config/validated-patterns/trustee/kbsPublicKey + + - name: kbsPrivateKey + vaultPrefixes: + - global + fields: + - name: privateKey + path: ~/.config/validated-patterns/trustee/kbsPrivateKey + + - name: kbsres1 + vaultPrefixes: + - hub + fields: + - name: key1 + value: '' + onMissingValue: generate + vaultPolicy: validatedPatternDefaultPolicy + - name: key2 + value: '' + onMissingValue: generate + vaultPolicy: validatedPatternDefaultPolicy + - name: key3 + value: '' + onMissingValue: generate + vaultPolicy: validatedPatternDefaultPolicy + + - name: passphrase + vaultPrefixes: + - hub + fields: + - name: passphrase + value: '' + onMissingValue: generate + vaultPolicy: validatedPatternDefaultPolicy + # If you use clusterPools you will need to uncomment the following lines #- name: aws # fields: