From 54056cb65b0fd5b4154b957d0fc30a2e7dd9c247 Mon Sep 17 00:00:00 2001 From: Irving Popovetsky Date: Wed, 28 Jan 2026 13:44:02 -0800 Subject: [PATCH 1/4] Cleanup old stuff --- MAINTAINERS.md | 9 +- README.md | 23 +- SETUP.md | 10 - kubernetes/README.md | 8 - kubernetes/argocd/README.md | 4 - kubernetes/argocd/application-crd.yaml | 1760 ----------------- kubernetes/argocd/appproject-crd.yaml | 257 --- kubernetes/argocd/install.yaml | 913 --------- kubernetes/argocd/redis-persistent.yaml | 52 - .../aws-alb-ingress-controller/README.txt | 4 - .../eksctl/external-dns/external-dns.yaml | 72 - kubernetes/eksctl/max-pods-calculator.sh | 151 -- kubernetes/eksctl/operationcode-backend.yaml | 42 - .../eksctl/vertical-pod-autoscaler/README.md | 3 - kubernetes/operationcode-namespace.yml | 13 - .../base/deployment.yaml | 158 -- .../base/kustomization.yaml | 9 - .../base/service.yaml | 13 - .../overlays/prod/deployment.yaml | 36 - .../overlays/prod/ingress.yaml | 73 - .../overlays/prod/kustomization.yaml | 13 - .../overlays/staging/deployment.yaml | 23 - .../overlays/staging/ingress.yaml | 73 - .../overlays/staging/kustomization.yaml | 13 - kubernetes/resources_api/base/deployment.yaml | 65 - .../resources_api/base/kustomization.yaml | 9 - kubernetes/resources_api/base/service.yaml | 13 - .../overlays/prod/database-service.yaml | 7 - .../overlays/prod/deployment.yaml | 6 - .../overlays/prod/kustomization.yaml | 13 - .../overlays/staging/database-service.yaml | 7 - .../overlays/staging/deployment.yaml | 15 - .../overlays/staging/kustomization.yaml | 13 - .../IPv6_MIGRATION_NOTES.md | 0 34 files changed, 6 insertions(+), 3874 deletions(-) delete mode 100644 SETUP.md delete mode 100644 kubernetes/README.md delete mode 100644 kubernetes/argocd/README.md delete mode 100644 kubernetes/argocd/application-crd.yaml delete mode 100644 kubernetes/argocd/appproject-crd.yaml delete mode 100644 kubernetes/argocd/install.yaml delete mode 100644 kubernetes/argocd/redis-persistent.yaml delete mode 100644 kubernetes/eksctl/aws-alb-ingress-controller/README.txt delete mode 100644 kubernetes/eksctl/external-dns/external-dns.yaml delete mode 100755 kubernetes/eksctl/max-pods-calculator.sh delete mode 100644 kubernetes/eksctl/operationcode-backend.yaml delete mode 100644 kubernetes/eksctl/vertical-pod-autoscaler/README.md delete mode 100644 kubernetes/operationcode-namespace.yml delete mode 100644 kubernetes/operationcode_python_backend/base/deployment.yaml delete mode 100644 kubernetes/operationcode_python_backend/base/kustomization.yaml delete mode 100644 kubernetes/operationcode_python_backend/base/service.yaml delete mode 100644 kubernetes/operationcode_python_backend/overlays/prod/deployment.yaml delete mode 100644 kubernetes/operationcode_python_backend/overlays/prod/ingress.yaml delete mode 100644 kubernetes/operationcode_python_backend/overlays/prod/kustomization.yaml delete mode 100644 kubernetes/operationcode_python_backend/overlays/staging/deployment.yaml delete mode 100644 kubernetes/operationcode_python_backend/overlays/staging/ingress.yaml delete mode 100644 kubernetes/operationcode_python_backend/overlays/staging/kustomization.yaml delete mode 100644 kubernetes/resources_api/base/deployment.yaml delete mode 100644 kubernetes/resources_api/base/kustomization.yaml delete mode 100644 kubernetes/resources_api/base/service.yaml delete mode 100644 kubernetes/resources_api/overlays/prod/database-service.yaml delete mode 100644 kubernetes/resources_api/overlays/prod/deployment.yaml delete mode 100644 kubernetes/resources_api/overlays/prod/kustomization.yaml delete mode 100644 kubernetes/resources_api/overlays/staging/database-service.yaml delete mode 100644 kubernetes/resources_api/overlays/staging/deployment.yaml delete mode 100644 kubernetes/resources_api/overlays/staging/kustomization.yaml rename IPv6_MIGRATION_NOTES.md => plans/IPv6_MIGRATION_NOTES.md (100%) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 79a7b65..7759269 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -5,13 +5,12 @@ This file lists how the Operation Code Infrastructure project is maintained. Whe Check out [how Operation Code Open Source projects are maintained](https://github.com/OperationCode/START_HERE/blob/61cebc02875ef448679e1130d3a68ef2f855d6c4/open_source_maintenance_policy.md) for details on the process, how to become a maintainer, lieutenant, or the project lead. # Project Lead +* [Irving Popovetsky](https://github.com/irvingpop) -* [Matthew Walter](https://github.com/ohaiwalt) -# Sergeant +# Past Maintainers +* [Matthew Walter](https://github.com/ohaiwalt) * [Nell Shamrell-Harrington](http://www.github.com/nellshamrell) - -# Maintainers - * [David Marchante](http://www.github.com/cdmarchante) + diff --git a/README.md b/README.md index 73e0dc7..f75796a 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,10 @@ # Operation Code Infra Platform infrastructure for the [Operation Code site](https://operationcode.org/). -[![CircleCI](https://circleci.com/gh/OperationCode/operationcode_infra/tree/master.svg?style=svg)](https://circleci.com/gh/OperationCode/operationcode_infra/tree/master) - -## warning - -This repository is using [ArgoCD](https://argoproj.github.io/argo-cd/) to deploy the Operation Code infrastructure. Changes landed on main in this repository are reflected in the real running infrastructure. - -To set up your workstation to access our Kubernetes cluster, please check the below instructions - ## Setup -### Operation Code's Kubernetes Cluster. -Greetings! Much of Operation Code's web site runs in a [Kubernetes](https://kubernetes.io/) cluster. These instructions will guide you through setting up access to our cluster so you can run rails console, tail logs, and more! - -### Getting access to the cluster -1. Ensure you have [AWS](https://aws.amazon.com) access, and the [aws CLI](https://aws.amazon.com/cli/) is operating correctly -2. Install eksctl: https://eksctl.io/introduction/#installation -3. Run: `eksctl utils write-kubeconfig --region us-east-2 --cluster operationcode-backend` -4. Ensure `kubectl` is working by running `kubectl version`, refer to [Kubectl Install Docs](https://kubernetes.io/docs/tasks/tools/#kubectl) - - Note: if there are issues refer to this [SO Post](https://stackoverflow.com/questions/55360666/kubernetes-kubectl-run-command-not-found) - -5. Verify everything works: `kubectl get namespaces` - +### Operation Code's ECS Cluster +Greetings! Much of Operation Code's web site runs in an AWS ECS cluster. These instructions will guide you through setting up access to our cluster so you can run rails console, tail logs, and more! ## Licensing [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) diff --git a/SETUP.md b/SETUP.md deleted file mode 100644 index 64c806f..0000000 --- a/SETUP.md +++ /dev/null @@ -1,10 +0,0 @@ -# Operation Code's Kubernetes Cluster. - -Greetings! Much of Operation Code's web site runs in a [Kubernetes](https://kubernetes.io/) cluster. These instructions will guide you through setting up access to our cluster so you can run rails console, tail logs, and more! - -# Getting access to the cluster - -1. Ensure you have AWS access, and the aws CLI is operating correctly -2. Install eksctl: https://eksctl.io/introduction/#installation -3. Run: `eksctl utils write-kubeconfig --region us-east-2 --cluster operationcode-backend` -4. Verify everything works: `kubectl get namespaces` diff --git a/kubernetes/README.md b/kubernetes/README.md deleted file mode 100644 index e42b22b..0000000 --- a/kubernetes/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Setup - -To re-create a cluster, everything you need is in the eksctl/ folder. Use eksctl with the `operationcode-backend.yaml` config file to create the cluster. -Then install the controllers: -* aws-alb-ingress-controller -* external-dns -* vertical-pod-autoscaler - diff --git a/kubernetes/argocd/README.md b/kubernetes/argocd/README.md deleted file mode 100644 index b36f1bf..0000000 --- a/kubernetes/argocd/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Setup - -The install.yaml in this folder is slightly modified from the [ArgoCD setup instructions](https://argoproj.github.io/argo-cd/getting_started/), the Redis server has been configured to persist data. Please keep that in mind and don't update from the stock install.yaml file - diff --git a/kubernetes/argocd/application-crd.yaml b/kubernetes/argocd/application-crd.yaml deleted file mode 100644 index 74364ae..0000000 --- a/kubernetes/argocd/application-crd.yaml +++ /dev/null @@ -1,1760 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - labels: - app.kubernetes.io/name: applications.argoproj.io - app.kubernetes.io/part-of: argocd - name: applications.argoproj.io -spec: - group: argoproj.io - names: - kind: Application - listKind: ApplicationList - plural: applications - shortNames: - - app - - apps - singular: application - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.sync.status - name: Sync Status - type: string - - jsonPath: .status.health.status - name: Health Status - type: string - - jsonPath: .status.sync.revision - name: Revision - priority: 10 - type: string - name: v1alpha1 - schema: - openAPIV3Schema: - description: Application is a definition of Application resource. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - operation: - description: Operation contains information about a requested or running operation - properties: - info: - description: Info is a list of informational items for this operation - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - initiatedBy: - description: InitiatedBy contains information about who initiated the operations - properties: - automated: - description: Automated is set to true if operation was initiated automatically by the application controller. - type: boolean - username: - description: Username contains the name of a user who started operation - type: string - type: object - retry: - description: Retry controls the strategy to apply if a sync fails - properties: - backoff: - description: Backoff controls how to backoff on subsequent retries of failed syncs - properties: - duration: - description: Duration is the amount to back off. Default unit is seconds, but could also be a duration (e.g. "2m", "1h") - type: string - factor: - description: Factor is a factor to multiply the base duration after each failed retry - format: int64 - type: integer - maxDuration: - description: MaxDuration is the maximum amount of time allowed for the backoff strategy - type: string - type: object - limit: - description: Limit is the maximum number of attempts for retrying a failed sync. If set to 0, no retries will be performed. - format: int64 - type: integer - type: object - sync: - description: Sync contains parameters for the operation - properties: - dryRun: - description: DryRun specifies to perform a `kubectl apply --dry-run` without actually performing the sync - type: boolean - manifests: - description: Manifests is an optional field that overrides sync source with a local directory for development - items: - type: string - type: array - prune: - description: Prune specifies to delete resources from the cluster that are no longer tracked in git - type: boolean - resources: - description: Resources describes which resources shall be part of the sync - items: - description: SyncOperationResource contains resources to sync. - properties: - group: - type: string - kind: - type: string - name: - type: string - namespace: - type: string - required: - - kind - - name - type: object - type: array - revision: - description: Revision is the revision (Git) or chart version (Helm) which to sync the application to If omitted, will use the revision specified in app spec. - type: string - source: - description: Source overrides the source definition set in the application. This is typically set in a Rollback operation and is nil during a Sync operation - properties: - chart: - description: Chart is a Helm chart name, and must be specified for applications sourced from a Helm repo. - type: string - directory: - description: Directory holds path/directory specific options - properties: - exclude: - description: Exclude contains a glob pattern to match paths against that should be explicitly excluded from being used during manifest generation - type: string - include: - description: Include contains a glob pattern to match paths against that should be explicitly included during manifest generation - type: string - jsonnet: - description: Jsonnet holds options specific to Jsonnet - properties: - extVars: - description: ExtVars is a list of Jsonnet External Variables - items: - description: JsonnetVar represents a variable to be passed to jsonnet during manifest generation - properties: - code: - type: boolean - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - libs: - description: Additional library search dirs - items: - type: string - type: array - tlas: - description: TLAS is a list of Jsonnet Top-level Arguments - items: - description: JsonnetVar represents a variable to be passed to jsonnet during manifest generation - properties: - code: - type: boolean - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - recurse: - description: Recurse specifies whether to scan a directory recursively for manifests - type: boolean - type: object - helm: - description: Helm holds helm specific options - properties: - fileParameters: - description: FileParameters are file parameters to the helm template - items: - description: HelmFileParameter is a file parameter that's passed to helm template during manifest generation - properties: - name: - description: Name is the name of the Helm parameter - type: string - path: - description: Path is the path to the file containing the values for the Helm parameter - type: string - type: object - type: array - parameters: - description: Parameters is a list of Helm parameters which are passed to the helm template command upon manifest generation - items: - description: HelmParameter is a parameter that's passed to helm template during manifest generation - properties: - forceString: - description: ForceString determines whether to tell Helm to interpret booleans and numbers as strings - type: boolean - name: - description: Name is the name of the Helm parameter - type: string - value: - description: Value is the value for the Helm parameter - type: string - type: object - type: array - releaseName: - description: ReleaseName is the Helm release name to use. If omitted it will use the application name - type: string - valueFiles: - description: ValuesFiles is a list of Helm value files to use when generating a template - items: - type: string - type: array - values: - description: Values specifies Helm values to be passed to helm template, typically defined as a block - type: string - version: - description: Version is the Helm version to use for templating (either "2" or "3") - type: string - type: object - ksonnet: - description: Ksonnet holds ksonnet specific options - properties: - environment: - description: Environment is a ksonnet application environment name - type: string - parameters: - description: Parameters are a list of ksonnet component parameter override values - items: - description: KsonnetParameter is a ksonnet component parameter - properties: - component: - type: string - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - kustomize: - description: Kustomize holds kustomize specific options - properties: - commonAnnotations: - additionalProperties: - type: string - description: CommonAnnotations is a list of additional annotations to add to rendered manifests - type: object - commonLabels: - additionalProperties: - type: string - description: CommonLabels is a list of additional labels to add to rendered manifests - type: object - images: - description: Images is a list of Kustomize image override specifications - items: - description: KustomizeImage represents a Kustomize image definition in the format [old_image_name=]: - type: string - type: array - namePrefix: - description: NamePrefix is a prefix appended to resources for Kustomize apps - type: string - nameSuffix: - description: NameSuffix is a suffix appended to resources for Kustomize apps - type: string - version: - description: Version controls which version of Kustomize to use for rendering manifests - type: string - type: object - path: - description: Path is a directory path within the Git repository, and is only valid for applications sourced from Git. - type: string - plugin: - description: ConfigManagementPlugin holds config management plugin specific options - properties: - env: - description: Env is a list of environment variable entries - items: - description: EnvEntry represents an entry in the application's environment - properties: - name: - description: Name is the name of the variable, usually expressed in uppercase - type: string - value: - description: Value is the value of the variable - type: string - required: - - name - - value - type: object - type: array - name: - type: string - type: object - repoURL: - description: RepoURL is the URL to the repository (Git or Helm) that contains the application manifests - type: string - targetRevision: - description: TargetRevision defines the revision of the source to sync the application to. In case of Git, this can be commit, tag, or branch. If omitted, will equal to HEAD. In case of Helm, this is a semver tag for the Chart's version. - type: string - required: - - repoURL - type: object - syncOptions: - description: SyncOptions provide per-sync sync-options, e.g. Validate=false - items: - type: string - type: array - syncStrategy: - description: SyncStrategy describes how to perform the sync - properties: - apply: - description: Apply will perform a `kubectl apply` to perform the sync. - properties: - force: - description: Force indicates whether or not to supply the --force flag to `kubectl apply`. The --force flag deletes and re-create the resource, when PATCH encounters conflict and has retried for 5 times. - type: boolean - type: object - hook: - description: Hook will submit any referenced resources to perform the sync. This is the default strategy - properties: - force: - description: Force indicates whether or not to supply the --force flag to `kubectl apply`. The --force flag deletes and re-create the resource, when PATCH encounters conflict and has retried for 5 times. - type: boolean - type: object - type: object - type: object - type: object - spec: - description: ApplicationSpec represents desired application state. Contains link to repository with application definition and additional parameters link definition revision. - properties: - destination: - description: Destination is a reference to the target Kubernetes server and namespace - properties: - name: - description: Name is an alternate way of specifying the target cluster by its symbolic name - type: string - namespace: - description: Namespace specifies the target namespace for the application's resources. The namespace will only be set for namespace-scoped resources that have not set a value for .metadata.namespace - type: string - server: - description: Server specifies the URL of the target cluster and must be set to the Kubernetes control plane API - type: string - type: object - ignoreDifferences: - description: IgnoreDifferences is a list of resources and their fields which should be ignored during comparison - items: - description: ResourceIgnoreDifferences contains resource filter and list of json paths which should be ignored during comparison with live state. - properties: - group: - type: string - jsonPointers: - items: - type: string - type: array - kind: - type: string - name: - type: string - namespace: - type: string - required: - - jsonPointers - - kind - type: object - type: array - info: - description: Info contains a list of information (URLs, email addresses, and plain text) that relates to the application - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - project: - description: Project is a reference to the project this application belongs to. The empty string means that application belongs to the 'default' project. - type: string - revisionHistoryLimit: - description: RevisionHistoryLimit limits the number of items kept in the application's revision history, which is used for informational purposes as well as for rollbacks to previous versions. This should only be changed in exceptional circumstances. Setting to zero will store no history. This will reduce storage used. Increasing will increase the space used to store the history, so we do not recommend increasing it. Default is 10. - format: int64 - type: integer - source: - description: Source is a reference to the location of the application's manifests or chart - properties: - chart: - description: Chart is a Helm chart name, and must be specified for applications sourced from a Helm repo. - type: string - directory: - description: Directory holds path/directory specific options - properties: - exclude: - description: Exclude contains a glob pattern to match paths against that should be explicitly excluded from being used during manifest generation - type: string - include: - description: Include contains a glob pattern to match paths against that should be explicitly included during manifest generation - type: string - jsonnet: - description: Jsonnet holds options specific to Jsonnet - properties: - extVars: - description: ExtVars is a list of Jsonnet External Variables - items: - description: JsonnetVar represents a variable to be passed to jsonnet during manifest generation - properties: - code: - type: boolean - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - libs: - description: Additional library search dirs - items: - type: string - type: array - tlas: - description: TLAS is a list of Jsonnet Top-level Arguments - items: - description: JsonnetVar represents a variable to be passed to jsonnet during manifest generation - properties: - code: - type: boolean - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - recurse: - description: Recurse specifies whether to scan a directory recursively for manifests - type: boolean - type: object - helm: - description: Helm holds helm specific options - properties: - fileParameters: - description: FileParameters are file parameters to the helm template - items: - description: HelmFileParameter is a file parameter that's passed to helm template during manifest generation - properties: - name: - description: Name is the name of the Helm parameter - type: string - path: - description: Path is the path to the file containing the values for the Helm parameter - type: string - type: object - type: array - parameters: - description: Parameters is a list of Helm parameters which are passed to the helm template command upon manifest generation - items: - description: HelmParameter is a parameter that's passed to helm template during manifest generation - properties: - forceString: - description: ForceString determines whether to tell Helm to interpret booleans and numbers as strings - type: boolean - name: - description: Name is the name of the Helm parameter - type: string - value: - description: Value is the value for the Helm parameter - type: string - type: object - type: array - releaseName: - description: ReleaseName is the Helm release name to use. If omitted it will use the application name - type: string - valueFiles: - description: ValuesFiles is a list of Helm value files to use when generating a template - items: - type: string - type: array - values: - description: Values specifies Helm values to be passed to helm template, typically defined as a block - type: string - version: - description: Version is the Helm version to use for templating (either "2" or "3") - type: string - type: object - ksonnet: - description: Ksonnet holds ksonnet specific options - properties: - environment: - description: Environment is a ksonnet application environment name - type: string - parameters: - description: Parameters are a list of ksonnet component parameter override values - items: - description: KsonnetParameter is a ksonnet component parameter - properties: - component: - type: string - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - kustomize: - description: Kustomize holds kustomize specific options - properties: - commonAnnotations: - additionalProperties: - type: string - description: CommonAnnotations is a list of additional annotations to add to rendered manifests - type: object - commonLabels: - additionalProperties: - type: string - description: CommonLabels is a list of additional labels to add to rendered manifests - type: object - images: - description: Images is a list of Kustomize image override specifications - items: - description: KustomizeImage represents a Kustomize image definition in the format [old_image_name=]: - type: string - type: array - namePrefix: - description: NamePrefix is a prefix appended to resources for Kustomize apps - type: string - nameSuffix: - description: NameSuffix is a suffix appended to resources for Kustomize apps - type: string - version: - description: Version controls which version of Kustomize to use for rendering manifests - type: string - type: object - path: - description: Path is a directory path within the Git repository, and is only valid for applications sourced from Git. - type: string - plugin: - description: ConfigManagementPlugin holds config management plugin specific options - properties: - env: - description: Env is a list of environment variable entries - items: - description: EnvEntry represents an entry in the application's environment - properties: - name: - description: Name is the name of the variable, usually expressed in uppercase - type: string - value: - description: Value is the value of the variable - type: string - required: - - name - - value - type: object - type: array - name: - type: string - type: object - repoURL: - description: RepoURL is the URL to the repository (Git or Helm) that contains the application manifests - type: string - targetRevision: - description: TargetRevision defines the revision of the source to sync the application to. In case of Git, this can be commit, tag, or branch. If omitted, will equal to HEAD. In case of Helm, this is a semver tag for the Chart's version. - type: string - required: - - repoURL - type: object - syncPolicy: - description: SyncPolicy controls when and how a sync will be performed - properties: - automated: - description: Automated will keep an application synced to the target revision - properties: - allowEmpty: - description: 'AllowEmpty allows apps have zero live resources (default: false)' - type: boolean - prune: - description: 'Prune specifies whether to delete resources from the cluster that are not found in the sources anymore as part of automated sync (default: false)' - type: boolean - selfHeal: - description: 'SelfHeal specifes whether to revert resources back to their desired state upon modification in the cluster (default: false)' - type: boolean - type: object - retry: - description: Retry controls failed sync retry behavior - properties: - backoff: - description: Backoff controls how to backoff on subsequent retries of failed syncs - properties: - duration: - description: Duration is the amount to back off. Default unit is seconds, but could also be a duration (e.g. "2m", "1h") - type: string - factor: - description: Factor is a factor to multiply the base duration after each failed retry - format: int64 - type: integer - maxDuration: - description: MaxDuration is the maximum amount of time allowed for the backoff strategy - type: string - type: object - limit: - description: Limit is the maximum number of attempts for retrying a failed sync. If set to 0, no retries will be performed. - format: int64 - type: integer - type: object - syncOptions: - description: Options allow you to specify whole app sync-options - items: - type: string - type: array - type: object - required: - - destination - - project - - source - type: object - status: - description: ApplicationStatus contains status information for the application - properties: - conditions: - description: Conditions is a list of currently observed application conditions - items: - description: ApplicationCondition contains details about an application condition, which is usally an error or warning - properties: - lastTransitionTime: - description: LastTransitionTime is the time the condition was last observed - format: date-time - type: string - message: - description: Message contains human-readable message indicating details about condition - type: string - type: - description: Type is an application condition type - type: string - required: - - message - - type - type: object - type: array - health: - description: Health contains information about the application's current health status - properties: - message: - description: Message is a human-readable informational message describing the health status - type: string - status: - description: Status holds the status code of the application or resource - type: string - type: object - history: - description: History contains information about the application's sync history - items: - description: RevisionHistory contains history information about a previous sync - properties: - deployStartedAt: - description: DeployStartedAt holds the time the sync operation started - format: date-time - type: string - deployedAt: - description: DeployedAt holds the time the sync operation completed - format: date-time - type: string - id: - description: ID is an auto incrementing identifier of the RevisionHistory - format: int64 - type: integer - revision: - description: Revision holds the revision the sync was performed against - type: string - source: - description: Source is a reference to the application source used for the sync operation - properties: - chart: - description: Chart is a Helm chart name, and must be specified for applications sourced from a Helm repo. - type: string - directory: - description: Directory holds path/directory specific options - properties: - exclude: - description: Exclude contains a glob pattern to match paths against that should be explicitly excluded from being used during manifest generation - type: string - include: - description: Include contains a glob pattern to match paths against that should be explicitly included during manifest generation - type: string - jsonnet: - description: Jsonnet holds options specific to Jsonnet - properties: - extVars: - description: ExtVars is a list of Jsonnet External Variables - items: - description: JsonnetVar represents a variable to be passed to jsonnet during manifest generation - properties: - code: - type: boolean - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - libs: - description: Additional library search dirs - items: - type: string - type: array - tlas: - description: TLAS is a list of Jsonnet Top-level Arguments - items: - description: JsonnetVar represents a variable to be passed to jsonnet during manifest generation - properties: - code: - type: boolean - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - recurse: - description: Recurse specifies whether to scan a directory recursively for manifests - type: boolean - type: object - helm: - description: Helm holds helm specific options - properties: - fileParameters: - description: FileParameters are file parameters to the helm template - items: - description: HelmFileParameter is a file parameter that's passed to helm template during manifest generation - properties: - name: - description: Name is the name of the Helm parameter - type: string - path: - description: Path is the path to the file containing the values for the Helm parameter - type: string - type: object - type: array - parameters: - description: Parameters is a list of Helm parameters which are passed to the helm template command upon manifest generation - items: - description: HelmParameter is a parameter that's passed to helm template during manifest generation - properties: - forceString: - description: ForceString determines whether to tell Helm to interpret booleans and numbers as strings - type: boolean - name: - description: Name is the name of the Helm parameter - type: string - value: - description: Value is the value for the Helm parameter - type: string - type: object - type: array - releaseName: - description: ReleaseName is the Helm release name to use. If omitted it will use the application name - type: string - valueFiles: - description: ValuesFiles is a list of Helm value files to use when generating a template - items: - type: string - type: array - values: - description: Values specifies Helm values to be passed to helm template, typically defined as a block - type: string - version: - description: Version is the Helm version to use for templating (either "2" or "3") - type: string - type: object - ksonnet: - description: Ksonnet holds ksonnet specific options - properties: - environment: - description: Environment is a ksonnet application environment name - type: string - parameters: - description: Parameters are a list of ksonnet component parameter override values - items: - description: KsonnetParameter is a ksonnet component parameter - properties: - component: - type: string - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - kustomize: - description: Kustomize holds kustomize specific options - properties: - commonAnnotations: - additionalProperties: - type: string - description: CommonAnnotations is a list of additional annotations to add to rendered manifests - type: object - commonLabels: - additionalProperties: - type: string - description: CommonLabels is a list of additional labels to add to rendered manifests - type: object - images: - description: Images is a list of Kustomize image override specifications - items: - description: KustomizeImage represents a Kustomize image definition in the format [old_image_name=]: - type: string - type: array - namePrefix: - description: NamePrefix is a prefix appended to resources for Kustomize apps - type: string - nameSuffix: - description: NameSuffix is a suffix appended to resources for Kustomize apps - type: string - version: - description: Version controls which version of Kustomize to use for rendering manifests - type: string - type: object - path: - description: Path is a directory path within the Git repository, and is only valid for applications sourced from Git. - type: string - plugin: - description: ConfigManagementPlugin holds config management plugin specific options - properties: - env: - description: Env is a list of environment variable entries - items: - description: EnvEntry represents an entry in the application's environment - properties: - name: - description: Name is the name of the variable, usually expressed in uppercase - type: string - value: - description: Value is the value of the variable - type: string - required: - - name - - value - type: object - type: array - name: - type: string - type: object - repoURL: - description: RepoURL is the URL to the repository (Git or Helm) that contains the application manifests - type: string - targetRevision: - description: TargetRevision defines the revision of the source to sync the application to. In case of Git, this can be commit, tag, or branch. If omitted, will equal to HEAD. In case of Helm, this is a semver tag for the Chart's version. - type: string - required: - - repoURL - type: object - required: - - deployedAt - - id - - revision - type: object - type: array - observedAt: - description: 'ObservedAt indicates when the application state was updated without querying latest git state Deprecated: controller no longer updates ObservedAt field' - format: date-time - type: string - operationState: - description: OperationState contains information about any ongoing operations, such as a sync - properties: - finishedAt: - description: FinishedAt contains time of operation completion - format: date-time - type: string - message: - description: Message holds any pertinent messages when attempting to perform operation (typically errors). - type: string - operation: - description: Operation is the original requested operation - properties: - info: - description: Info is a list of informational items for this operation - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - initiatedBy: - description: InitiatedBy contains information about who initiated the operations - properties: - automated: - description: Automated is set to true if operation was initiated automatically by the application controller. - type: boolean - username: - description: Username contains the name of a user who started operation - type: string - type: object - retry: - description: Retry controls the strategy to apply if a sync fails - properties: - backoff: - description: Backoff controls how to backoff on subsequent retries of failed syncs - properties: - duration: - description: Duration is the amount to back off. Default unit is seconds, but could also be a duration (e.g. "2m", "1h") - type: string - factor: - description: Factor is a factor to multiply the base duration after each failed retry - format: int64 - type: integer - maxDuration: - description: MaxDuration is the maximum amount of time allowed for the backoff strategy - type: string - type: object - limit: - description: Limit is the maximum number of attempts for retrying a failed sync. If set to 0, no retries will be performed. - format: int64 - type: integer - type: object - sync: - description: Sync contains parameters for the operation - properties: - dryRun: - description: DryRun specifies to perform a `kubectl apply --dry-run` without actually performing the sync - type: boolean - manifests: - description: Manifests is an optional field that overrides sync source with a local directory for development - items: - type: string - type: array - prune: - description: Prune specifies to delete resources from the cluster that are no longer tracked in git - type: boolean - resources: - description: Resources describes which resources shall be part of the sync - items: - description: SyncOperationResource contains resources to sync. - properties: - group: - type: string - kind: - type: string - name: - type: string - namespace: - type: string - required: - - kind - - name - type: object - type: array - revision: - description: Revision is the revision (Git) or chart version (Helm) which to sync the application to If omitted, will use the revision specified in app spec. - type: string - source: - description: Source overrides the source definition set in the application. This is typically set in a Rollback operation and is nil during a Sync operation - properties: - chart: - description: Chart is a Helm chart name, and must be specified for applications sourced from a Helm repo. - type: string - directory: - description: Directory holds path/directory specific options - properties: - exclude: - description: Exclude contains a glob pattern to match paths against that should be explicitly excluded from being used during manifest generation - type: string - include: - description: Include contains a glob pattern to match paths against that should be explicitly included during manifest generation - type: string - jsonnet: - description: Jsonnet holds options specific to Jsonnet - properties: - extVars: - description: ExtVars is a list of Jsonnet External Variables - items: - description: JsonnetVar represents a variable to be passed to jsonnet during manifest generation - properties: - code: - type: boolean - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - libs: - description: Additional library search dirs - items: - type: string - type: array - tlas: - description: TLAS is a list of Jsonnet Top-level Arguments - items: - description: JsonnetVar represents a variable to be passed to jsonnet during manifest generation - properties: - code: - type: boolean - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - recurse: - description: Recurse specifies whether to scan a directory recursively for manifests - type: boolean - type: object - helm: - description: Helm holds helm specific options - properties: - fileParameters: - description: FileParameters are file parameters to the helm template - items: - description: HelmFileParameter is a file parameter that's passed to helm template during manifest generation - properties: - name: - description: Name is the name of the Helm parameter - type: string - path: - description: Path is the path to the file containing the values for the Helm parameter - type: string - type: object - type: array - parameters: - description: Parameters is a list of Helm parameters which are passed to the helm template command upon manifest generation - items: - description: HelmParameter is a parameter that's passed to helm template during manifest generation - properties: - forceString: - description: ForceString determines whether to tell Helm to interpret booleans and numbers as strings - type: boolean - name: - description: Name is the name of the Helm parameter - type: string - value: - description: Value is the value for the Helm parameter - type: string - type: object - type: array - releaseName: - description: ReleaseName is the Helm release name to use. If omitted it will use the application name - type: string - valueFiles: - description: ValuesFiles is a list of Helm value files to use when generating a template - items: - type: string - type: array - values: - description: Values specifies Helm values to be passed to helm template, typically defined as a block - type: string - version: - description: Version is the Helm version to use for templating (either "2" or "3") - type: string - type: object - ksonnet: - description: Ksonnet holds ksonnet specific options - properties: - environment: - description: Environment is a ksonnet application environment name - type: string - parameters: - description: Parameters are a list of ksonnet component parameter override values - items: - description: KsonnetParameter is a ksonnet component parameter - properties: - component: - type: string - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - kustomize: - description: Kustomize holds kustomize specific options - properties: - commonAnnotations: - additionalProperties: - type: string - description: CommonAnnotations is a list of additional annotations to add to rendered manifests - type: object - commonLabels: - additionalProperties: - type: string - description: CommonLabels is a list of additional labels to add to rendered manifests - type: object - images: - description: Images is a list of Kustomize image override specifications - items: - description: KustomizeImage represents a Kustomize image definition in the format [old_image_name=]: - type: string - type: array - namePrefix: - description: NamePrefix is a prefix appended to resources for Kustomize apps - type: string - nameSuffix: - description: NameSuffix is a suffix appended to resources for Kustomize apps - type: string - version: - description: Version controls which version of Kustomize to use for rendering manifests - type: string - type: object - path: - description: Path is a directory path within the Git repository, and is only valid for applications sourced from Git. - type: string - plugin: - description: ConfigManagementPlugin holds config management plugin specific options - properties: - env: - description: Env is a list of environment variable entries - items: - description: EnvEntry represents an entry in the application's environment - properties: - name: - description: Name is the name of the variable, usually expressed in uppercase - type: string - value: - description: Value is the value of the variable - type: string - required: - - name - - value - type: object - type: array - name: - type: string - type: object - repoURL: - description: RepoURL is the URL to the repository (Git or Helm) that contains the application manifests - type: string - targetRevision: - description: TargetRevision defines the revision of the source to sync the application to. In case of Git, this can be commit, tag, or branch. If omitted, will equal to HEAD. In case of Helm, this is a semver tag for the Chart's version. - type: string - required: - - repoURL - type: object - syncOptions: - description: SyncOptions provide per-sync sync-options, e.g. Validate=false - items: - type: string - type: array - syncStrategy: - description: SyncStrategy describes how to perform the sync - properties: - apply: - description: Apply will perform a `kubectl apply` to perform the sync. - properties: - force: - description: Force indicates whether or not to supply the --force flag to `kubectl apply`. The --force flag deletes and re-create the resource, when PATCH encounters conflict and has retried for 5 times. - type: boolean - type: object - hook: - description: Hook will submit any referenced resources to perform the sync. This is the default strategy - properties: - force: - description: Force indicates whether or not to supply the --force flag to `kubectl apply`. The --force flag deletes and re-create the resource, when PATCH encounters conflict and has retried for 5 times. - type: boolean - type: object - type: object - type: object - type: object - phase: - description: Phase is the current phase of the operation - type: string - retryCount: - description: RetryCount contains time of operation retries - format: int64 - type: integer - startedAt: - description: StartedAt contains time of operation start - format: date-time - type: string - syncResult: - description: SyncResult is the result of a Sync operation - properties: - resources: - description: Resources contains a list of sync result items for each individual resource in a sync operation - items: - description: ResourceResult holds the operation result details of a specific resource - properties: - group: - description: Group specifies the API group of the resource - type: string - hookPhase: - description: HookPhase contains the state of any operation associated with this resource OR hook This can also contain values for non-hook resources. - type: string - hookType: - description: HookType specifies the type of the hook. Empty for non-hook resources - type: string - kind: - description: Kind specifies the API kind of the resource - type: string - message: - description: Message contains an informational or error message for the last sync OR operation - type: string - name: - description: Name specifies the name of the resource - type: string - namespace: - description: Namespace specifies the target namespace of the resource - type: string - status: - description: Status holds the final result of the sync. Will be empty if the resources is yet to be applied/pruned and is always zero-value for hooks - type: string - syncPhase: - description: SyncPhase indicates the particular phase of the sync that this result was acquired in - type: string - version: - description: Version specifies the API version of the resource - type: string - required: - - group - - kind - - name - - namespace - - version - type: object - type: array - revision: - description: Revision holds the revision this sync operation was performed to - type: string - source: - description: Source records the application source information of the sync, used for comparing auto-sync - properties: - chart: - description: Chart is a Helm chart name, and must be specified for applications sourced from a Helm repo. - type: string - directory: - description: Directory holds path/directory specific options - properties: - exclude: - description: Exclude contains a glob pattern to match paths against that should be explicitly excluded from being used during manifest generation - type: string - include: - description: Include contains a glob pattern to match paths against that should be explicitly included during manifest generation - type: string - jsonnet: - description: Jsonnet holds options specific to Jsonnet - properties: - extVars: - description: ExtVars is a list of Jsonnet External Variables - items: - description: JsonnetVar represents a variable to be passed to jsonnet during manifest generation - properties: - code: - type: boolean - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - libs: - description: Additional library search dirs - items: - type: string - type: array - tlas: - description: TLAS is a list of Jsonnet Top-level Arguments - items: - description: JsonnetVar represents a variable to be passed to jsonnet during manifest generation - properties: - code: - type: boolean - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - recurse: - description: Recurse specifies whether to scan a directory recursively for manifests - type: boolean - type: object - helm: - description: Helm holds helm specific options - properties: - fileParameters: - description: FileParameters are file parameters to the helm template - items: - description: HelmFileParameter is a file parameter that's passed to helm template during manifest generation - properties: - name: - description: Name is the name of the Helm parameter - type: string - path: - description: Path is the path to the file containing the values for the Helm parameter - type: string - type: object - type: array - parameters: - description: Parameters is a list of Helm parameters which are passed to the helm template command upon manifest generation - items: - description: HelmParameter is a parameter that's passed to helm template during manifest generation - properties: - forceString: - description: ForceString determines whether to tell Helm to interpret booleans and numbers as strings - type: boolean - name: - description: Name is the name of the Helm parameter - type: string - value: - description: Value is the value for the Helm parameter - type: string - type: object - type: array - releaseName: - description: ReleaseName is the Helm release name to use. If omitted it will use the application name - type: string - valueFiles: - description: ValuesFiles is a list of Helm value files to use when generating a template - items: - type: string - type: array - values: - description: Values specifies Helm values to be passed to helm template, typically defined as a block - type: string - version: - description: Version is the Helm version to use for templating (either "2" or "3") - type: string - type: object - ksonnet: - description: Ksonnet holds ksonnet specific options - properties: - environment: - description: Environment is a ksonnet application environment name - type: string - parameters: - description: Parameters are a list of ksonnet component parameter override values - items: - description: KsonnetParameter is a ksonnet component parameter - properties: - component: - type: string - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - kustomize: - description: Kustomize holds kustomize specific options - properties: - commonAnnotations: - additionalProperties: - type: string - description: CommonAnnotations is a list of additional annotations to add to rendered manifests - type: object - commonLabels: - additionalProperties: - type: string - description: CommonLabels is a list of additional labels to add to rendered manifests - type: object - images: - description: Images is a list of Kustomize image override specifications - items: - description: KustomizeImage represents a Kustomize image definition in the format [old_image_name=]: - type: string - type: array - namePrefix: - description: NamePrefix is a prefix appended to resources for Kustomize apps - type: string - nameSuffix: - description: NameSuffix is a suffix appended to resources for Kustomize apps - type: string - version: - description: Version controls which version of Kustomize to use for rendering manifests - type: string - type: object - path: - description: Path is a directory path within the Git repository, and is only valid for applications sourced from Git. - type: string - plugin: - description: ConfigManagementPlugin holds config management plugin specific options - properties: - env: - description: Env is a list of environment variable entries - items: - description: EnvEntry represents an entry in the application's environment - properties: - name: - description: Name is the name of the variable, usually expressed in uppercase - type: string - value: - description: Value is the value of the variable - type: string - required: - - name - - value - type: object - type: array - name: - type: string - type: object - repoURL: - description: RepoURL is the URL to the repository (Git or Helm) that contains the application manifests - type: string - targetRevision: - description: TargetRevision defines the revision of the source to sync the application to. In case of Git, this can be commit, tag, or branch. If omitted, will equal to HEAD. In case of Helm, this is a semver tag for the Chart's version. - type: string - required: - - repoURL - type: object - required: - - revision - type: object - required: - - operation - - phase - - startedAt - type: object - reconciledAt: - description: ReconciledAt indicates when the application state was reconciled using the latest git version - format: date-time - type: string - resources: - description: Resources is a list of Kubernetes resources managed by this application - items: - description: 'ResourceStatus holds the current sync and health status of a resource TODO: describe members of this type' - properties: - group: - type: string - health: - description: HealthStatus contains information about the currently observed health state of an application or resource - properties: - message: - description: Message is a human-readable informational message describing the health status - type: string - status: - description: Status holds the status code of the application or resource - type: string - type: object - hook: - type: boolean - kind: - type: string - name: - type: string - namespace: - type: string - requiresPruning: - type: boolean - status: - description: SyncStatusCode is a type which represents possible comparison results - type: string - version: - type: string - type: object - type: array - sourceType: - description: SourceType specifies the type of this application - type: string - summary: - description: Summary contains a list of URLs and container images used by this application - properties: - externalURLs: - description: ExternalURLs holds all external URLs of application child resources. - items: - type: string - type: array - images: - description: Images holds all images of application child resources. - items: - type: string - type: array - type: object - sync: - description: Sync contains information about the application's current sync status - properties: - comparedTo: - description: ComparedTo contains information about what has been compared - properties: - destination: - description: Destination is a reference to the application's destination used for comparison - properties: - name: - description: Name is an alternate way of specifying the target cluster by its symbolic name - type: string - namespace: - description: Namespace specifies the target namespace for the application's resources. The namespace will only be set for namespace-scoped resources that have not set a value for .metadata.namespace - type: string - server: - description: Server specifies the URL of the target cluster and must be set to the Kubernetes control plane API - type: string - type: object - source: - description: Source is a reference to the application's source used for comparison - properties: - chart: - description: Chart is a Helm chart name, and must be specified for applications sourced from a Helm repo. - type: string - directory: - description: Directory holds path/directory specific options - properties: - exclude: - description: Exclude contains a glob pattern to match paths against that should be explicitly excluded from being used during manifest generation - type: string - include: - description: Include contains a glob pattern to match paths against that should be explicitly included during manifest generation - type: string - jsonnet: - description: Jsonnet holds options specific to Jsonnet - properties: - extVars: - description: ExtVars is a list of Jsonnet External Variables - items: - description: JsonnetVar represents a variable to be passed to jsonnet during manifest generation - properties: - code: - type: boolean - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - libs: - description: Additional library search dirs - items: - type: string - type: array - tlas: - description: TLAS is a list of Jsonnet Top-level Arguments - items: - description: JsonnetVar represents a variable to be passed to jsonnet during manifest generation - properties: - code: - type: boolean - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - recurse: - description: Recurse specifies whether to scan a directory recursively for manifests - type: boolean - type: object - helm: - description: Helm holds helm specific options - properties: - fileParameters: - description: FileParameters are file parameters to the helm template - items: - description: HelmFileParameter is a file parameter that's passed to helm template during manifest generation - properties: - name: - description: Name is the name of the Helm parameter - type: string - path: - description: Path is the path to the file containing the values for the Helm parameter - type: string - type: object - type: array - parameters: - description: Parameters is a list of Helm parameters which are passed to the helm template command upon manifest generation - items: - description: HelmParameter is a parameter that's passed to helm template during manifest generation - properties: - forceString: - description: ForceString determines whether to tell Helm to interpret booleans and numbers as strings - type: boolean - name: - description: Name is the name of the Helm parameter - type: string - value: - description: Value is the value for the Helm parameter - type: string - type: object - type: array - releaseName: - description: ReleaseName is the Helm release name to use. If omitted it will use the application name - type: string - valueFiles: - description: ValuesFiles is a list of Helm value files to use when generating a template - items: - type: string - type: array - values: - description: Values specifies Helm values to be passed to helm template, typically defined as a block - type: string - version: - description: Version is the Helm version to use for templating (either "2" or "3") - type: string - type: object - ksonnet: - description: Ksonnet holds ksonnet specific options - properties: - environment: - description: Environment is a ksonnet application environment name - type: string - parameters: - description: Parameters are a list of ksonnet component parameter override values - items: - description: KsonnetParameter is a ksonnet component parameter - properties: - component: - type: string - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - kustomize: - description: Kustomize holds kustomize specific options - properties: - commonAnnotations: - additionalProperties: - type: string - description: CommonAnnotations is a list of additional annotations to add to rendered manifests - type: object - commonLabels: - additionalProperties: - type: string - description: CommonLabels is a list of additional labels to add to rendered manifests - type: object - images: - description: Images is a list of Kustomize image override specifications - items: - description: KustomizeImage represents a Kustomize image definition in the format [old_image_name=]: - type: string - type: array - namePrefix: - description: NamePrefix is a prefix appended to resources for Kustomize apps - type: string - nameSuffix: - description: NameSuffix is a suffix appended to resources for Kustomize apps - type: string - version: - description: Version controls which version of Kustomize to use for rendering manifests - type: string - type: object - path: - description: Path is a directory path within the Git repository, and is only valid for applications sourced from Git. - type: string - plugin: - description: ConfigManagementPlugin holds config management plugin specific options - properties: - env: - description: Env is a list of environment variable entries - items: - description: EnvEntry represents an entry in the application's environment - properties: - name: - description: Name is the name of the variable, usually expressed in uppercase - type: string - value: - description: Value is the value of the variable - type: string - required: - - name - - value - type: object - type: array - name: - type: string - type: object - repoURL: - description: RepoURL is the URL to the repository (Git or Helm) that contains the application manifests - type: string - targetRevision: - description: TargetRevision defines the revision of the source to sync the application to. In case of Git, this can be commit, tag, or branch. If omitted, will equal to HEAD. In case of Helm, this is a semver tag for the Chart's version. - type: string - required: - - repoURL - type: object - required: - - destination - - source - type: object - revision: - description: Revision contains information about the revision the comparison has been performed to - type: string - status: - description: Status is the sync state of the comparison - type: string - required: - - status - type: object - type: object - required: - - metadata - - spec - type: object - served: true - storage: true - subresources: {} diff --git a/kubernetes/argocd/appproject-crd.yaml b/kubernetes/argocd/appproject-crd.yaml deleted file mode 100644 index 7bb0965..0000000 --- a/kubernetes/argocd/appproject-crd.yaml +++ /dev/null @@ -1,257 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - labels: - app.kubernetes.io/name: appprojects.argoproj.io - app.kubernetes.io/part-of: argocd - name: appprojects.argoproj.io -spec: - group: argoproj.io - names: - kind: AppProject - listKind: AppProjectList - plural: appprojects - shortNames: - - appproj - - appprojs - singular: appproject - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: 'AppProject provides a logical grouping of applications, providing controls for: * where the apps may deploy to (cluster whitelist) * what may be deployed (repository whitelist, resource whitelist/blacklist) * who can access these applications (roles, OIDC group claims bindings) * and what they can do (RBAC policies) * automation access to these roles (JWT tokens)' - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: AppProjectSpec is the specification of an AppProject - properties: - clusterResourceBlacklist: - description: ClusterResourceBlacklist contains list of blacklisted cluster level resources - items: - description: GroupKind specifies a Group and a Kind, but does not force a version. This is useful for identifying concepts during lookup stages without having partially valid types - properties: - group: - type: string - kind: - type: string - required: - - group - - kind - type: object - type: array - clusterResourceWhitelist: - description: ClusterResourceWhitelist contains list of whitelisted cluster level resources - items: - description: GroupKind specifies a Group and a Kind, but does not force a version. This is useful for identifying concepts during lookup stages without having partially valid types - properties: - group: - type: string - kind: - type: string - required: - - group - - kind - type: object - type: array - description: - description: Description contains optional project description - type: string - destinations: - description: Destinations contains list of destinations available for deployment - items: - description: ApplicationDestination holds information about the application's destination - properties: - name: - description: Name is an alternate way of specifying the target cluster by its symbolic name - type: string - namespace: - description: Namespace specifies the target namespace for the application's resources. The namespace will only be set for namespace-scoped resources that have not set a value for .metadata.namespace - type: string - server: - description: Server specifies the URL of the target cluster and must be set to the Kubernetes control plane API - type: string - type: object - type: array - namespaceResourceBlacklist: - description: NamespaceResourceBlacklist contains list of blacklisted namespace level resources - items: - description: GroupKind specifies a Group and a Kind, but does not force a version. This is useful for identifying concepts during lookup stages without having partially valid types - properties: - group: - type: string - kind: - type: string - required: - - group - - kind - type: object - type: array - namespaceResourceWhitelist: - description: NamespaceResourceWhitelist contains list of whitelisted namespace level resources - items: - description: GroupKind specifies a Group and a Kind, but does not force a version. This is useful for identifying concepts during lookup stages without having partially valid types - properties: - group: - type: string - kind: - type: string - required: - - group - - kind - type: object - type: array - orphanedResources: - description: OrphanedResources specifies if controller should monitor orphaned resources of apps in this project - properties: - ignore: - description: Ignore contains a list of resources that are to be excluded from orphaned resources monitoring - items: - description: OrphanedResourceKey is a reference to a resource to be ignored from - properties: - group: - type: string - kind: - type: string - name: - type: string - type: object - type: array - warn: - description: Warn indicates if warning condition should be created for apps which have orphaned resources - type: boolean - type: object - roles: - description: Roles are user defined RBAC roles associated with this project - items: - description: ProjectRole represents a role that has access to a project - properties: - description: - description: Description is a description of the role - type: string - groups: - description: Groups are a list of OIDC group claims bound to this role - items: - type: string - type: array - jwtTokens: - description: JWTTokens are a list of generated JWT tokens bound to this role - items: - description: JWTToken holds the issuedAt and expiresAt values of a token - properties: - exp: - format: int64 - type: integer - iat: - format: int64 - type: integer - id: - type: string - required: - - iat - type: object - type: array - name: - description: Name is a name for this role - type: string - policies: - description: Policies Stores a list of casbin formated strings that define access policies for the role in the project - items: - type: string - type: array - required: - - name - type: object - type: array - signatureKeys: - description: SignatureKeys contains a list of PGP key IDs that commits in Git must be signed with in order to be allowed for sync - items: - description: SignatureKey is the specification of a key required to verify commit signatures with - properties: - keyID: - description: The ID of the key in hexadecimal notation - type: string - required: - - keyID - type: object - type: array - sourceRepos: - description: SourceRepos contains list of repository URLs which can be used for deployment - items: - type: string - type: array - syncWindows: - description: SyncWindows controls when syncs can be run for apps in this project - items: - description: SyncWindow contains the kind, time, duration and attributes that are used to assign the syncWindows to apps - properties: - applications: - description: Applications contains a list of applications that the window will apply to - items: - type: string - type: array - clusters: - description: Clusters contains a list of clusters that the window will apply to - items: - type: string - type: array - duration: - description: Duration is the amount of time the sync window will be open - type: string - kind: - description: Kind defines if the window allows or blocks syncs - type: string - manualSync: - description: ManualSync enables manual syncs when they would otherwise be blocked - type: boolean - namespaces: - description: Namespaces contains a list of namespaces that the window will apply to - items: - type: string - type: array - schedule: - description: Schedule is the time the window will begin, specified in cron format - type: string - type: object - type: array - type: object - status: - description: AppProjectStatus contains status information for AppProject CRs - properties: - jwtTokensByRole: - additionalProperties: - description: JWTTokens represents a list of JWT tokens - properties: - items: - items: - description: JWTToken holds the issuedAt and expiresAt values of a token - properties: - exp: - format: int64 - type: integer - iat: - format: int64 - type: integer - id: - type: string - required: - - iat - type: object - type: array - type: object - description: JWTTokensByRole contains a list of JWT tokens issued for a given role - type: object - type: object - required: - - metadata - - spec - type: object - served: true - storage: true diff --git a/kubernetes/argocd/install.yaml b/kubernetes/argocd/install.yaml deleted file mode 100644 index 7f1e321..0000000 --- a/kubernetes/argocd/install.yaml +++ /dev/null @@ -1,913 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: application-controller - app.kubernetes.io/name: argocd-application-controller - app.kubernetes.io/part-of: argocd - name: argocd-application-controller ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: dex-server - app.kubernetes.io/name: argocd-dex-server - app.kubernetes.io/part-of: argocd - name: argocd-dex-server ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: redis - app.kubernetes.io/name: argocd-redis - app.kubernetes.io/part-of: argocd - name: argocd-redis ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: server - app.kubernetes.io/name: argocd-server - app.kubernetes.io/part-of: argocd - name: argocd-server ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: application-controller - app.kubernetes.io/name: argocd-application-controller - app.kubernetes.io/part-of: argocd - name: argocd-application-controller -rules: -- apiGroups: - - "" - resources: - - secrets - - configmaps - verbs: - - get - - list - - watch -- apiGroups: - - argoproj.io - resources: - - applications - - appprojects - verbs: - - create - - get - - list - - watch - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: dex-server - app.kubernetes.io/name: argocd-dex-server - app.kubernetes.io/part-of: argocd - name: argocd-dex-server -rules: -- apiGroups: - - "" - resources: - - secrets - - configmaps - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: redis - app.kubernetes.io/name: argocd-redis - app.kubernetes.io/part-of: argocd - name: argocd-redis -rules: -- apiGroups: - - security.openshift.io - resourceNames: - - nonroot - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: server - app.kubernetes.io/name: argocd-server - app.kubernetes.io/part-of: argocd - name: argocd-server -rules: -- apiGroups: - - "" - resources: - - secrets - - configmaps - verbs: - - create - - get - - list - - watch - - update - - patch - - delete -- apiGroups: - - argoproj.io - resources: - - applications - - appprojects - verbs: - - create - - get - - list - - watch - - update - - delete - - patch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: application-controller - app.kubernetes.io/name: argocd-application-controller - app.kubernetes.io/part-of: argocd - name: argocd-application-controller -rules: -- apiGroups: - - '*' - resources: - - '*' - verbs: - - '*' -- nonResourceURLs: - - '*' - verbs: - - '*' ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: server - app.kubernetes.io/name: argocd-server - app.kubernetes.io/part-of: argocd - name: argocd-server -rules: -- apiGroups: - - '*' - resources: - - '*' - verbs: - - delete - - get - - patch -- apiGroups: - - "" - resources: - - events - verbs: - - list -- apiGroups: - - "" - resources: - - pods - - pods/log - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: application-controller - app.kubernetes.io/name: argocd-application-controller - app.kubernetes.io/part-of: argocd - name: argocd-application-controller -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: argocd-application-controller -subjects: -- kind: ServiceAccount - name: argocd-application-controller ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: dex-server - app.kubernetes.io/name: argocd-dex-server - app.kubernetes.io/part-of: argocd - name: argocd-dex-server -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: argocd-dex-server -subjects: -- kind: ServiceAccount - name: argocd-dex-server ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: redis - app.kubernetes.io/name: argocd-redis - app.kubernetes.io/part-of: argocd - name: argocd-redis -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: argocd-redis -subjects: -- kind: ServiceAccount - name: argocd-redis ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: server - app.kubernetes.io/name: argocd-server - app.kubernetes.io/part-of: argocd - name: argocd-server -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: argocd-server -subjects: -- kind: ServiceAccount - name: argocd-server ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: application-controller - app.kubernetes.io/name: argocd-application-controller - app.kubernetes.io/part-of: argocd - name: argocd-application-controller -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: argocd-application-controller -subjects: -- kind: ServiceAccount - name: argocd-application-controller - namespace: argocd ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: server - app.kubernetes.io/name: argocd-server - app.kubernetes.io/part-of: argocd - name: argocd-server -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: argocd-server -subjects: -- kind: ServiceAccount - name: argocd-server - namespace: argocd ---- -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/name: argocd-cm - app.kubernetes.io/part-of: argocd - name: argocd-cm ---- -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/name: argocd-gpg-keys-cm - app.kubernetes.io/part-of: argocd - name: argocd-gpg-keys-cm ---- -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/name: argocd-rbac-cm - app.kubernetes.io/part-of: argocd - name: argocd-rbac-cm ---- -apiVersion: v1 -data: - ssh_known_hosts: | - bitbucket.org ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAubiN81eDcafrgMeLzaFPsw2kNvEcqTKl/VqLat/MaB33pZy0y3rJZtnqwR2qOOvbwKZYKiEO1O6VqNEBxKvJJelCq0dTXWT5pbO2gDXC6h6QDXCaHo6pOHGPUy+YBaGQRGuSusMEASYiWunYN0vCAI8QaXnWMXNMdFP3jHAJH0eDsoiGnLPBlBp4TNm6rYI74nMzgz3B9IikW4WVK+dc8KZJZWYjAuORU3jc1c/NPskD2ASinf8v3xnfXeukU0sJ5N6m5E8VLjObPEO+mN2t/FZTMZLiFqPWc/ALSqnMnnhwrNi2rbfg/rd/IpL8Le3pSBne8+seeFVBoGqzHM9yXw== - github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== - gitlab.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFSMqzJeV9rUzU4kWitGjeR4PWSa29SPqJ1fVkhtj3Hw9xjLVXVYrU9QlYWrOLXBpQ6KWjbjTDTdDkoohFzgbEY= - gitlab.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf - gitlab.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsj2bNKTBSpIYDEGk9KxsGh3mySTRgMtXL583qmBpzeQ+jqCMRgBqB98u3z++J1sKlXHWfM9dyhSevkMwSbhoR8XIq/U0tCNyokEi/ueaBMCvbcTHhO7FcwzY92WK4Yt0aGROY5qX2UKSeOvuP4D6TPqKF1onrSzH9bx9XUf2lEdWT/ia1NEKjunUqu1xOB/StKDHMoX4/OKyIzuS0q/T1zOATthvasJFoPrAjkohTyaDUz2LN5JoH839hViyEG82yB+MjcFV5MU3N1l1QL3cVUCh93xSaua1N85qivl+siMkPGbO5xR/En4iEY6K2XPASUEMaieWVNTRCtJ4S8H+9 - ssh.dev.azure.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Hr1oTWqNqOlzGJOfGJ4NakVyIzf1rXYd4d7wo6jBlkLvCA4odBlL0mDUyZ0/QUfTTqeu+tm22gOsv+VrVTMk6vwRU75gY/y9ut5Mb3bR5BV58dKXyq9A9UeB5Cakehn5Zgm6x1mKoVyf+FFn26iYqXJRgzIZZcZ5V6hrE0Qg39kZm4az48o0AUbf6Sp4SLdvnuMa2sVNwHBboS7EJkm57XQPVU3/QpyNLHbWDdzwtrlS+ez30S3AdYhLKEOxAG8weOnyrtLJAUen9mTkol8oII1edf7mWWbWVf0nBmly21+nZcmCTISQBtdcyPaEno7fFQMDD26/s0lfKob4Kw8H - vs-ssh.visualstudio.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Hr1oTWqNqOlzGJOfGJ4NakVyIzf1rXYd4d7wo6jBlkLvCA4odBlL0mDUyZ0/QUfTTqeu+tm22gOsv+VrVTMk6vwRU75gY/y9ut5Mb3bR5BV58dKXyq9A9UeB5Cakehn5Zgm6x1mKoVyf+FFn26iYqXJRgzIZZcZ5V6hrE0Qg39kZm4az48o0AUbf6Sp4SLdvnuMa2sVNwHBboS7EJkm57XQPVU3/QpyNLHbWDdzwtrlS+ez30S3AdYhLKEOxAG8weOnyrtLJAUen9mTkol8oII1edf7mWWbWVf0nBmly21+nZcmCTISQBtdcyPaEno7fFQMDD26/s0lfKob4Kw8H -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/name: argocd-ssh-known-hosts-cm - app.kubernetes.io/part-of: argocd - name: argocd-ssh-known-hosts-cm ---- -apiVersion: v1 -data: null -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/name: argocd-tls-certs-cm - app.kubernetes.io/part-of: argocd - name: argocd-tls-certs-cm ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - app.kubernetes.io/name: argocd-secret - app.kubernetes.io/part-of: argocd - name: argocd-secret -type: Opaque ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: dex-server - app.kubernetes.io/name: argocd-dex-server - app.kubernetes.io/part-of: argocd - name: argocd-dex-server -spec: - ports: - - name: http - port: 5556 - protocol: TCP - targetPort: 5556 - - name: grpc - port: 5557 - protocol: TCP - targetPort: 5557 - - name: metrics - port: 5558 - protocol: TCP - targetPort: 5558 - selector: - app.kubernetes.io/name: argocd-dex-server ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: metrics - app.kubernetes.io/name: argocd-metrics - app.kubernetes.io/part-of: argocd - name: argocd-metrics -spec: - ports: - - name: metrics - port: 8082 - protocol: TCP - targetPort: 8082 - selector: - app.kubernetes.io/name: argocd-application-controller ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: redis - app.kubernetes.io/name: argocd-redis - app.kubernetes.io/part-of: argocd - name: argocd-redis -spec: - ports: - - name: tcp-redis - port: 6379 - targetPort: 6379 - selector: - app.kubernetes.io/name: argocd-redis ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: repo-server - app.kubernetes.io/name: argocd-repo-server - app.kubernetes.io/part-of: argocd - name: argocd-repo-server -spec: - ports: - - name: server - port: 8081 - protocol: TCP - targetPort: 8081 - - name: metrics - port: 8084 - protocol: TCP - targetPort: 8084 - selector: - app.kubernetes.io/name: argocd-repo-server ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: server - app.kubernetes.io/name: argocd-server - app.kubernetes.io/part-of: argocd - name: argocd-server -spec: - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 8080 - - name: https - port: 443 - protocol: TCP - targetPort: 8080 - selector: - app.kubernetes.io/name: argocd-server ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: server - app.kubernetes.io/name: argocd-server-metrics - app.kubernetes.io/part-of: argocd - name: argocd-server-metrics -spec: - ports: - - name: metrics - port: 8083 - protocol: TCP - targetPort: 8083 - selector: - app.kubernetes.io/name: argocd-server ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/component: dex-server - app.kubernetes.io/name: argocd-dex-server - app.kubernetes.io/part-of: argocd - name: argocd-dex-server -spec: - selector: - matchLabels: - app.kubernetes.io/name: argocd-dex-server - template: - metadata: - labels: - app.kubernetes.io/name: argocd-dex-server - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchLabels: - app.kubernetes.io/part-of: argocd - topologyKey: kubernetes.io/hostname - weight: 5 - containers: - - command: - - /shared/argocd-dex - - rundex - image: ghcr.io/dexidp/dex:v2.27.0 - imagePullPolicy: Always - name: dex - ports: - - containerPort: 5556 - - containerPort: 5557 - - containerPort: 5558 - volumeMounts: - - mountPath: /shared - name: static-files - initContainers: - - command: - - cp - - -n - - /usr/local/bin/argocd - - /shared/argocd-dex - image: quay.io/argoproj/argocd:v2.0.5 - imagePullPolicy: Always - name: copyutil - volumeMounts: - - mountPath: /shared - name: static-files - serviceAccountName: argocd-dex-server - volumes: - - emptyDir: {} - name: static-files - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/component: repo-server - app.kubernetes.io/name: argocd-repo-server - app.kubernetes.io/part-of: argocd - name: argocd-repo-server -spec: - selector: - matchLabels: - app.kubernetes.io/name: argocd-repo-server - template: - metadata: - labels: - app.kubernetes.io/name: argocd-repo-server - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchLabels: - app.kubernetes.io/name: argocd-repo-server - topologyKey: kubernetes.io/hostname - weight: 100 - - podAffinityTerm: - labelSelector: - matchLabels: - app.kubernetes.io/part-of: argocd - topologyKey: kubernetes.io/hostname - weight: 5 - automountServiceAccountToken: false - containers: - - command: - - uid_entrypoint.sh - - argocd-repo-server - - --redis - - argocd-redis:6379 - image: quay.io/argoproj/argocd:v2.0.5 - imagePullPolicy: Always - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthz?full=true - port: 8084 - initialDelaySeconds: 30 - periodSeconds: 5 - name: argocd-repo-server - ports: - - containerPort: 8081 - - containerPort: 8084 - readinessProbe: - httpGet: - path: /healthz - port: 8084 - initialDelaySeconds: 5 - periodSeconds: 10 - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - all - volumeMounts: - - mountPath: /app/config/ssh - name: ssh-known-hosts - - mountPath: /app/config/tls - name: tls-certs - - mountPath: /app/config/gpg/source - name: gpg-keys - - mountPath: /app/config/gpg/keys - name: gpg-keyring - - mountPath: /app/config/reposerver/tls - name: argocd-repo-server-tls - volumes: - - configMap: - name: argocd-ssh-known-hosts-cm - name: ssh-known-hosts - - configMap: - name: argocd-tls-certs-cm - name: tls-certs - - configMap: - name: argocd-gpg-keys-cm - name: gpg-keys - - emptyDir: {} - name: gpg-keyring - - name: argocd-repo-server-tls - secret: - items: - - key: tls.crt - path: tls.crt - - key: tls.key - path: tls.key - - key: ca.crt - path: ca.crt - optional: true - secretName: argocd-repo-server-tls ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/component: server - app.kubernetes.io/name: argocd-server - app.kubernetes.io/part-of: argocd - name: argocd-server -spec: - selector: - matchLabels: - app.kubernetes.io/name: argocd-server - template: - metadata: - labels: - app.kubernetes.io/name: argocd-server - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchLabels: - app.kubernetes.io/name: argocd-server - topologyKey: kubernetes.io/hostname - weight: 100 - - podAffinityTerm: - labelSelector: - matchLabels: - app.kubernetes.io/part-of: argocd - topologyKey: kubernetes.io/hostname - weight: 5 - containers: - - command: - - argocd-server - - --staticassets - - /shared/app - image: quay.io/argoproj/argocd:v2.0.5 - imagePullPolicy: Always - livenessProbe: - httpGet: - path: /healthz?full=true - port: 8080 - initialDelaySeconds: 3 - periodSeconds: 30 - name: argocd-server - ports: - - containerPort: 8080 - - containerPort: 8083 - readinessProbe: - httpGet: - path: /healthz - port: 8080 - initialDelaySeconds: 3 - periodSeconds: 30 - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - all - volumeMounts: - - mountPath: /app/config/ssh - name: ssh-known-hosts - - mountPath: /app/config/tls - name: tls-certs - - mountPath: /app/config/server/tls - name: argocd-repo-server-tls - serviceAccountName: argocd-server - volumes: - - emptyDir: {} - name: static-files - - configMap: - name: argocd-ssh-known-hosts-cm - name: ssh-known-hosts - - configMap: - name: argocd-tls-certs-cm - name: tls-certs - - name: argocd-repo-server-tls - secret: - items: - - key: tls.crt - path: tls.crt - - key: tls.key - path: tls.key - - key: ca.crt - path: ca.crt - optional: true - secretName: argocd-repo-server-tls ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - labels: - app.kubernetes.io/component: application-controller - app.kubernetes.io/name: argocd-application-controller - app.kubernetes.io/part-of: argocd - name: argocd-application-controller -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: argocd-application-controller - serviceName: argocd-application-controller - template: - metadata: - labels: - app.kubernetes.io/name: argocd-application-controller - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchLabels: - app.kubernetes.io/name: argocd-application-controller - topologyKey: kubernetes.io/hostname - weight: 100 - - podAffinityTerm: - labelSelector: - matchLabels: - app.kubernetes.io/part-of: argocd - topologyKey: kubernetes.io/hostname - weight: 5 - containers: - - command: - - argocd-application-controller - - --status-processors - - "20" - - --operation-processors - - "10" - image: quay.io/argoproj/argocd:v2.0.5 - imagePullPolicy: Always - livenessProbe: - httpGet: - path: /healthz - port: 8082 - initialDelaySeconds: 5 - periodSeconds: 10 - name: argocd-application-controller - ports: - - containerPort: 8082 - readinessProbe: - httpGet: - path: /healthz - port: 8082 - initialDelaySeconds: 5 - periodSeconds: 10 - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - all - volumeMounts: - - mountPath: /app/config/controller/tls - name: argocd-repo-server-tls - serviceAccountName: argocd-application-controller - volumes: - - name: argocd-repo-server-tls - secret: - items: - - key: tls.crt - path: tls.crt - - key: tls.key - path: tls.key - - key: ca.crt - path: ca.crt - optional: true - secretName: argocd-repo-server-tls ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: argocd-application-controller-network-policy -spec: - ingress: - - from: - - namespaceSelector: {} - ports: - - port: 8082 - podSelector: - matchLabels: - app.kubernetes.io/name: argocd-application-controller - policyTypes: - - Ingress ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: argocd-dex-server-network-policy -spec: - ingress: - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: argocd-server - ports: - - port: 5556 - protocol: TCP - - port: 5557 - protocol: TCP - - from: - - namespaceSelector: {} - ports: - - port: 5558 - protocol: TCP - podSelector: - matchLabels: - app.kubernetes.io/name: argocd-dex-server - policyTypes: - - Ingress ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: argocd-redis-network-policy -spec: - ingress: - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: argocd-server - - podSelector: - matchLabels: - app.kubernetes.io/name: argocd-repo-server - - podSelector: - matchLabels: - app.kubernetes.io/name: argocd-application-controller - ports: - - port: 6379 - protocol: TCP - podSelector: - matchLabels: - app.kubernetes.io/name: argocd-redis - policyTypes: - - Ingress ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: argocd-repo-server-network-policy -spec: - ingress: - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: argocd-server - - podSelector: - matchLabels: - app.kubernetes.io/name: argocd-application-controller - - podSelector: - matchLabels: - app.kubernetes.io/name: argocd-notifications-controller - ports: - - port: 8081 - protocol: TCP - - from: - - namespaceSelector: {} - ports: - - port: 8084 - podSelector: - matchLabels: - app.kubernetes.io/name: argocd-repo-server - policyTypes: - - Ingress ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: argocd-server-network-policy -spec: - ingress: - - {} - podSelector: - matchLabels: - app.kubernetes.io/name: argocd-server - policyTypes: - - Ingress diff --git a/kubernetes/argocd/redis-persistent.yaml b/kubernetes/argocd/redis-persistent.yaml deleted file mode 100644 index 90644ca..0000000 --- a/kubernetes/argocd/redis-persistent.yaml +++ /dev/null @@ -1,52 +0,0 @@ ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - labels: - app.kubernetes.io/component: redis - app.kubernetes.io/name: argocd-redis - app.kubernetes.io/part-of: argocd - name: argocd-redis -spec: - selector: - matchLabels: - app.kubernetes.io/name: argocd-redis - serviceName: argocd-redis - template: - metadata: - labels: - app.kubernetes.io/name: argocd-redis - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: redis - resources: - requests: - memory: "100Mi" - cpu: "100m" # equivalent to 0.1 of a CPU core - args: - - --save - - "60 1000" - - --appendonly - - "yes" - image: redis:6.2.4-alpine - imagePullPolicy: Always - ports: - - containerPort: 6379 - volumeMounts: - - name: redis-data - mountPath: /data - securityContext: - fsGroup: 1000 - runAsGroup: 1000 - runAsNonRoot: true - runAsUser: 1000 - serviceAccountName: argocd-redis - volumeClaimTemplates: - - metadata: - name: redis-data - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 10Gi diff --git a/kubernetes/eksctl/aws-alb-ingress-controller/README.txt b/kubernetes/eksctl/aws-alb-ingress-controller/README.txt deleted file mode 100644 index caabad5..0000000 --- a/kubernetes/eksctl/aws-alb-ingress-controller/README.txt +++ /dev/null @@ -1,4 +0,0 @@ - -# Recreating the ALB ingress controller -1. Follow these instructions: https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html -2. Create the external DNS controller from this folder diff --git a/kubernetes/eksctl/external-dns/external-dns.yaml b/kubernetes/eksctl/external-dns/external-dns.yaml deleted file mode 100644 index b84b621..0000000 --- a/kubernetes/eksctl/external-dns/external-dns.yaml +++ /dev/null @@ -1,72 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: external-dns - namespace: kube-system - annotations: - eks.amazonaws.com/role-arn: arn:aws:iam::633607774026:role/eksctl-operationcode-backend-addon-iamservic-Role1-Z635VDQWDH8 ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: external-dns - namespace: kube-system -rules: -- apiGroups: [""] - resources: ["services"] - verbs: ["get","watch","list"] -- apiGroups: [""] - resources: ["pods"] - verbs: ["get","watch","list"] -- apiGroups: ["extensions"] - resources: ["ingresses"] - verbs: ["get","watch","list"] -- apiGroups: [""] - resources: ["nodes"] - verbs: ["list"] ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: external-dns-viewer -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: external-dns -subjects: -- kind: ServiceAccount - name: external-dns - namespace: kube-system ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: external-dns - namespace: kube-system -spec: - selector: - matchLabels: - app: external-dns - strategy: - type: Recreate - template: - metadata: - labels: - app: external-dns - spec: - serviceAccountName: external-dns - containers: - - name: external-dns - image: us.gcr.io/k8s-artifacts-prod/external-dns/external-dns:v0.6.0 - args: - - --source=service - - --source=ingress - - --domain-filter=k8s.operationcode.org - - --provider=aws - - --policy=upsert-only # would prevent ExternalDNS from deleting any records, omit to enable full synchronization - - --aws-zone-type=public # only look at public hosted zones (valid values are public, private or no value for both) - - --registry=txt - - --txt-owner-id=operationcode-backend - # - --log-level=debug - securityContext: - fsGroup: 65534 # For ExternalDNS to be able to read Kubernetes and AWS token files diff --git a/kubernetes/eksctl/max-pods-calculator.sh b/kubernetes/eksctl/max-pods-calculator.sh deleted file mode 100755 index 0b9f8e6..0000000 --- a/kubernetes/eksctl/max-pods-calculator.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash - -set -o pipefail -set -o nounset -set -o errexit - -err_report() { - echo "Exited with error on line $1" -} -trap 'err_report $LINENO' ERR - -function print_help { - echo "usage: $0 [options]" - echo "Calculates maxPods value to be used when starting up the kubelet." - echo "-h,--help print this help." - echo "--instance-type Specify the instance type to calculate max pods value." - echo "--instance-type-from-imds Use this flag if the instance type should be fetched from IMDS." - echo "--cni-version Specify the version of the CNI (example - 1.7.5)." - echo "--cni-custom-networking-enabled Use this flag to indicate if CNI custom networking mode has been enabled." - echo "--cni-prefix-delegation-enabled Use this flag to indicate if CNI prefix delegation has been enabled." - echo "--cni-max-eni specify how many ENIs should be used for prefix delegation. Defaults to using all ENIs per instance." -} - -POSITIONAL=() - -while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -h|--help) - print_help - exit 1 - ;; - --instance-type) - INSTANCE_TYPE=$2 - shift - shift - ;; - --instance-type-from-imds) - INSTANCE_TYPE_FROM_IMDS=true - shift - ;; - --cni-version) - CNI_VERSION=$2 - shift - shift - ;; - --cni-custom-networking-enabled) - CNI_CUSTOM_NETWORKING_ENABLED=true - shift - ;; - --cni-prefix-delegation-enabled) - CNI_PREFIX_DELEGATION_ENABLED=true - shift - ;; - --cni-max-eni) - CNI_MAX_ENI=$2 - shift - shift - ;; - *) # unknown option - POSITIONAL+=("$1") # save it in an array for later - shift # past argument - ;; - esac -done - -CNI_VERSION="${CNI_VERSION:-}" -CNI_CUSTOM_NETWORKING_ENABLED="${CNI_CUSTOM_NETWORKING_ENABLED:-false}" -CNI_PREFIX_DELEGATION_ENABLED="${CNI_PREFIX_DELEGATION_ENABLED:-false}" -CNI_MAX_ENI="${CNI_MAX_ENI:-}" -INSTANCE_TYPE="${INSTANCE_TYPE:-}" -INSTANCE_TYPE_FROM_IMDS="${INSTANCE_TYPE_FROM_IMDS:-false}" - -PREFIX_DELEGATION_SUPPORTED=false -IPS_PER_PREFIX=16 - -if [ "$INSTANCE_TYPE_FROM_IMDS" = true ]; then - TOKEN=$(curl -m 10 -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 600" -s "http://169.254.169.254/latest/api/token") - export AWS_DEFAULT_REGION=$(curl -s --retry 5 -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/dynamic/instance-identity/document | jq .region -r) - INSTANCE_TYPE=$(curl -m 10 -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-type) -elif [ -z "$INSTANCE_TYPE" ]; - # There's no reasonable default for an instanceType so force one to be provided to the script. - then echo "You must specify an instance type to calculate max pods value." - exit 1 -fi - -if [ -z "$CNI_VERSION" ]; - then echo "You must specify a CNI Version to use. Example - 1.7.5" - exit 1 -fi - -calculate_max_ip_addresses_prefix_delegation() { - enis=$1 - instance_max_eni_ips=$2 - echo $(($enis * (($instance_max_eni_ips - 1) * $IPS_PER_PREFIX ) + 2)) -} - -calculate_max_ip_addresses_secondary_ips() { - enis=$1 - instance_max_eni_ips=$2 - echo $(($enis * ($instance_max_eni_ips - 1) + 2)) -} - -min_number() { - printf "%s\n" "$@" | sort -g | head -n1 -} - - -VERSION_SPLIT=(${CNI_VERSION//./ }) -CNI_MAJOR_VERSION="${VERSION_SPLIT[0]}" -CNI_MINOR_VERSION="${VERSION_SPLIT[1]}" -if [[ "$CNI_MAJOR_VERSION" -gt 1 ]] || ([[ "$CNI_MAJOR_VERSION" = 1 ]] && [[ "$CNI_MINOR_VERSION" -gt 8 ]]); then - PREFIX_DELEGATION_SUPPORTED=true -fi - -DESCRIBE_INSTANCES_RESULT=$(aws ec2 describe-instance-types --instance-type $INSTANCE_TYPE --query 'InstanceTypes[0].{Hypervisor: Hypervisor, EniCount: NetworkInfo.MaximumNetworkInterfaces, PodsPerEniCount: NetworkInfo.Ipv4AddressesPerInterface, CpuCount: VCpuInfo.DefaultVCpus'}) - -HYPERVISOR_TYPE=$(echo $DESCRIBE_INSTANCES_RESULT | jq -r '.Hypervisor' ) -IS_NITRO=false -if [[ "$HYPERVISOR_TYPE" == "nitro" ]]; then - IS_NITRO=true -fi -INSTANCE_MAX_ENIS=$(echo $DESCRIBE_INSTANCES_RESULT | jq -r '.EniCount' ) -INSTANCE_MAX_ENIS_IPS=$(echo $DESCRIBE_INSTANCES_RESULT | jq -r '.PodsPerEniCount' ) - -if [ -z "$CNI_MAX_ENI" ] ; then - enis_for_pods=$INSTANCE_MAX_ENIS -else - enis_for_pods="$(min_number $CNI_MAX_ENI $INSTANCE_MAX_ENIS)" -fi - -if [ "$CNI_CUSTOM_NETWORKING_ENABLED" = true ] ; then - enis_for_pods=$((enis_for_pods-1)) -fi - - -if [ "$IS_NITRO" = true ] && [ "$CNI_PREFIX_DELEGATION_ENABLED" = true ] && [ "$PREFIX_DELEGATION_SUPPORTED" = true ]; then - max_pods=$(calculate_max_ip_addresses_prefix_delegation $enis_for_pods $INSTANCE_MAX_ENIS_IPS) -else - max_pods=$(calculate_max_ip_addresses_secondary_ips $enis_for_pods $INSTANCE_MAX_ENIS_IPS) -fi - -# Limit the total number of pods that can be launched on any instance type based on the vCPUs on that instance type. -MAX_POD_CEILING_FOR_LOW_CPU=110 -MAX_POD_CEILING_FOR_HIGH_CPU=250 -CPU_COUNT=$(echo $DESCRIBE_INSTANCES_RESULT | jq -r '.CpuCount' ) -if [ "$CPU_COUNT" -gt 30 ] ; then - echo $(min_number $MAX_POD_CEILING_FOR_HIGH_CPU $max_pods) -else - echo $(min_number $MAX_POD_CEILING_FOR_LOW_CPU $max_pods) -fi diff --git a/kubernetes/eksctl/operationcode-backend.yaml b/kubernetes/eksctl/operationcode-backend.yaml deleted file mode 100644 index 8ce786b..0000000 --- a/kubernetes/eksctl/operationcode-backend.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -apiVersion: eksctl.io/v1alpha5 -kind: ClusterConfig - -metadata: - name: operationcode-backend - region: us-east-2 - -managedNodeGroups: - - name: eks-infra-spot-v2 - instanceTypes: - - t3.small - spot: true - minSize: 3 - desiredCapacity: 3 - maxSize: 5 - volumeSize: 20 - volumeType: gp3 - # For this to be valid, run: - # kubectl set env daemonset aws-node -n kube-system ENABLE_PREFIX_DELEGATION=true - # kubectl set env daemonset aws-node -n kube-system WARM_PREFIX_TARGET=1 - maxPodsPerNode: 30 - ssh: - allow: true - publicKeyName: oc-ops - labels: - nodegroup-type: infra - tags: - Name: eks-infra-spot-v2 - iam: - withAddonPolicies: - imageBuilder: true - autoScaler: true - externalDNS: true - certManager: true - appMesh: true - ebs: true - fsx: true - efs: true - albIngress: true - xRay: true - cloudWatch: true diff --git a/kubernetes/eksctl/vertical-pod-autoscaler/README.md b/kubernetes/eksctl/vertical-pod-autoscaler/README.md deleted file mode 100644 index 37fec87..0000000 --- a/kubernetes/eksctl/vertical-pod-autoscaler/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Setup - -to recreate, follow these instructions: https://docs.aws.amazon.com/eks/latest/userguide/vertical-pod-autoscaler.html diff --git a/kubernetes/operationcode-namespace.yml b/kubernetes/operationcode-namespace.yml deleted file mode 100644 index 9e9cb33..0000000 --- a/kubernetes/operationcode-namespace.yml +++ /dev/null @@ -1,13 +0,0 @@ -kind: Namespace -apiVersion: v1 -metadata: - name: operationcode - labels: - name: operationcode ---- -kind: Namespace -apiVersion: v1 -metadata: - name: operationcode-staging - labels: - name: operationcode-staging diff --git a/kubernetes/operationcode_python_backend/base/deployment.yaml b/kubernetes/operationcode_python_backend/base/deployment.yaml deleted file mode 100644 index 4560894..0000000 --- a/kubernetes/operationcode_python_backend/base/deployment.yaml +++ /dev/null @@ -1,158 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: back-end -spec: - replicas: 1 - revisionHistoryLimit: 1 - template: - spec: - containers: - - name: app - image: operationcode/back-end:latest - imagePullPolicy: Always - ports: - - containerPort: 8000 - resources: - requests: - memory: 200Mi - cpu: 100m - readinessProbe: - httpGet: - path: /healthz - port: 8000 - initialDelaySeconds: 5 - periodSeconds: 10 - livenessProbe: - httpGet: - path: /healthz - port: 8000 - initialDelaySeconds: 15 - periodSeconds: 20 - env: - - name: DB_HOST - value: # Requires overlay - - name: ENVIRONMENT - value: # Requires overlay - - name: RELEASE - value: # Requires overlay - - name: SITE_ID - value: "3" - - name: DJANGO_ENV - value: production - - name: DB_ENGINE - value: django.db.backends.postgresql - - name: DB_NAME - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: db_name - - name: DB_USER - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: db_user - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: db_password - - name: DB_PORT - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: db_port - - name: AWS_STORAGE_BUCKET_NAME - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: aws_storage_bucket_name - - name: BUCKET_REGION_NAME - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: bucket_region_name - - name: SECRET_KEY - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: secret_key - - name: PYBOT_AUTH_TOKEN - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: pybot_auth_token - - name: PYBOT_URL - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: pybot_url - - name: MAILCHIMP_API_KEY - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: mailchimp_api_key - - name: MAILCHIMP_LIST_ID - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: mailchimp_list_id - - name: MANDRILL_API_KEY - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: mandrill_api_key - - name: SENTRY_DSN - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: sentry_dsn - - name: GOOGLE_OAUTH_CLIENT_ID - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: google_oauth_client_id - - name: GOOGLE_OAUTH_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: google_oauth_client_secret - - name: EXTRA_HOSTS - value: api.staging.operationcode.org - - name: RECAPTCHA_PUBLIC_KEY - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: recaptcha_public_key - - name: RECAPTCHA_PRIVATE_KEY - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: recaptcha_private_key - - name: GITHUB_JWT - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: github_jwt - - name: HONEYCOMB_WRITEKEY - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: honeycomb_writekey - - name: HONEYCOMB_DATASET - value: production-traces - - name: JWT_SECRET_KEY - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: jwt_secret_key - - name: JWT_PUBLIC_KEY - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: jwt_public_key - volumes: - - name: python-backend-secrets - secret: - secretName: python-backend-secrets diff --git a/kubernetes/operationcode_python_backend/base/kustomization.yaml b/kubernetes/operationcode_python_backend/base/kustomization.yaml deleted file mode 100644 index 1c90fd5..0000000 --- a/kubernetes/operationcode_python_backend/base/kustomization.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -commonLabels: - app: back-end - -resources: -- deployment.yaml -- service.yaml diff --git a/kubernetes/operationcode_python_backend/base/service.yaml b/kubernetes/operationcode_python_backend/base/service.yaml deleted file mode 100644 index 15760b8..0000000 --- a/kubernetes/operationcode_python_backend/base/service.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: back-end-service -spec: - selector: - app: back-end - ports: - - protocol: TCP - name: http - port: 80 - targetPort: 8000 - type: NodePort diff --git a/kubernetes/operationcode_python_backend/overlays/prod/deployment.yaml b/kubernetes/operationcode_python_backend/overlays/prod/deployment.yaml deleted file mode 100644 index b5fe4e6..0000000 --- a/kubernetes/operationcode_python_backend/overlays/prod/deployment.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: back-end -spec: - replicas: 2 - template: - spec: - containers: - - name: app - image: operationcode/back-end:master - env: - - name: DB_HOST - value: python-prod.czwauqf3tjaz.us-east-2.rds.amazonaws.com - - name: ENVIRONMENT - value: aws_prod - - name: EXTRA_HOSTS - value: backend.k8s.operationcode.org - - name: RELEASE - value: 1.0.1 - - name: SITE_ID - value: "4" - - name: DJANGO_ENV - value: production - - name: HONEYCOMB_DATASET - value: production-traces - - name: AWS_S3_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: aws_s3_access_key_id - - name: AWS_S3_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: aws_s3_secret_access_key diff --git a/kubernetes/operationcode_python_backend/overlays/prod/ingress.yaml b/kubernetes/operationcode_python_backend/overlays/prod/ingress.yaml deleted file mode 100644 index 0c417dc..0000000 --- a/kubernetes/operationcode_python_backend/overlays/prod/ingress.yaml +++ /dev/null @@ -1,73 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: back-end - annotations: - kubernetes.io/ingress.class: alb - alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:633607774026:certificate/8de9fd02-191c-485f-b952-e5ba32e90acb - alb.ingress.kubernetes.io/healthcheck-path: /healthz - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS": 443}]' - alb.ingress.kubernetes.io/scheme: internet-facing - alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS-1-2-2017-01 - alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' - alb.ingress.kubernetes.io/actions.response-401: '{"Type":"fixed-response","FixedResponseConfig":{"ContentType":"text/plain","StatusCode":"401","MessageBody":"401 Not Authorized"}}' - alb.ingress.kubernetes.io/load-balancer-attributes: access_logs.s3.enabled=true,access_logs.s3.bucket=oc-alb-logs,access_logs.s3.prefix=oc-prod - alb.ingress.kubernetes.io/load-balancer-attributes: routing.http2.enabled=true - alb.ingress.kubernetes.io/load-balancer-attributes: idle_timeout.timeout_seconds=600 - labels: - app: back-end -spec: - rules: - # http redirect must come first - - http: - paths: - - path: /* - backend: - serviceName: ssl-redirect - servicePort: use-annotation - # back-end production - - host: backend.k8s.operationcode.org - http: - paths: - - path: /* - backend: - serviceName: back-end-service - servicePort: 80 - - host: api.operationcode.org - http: - paths: - - path: /* - backend: - serviceName: back-end-service - servicePort: 80 - # resources-api production - - host: resources.k8s.operationcode.org - http: - paths: - - path: /metrics - backend: - serviceName: response-401 - servicePort: use-annotation - - path: /metrics/* - backend: - serviceName: response-401 - servicePort: use-annotation - - path: /* - backend: - serviceName: resources-api-service - servicePort: 80 - - host: resources.operationcode.org - http: - paths: - - path: /metrics - backend: - serviceName: response-401 - servicePort: use-annotation - - path: /metrics/* - backend: - serviceName: response-401 - servicePort: use-annotation - - path: /* - backend: - serviceName: resources-api-service - servicePort: 80 diff --git a/kubernetes/operationcode_python_backend/overlays/prod/kustomization.yaml b/kubernetes/operationcode_python_backend/overlays/prod/kustomization.yaml deleted file mode 100644 index 1d8f44b..0000000 --- a/kubernetes/operationcode_python_backend/overlays/prod/kustomization.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namespace: operationcode - -bases: -- ../../base - -resources: -- ingress.yaml - -patchesStrategicMerge: - - deployment.yaml \ No newline at end of file diff --git a/kubernetes/operationcode_python_backend/overlays/staging/deployment.yaml b/kubernetes/operationcode_python_backend/overlays/staging/deployment.yaml deleted file mode 100644 index d8d2775..0000000 --- a/kubernetes/operationcode_python_backend/overlays/staging/deployment.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: back-end -spec: - template: - spec: - containers: - - name: app - image: operationcode/back-end:staging - env: - - name: DB_HOST - value: postgres.pgo.svc.cluster.local - - name: ENVIRONMENT - value: aws_staging - - name: EXTRA_HOSTS - value: backend-staging.k8s.operationcode.org - - name: RELEASE - value: 1.0.1 - - name: DJANGO_ENV - value: staging - - name: HONEYCOMB_DATASET - value: staging-traces diff --git a/kubernetes/operationcode_python_backend/overlays/staging/ingress.yaml b/kubernetes/operationcode_python_backend/overlays/staging/ingress.yaml deleted file mode 100644 index ca9e997..0000000 --- a/kubernetes/operationcode_python_backend/overlays/staging/ingress.yaml +++ /dev/null @@ -1,73 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: back-end - annotations: - kubernetes.io/ingress.class: alb - alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:633607774026:certificate/8de9fd02-191c-485f-b952-e5ba32e90acb - alb.ingress.kubernetes.io/healthcheck-path: /healthz - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS": 443}]' - alb.ingress.kubernetes.io/scheme: internet-facing - alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS-1-2-2017-01 - alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' - alb.ingress.kubernetes.io/actions.response-401: '{"Type":"fixed-response","FixedResponseConfig":{"ContentType":"text/plain","StatusCode":"401","MessageBody":"401 Not Authorized"}}' - alb.ingress.kubernetes.io/load-balancer-attributes: access_logs.s3.enabled=true,access_logs.s3.bucket=oc-alb-logs,access_logs.s3.prefix=oc-staging - alb.ingress.kubernetes.io/load-balancer-attributes: routing.http2.enabled=true - alb.ingress.kubernetes.io/load-balancer-attributes: idle_timeout.timeout_seconds=600 - labels: - app: back-end -spec: - rules: - # http redirect must come first - - http: - paths: - - path: /* - backend: - serviceName: ssl-redirect - servicePort: use-annotation - # back-end staging - - host: backend-staging.k8s.operationcode.org - http: - paths: - - path: /* - backend: - serviceName: back-end-service - servicePort: 80 - - host: api.staging.operationcode.org - http: - paths: - - path: /* - backend: - serviceName: back-end-service - servicePort: 80 - # resources-api staging - - host: resources-staging.k8s.operationcode.org - http: - paths: - - path: /metrics - backend: - serviceName: response-401 - servicePort: use-annotation - - path: /metrics/* - backend: - serviceName: response-401 - servicePort: use-annotation - - path: /* - backend: - serviceName: resources-api-service - servicePort: 80 - - host: resources.staging.operationcode.org - http: - paths: - - path: /metrics - backend: - serviceName: response-401 - servicePort: use-annotation - - path: /metrics/* - backend: - serviceName: response-401 - servicePort: use-annotation - - path: /* - backend: - serviceName: resources-api-service - servicePort: 80 diff --git a/kubernetes/operationcode_python_backend/overlays/staging/kustomization.yaml b/kubernetes/operationcode_python_backend/overlays/staging/kustomization.yaml deleted file mode 100644 index 962b1ef..0000000 --- a/kubernetes/operationcode_python_backend/overlays/staging/kustomization.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namespace: operationcode-staging - -bases: -- ../../base - -resources: -- ingress.yaml - -patchesStrategicMerge: - - deployment.yaml \ No newline at end of file diff --git a/kubernetes/resources_api/base/deployment.yaml b/kubernetes/resources_api/base/deployment.yaml deleted file mode 100644 index 97a8c13..0000000 --- a/kubernetes/resources_api/base/deployment.yaml +++ /dev/null @@ -1,65 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: resources-api -spec: - replicas: 1 - revisionHistoryLimit: 1 - template: - spec: - containers: - - name: app - image: operationcode/resources-api:latest - command: ["uwsgi"] - args: ["--ini", "app.ini"] - imagePullPolicy: Always - ports: - - containerPort: 5000 - resources: - requests: - memory: 200Mi - cpu: 100m - env: - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: resources-api-secrets - key: postgres_user - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: resources-api-secrets - key: postgres_password - - name: ALGOLIA_APP_ID - valueFrom: - secretKeyRef: - name: resources-api-secrets - key: algolia_app_id - - name: ALGOLIA_API_KEY - valueFrom: - secretKeyRef: - name: resources-api-secrets - key: algolia_api_key - - name: INDEX_NAME - value: resources_api - - name: POSTGRES_DB - value: resources_api - - name: POSTGRES_HOST - value: resources-postgres - - name: HONEYCOMB_WRITEKEY - valueFrom: - secretKeyRef: - name: python-backend-secrets - key: honeycomb_writekey - - name: HONEYCOMB_DATASET - value: production-traces - - name: JWT_PUBLIC_KEY - valueFrom: - secretKeyRef: - name: resources-api-secrets - key: jwt_public_key - volumes: - - name: resources-api-secrets - secret: - secretName: resources-api-secrets diff --git a/kubernetes/resources_api/base/kustomization.yaml b/kubernetes/resources_api/base/kustomization.yaml deleted file mode 100644 index 0f1a2ee..0000000 --- a/kubernetes/resources_api/base/kustomization.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -commonLabels: - app: resources-api - -resources: -- deployment.yaml -- service.yaml diff --git a/kubernetes/resources_api/base/service.yaml b/kubernetes/resources_api/base/service.yaml deleted file mode 100644 index a30671d..0000000 --- a/kubernetes/resources_api/base/service.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: resources-api-service -spec: - selector: - app: resources-api - ports: - - protocol: TCP - name: http - port: 80 - targetPort: 5000 - type: NodePort diff --git a/kubernetes/resources_api/overlays/prod/database-service.yaml b/kubernetes/resources_api/overlays/prod/database-service.yaml deleted file mode 100644 index 2f308b2..0000000 --- a/kubernetes/resources_api/overlays/prod/database-service.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: resources-postgres -spec: - type: ExternalName - externalName: python-prod.czwauqf3tjaz.us-east-2.rds.amazonaws.com diff --git a/kubernetes/resources_api/overlays/prod/deployment.yaml b/kubernetes/resources_api/overlays/prod/deployment.yaml deleted file mode 100644 index e42b89c..0000000 --- a/kubernetes/resources_api/overlays/prod/deployment.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: resources-api -spec: - replicas: 2 diff --git a/kubernetes/resources_api/overlays/prod/kustomization.yaml b/kubernetes/resources_api/overlays/prod/kustomization.yaml deleted file mode 100644 index 0345baa..0000000 --- a/kubernetes/resources_api/overlays/prod/kustomization.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namespace: operationcode - -bases: -- ../../base - -resources: -- database-service.yaml - -patchesStrategicMerge: - - deployment.yaml diff --git a/kubernetes/resources_api/overlays/staging/database-service.yaml b/kubernetes/resources_api/overlays/staging/database-service.yaml deleted file mode 100644 index 9f442bf..0000000 --- a/kubernetes/resources_api/overlays/staging/database-service.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: resources-postgres -spec: - type: ExternalName - externalName: staging-postgres.pgo.svc.cluster.local diff --git a/kubernetes/resources_api/overlays/staging/deployment.yaml b/kubernetes/resources_api/overlays/staging/deployment.yaml deleted file mode 100644 index 16d62ea..0000000 --- a/kubernetes/resources_api/overlays/staging/deployment.yaml +++ /dev/null @@ -1,15 +0,0 @@ - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: resources-api -spec: - template: - spec: - containers: - - name: app - env: - - name: FLASK_ENV - value: staging - - name: HONEYCOMB_DATASET - value: staging-traces diff --git a/kubernetes/resources_api/overlays/staging/kustomization.yaml b/kubernetes/resources_api/overlays/staging/kustomization.yaml deleted file mode 100644 index 0e6b046..0000000 --- a/kubernetes/resources_api/overlays/staging/kustomization.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namespace: operationcode-staging - -bases: -- ../../base - -resources: -- database-service.yaml - -patchesStrategicMerge: - - deployment.yaml diff --git a/IPv6_MIGRATION_NOTES.md b/plans/IPv6_MIGRATION_NOTES.md similarity index 100% rename from IPv6_MIGRATION_NOTES.md rename to plans/IPv6_MIGRATION_NOTES.md From 582140fe4bece5ca2ca08975d9922e2313b4fb09 Mon Sep 17 00:00:00 2001 From: Irving Popovetsky Date: Wed, 28 Jan 2026 17:11:49 -0800 Subject: [PATCH 2/4] SES Forwarding setup for @coders.operationcode.org email addresses --- .github/workflows/terraform.yml | 6 + .gitignore | 28 + README.md | 37 +- lambda/ses_email_forwarder/.gitignore | 17 + lambda/ses_email_forwarder/README.md | 73 ++ lambda/ses_email_forwarder/handler.py | 375 +++++++++ lambda/ses_email_forwarder/requirements.txt | 2 + lambda/ses_email_forwarder/tests/__init__.py | 1 + .../tests/fixtures/sample_ses_event.json | 74 ++ .../ses_email_forwarder/tests/test_handler.py | 300 +++++++ plans/ses-email-forwarding-guide.md | 795 ++++++++++++++++++ terraform/.terraform.lock.hcl | 20 + terraform/main.tf | 8 + terraform/route53.tf | 41 + terraform/ses_email_forwarding.tf | 32 + terraform/ses_email_forwarding/data.tf | 26 + terraform/ses_email_forwarding/main.tf | 242 ++++++ terraform/ses_email_forwarding/outputs.tf | 29 + terraform/ses_email_forwarding/variables.tf | 24 + terraform/ses_email_forwarding/versions.tf | 15 + 20 files changed, 2138 insertions(+), 7 deletions(-) create mode 100644 lambda/ses_email_forwarder/.gitignore create mode 100644 lambda/ses_email_forwarder/README.md create mode 100644 lambda/ses_email_forwarder/handler.py create mode 100644 lambda/ses_email_forwarder/requirements.txt create mode 100644 lambda/ses_email_forwarder/tests/__init__.py create mode 100644 lambda/ses_email_forwarder/tests/fixtures/sample_ses_event.json create mode 100644 lambda/ses_email_forwarder/tests/test_handler.py create mode 100644 plans/ses-email-forwarding-guide.md create mode 100644 terraform/route53.tf create mode 100644 terraform/ses_email_forwarding.tf create mode 100644 terraform/ses_email_forwarding/data.tf create mode 100644 terraform/ses_email_forwarding/main.tf create mode 100644 terraform/ses_email_forwarding/outputs.tf create mode 100644 terraform/ses_email_forwarding/variables.tf create mode 100644 terraform/ses_email_forwarding/versions.tf diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index 76719d8..c32724c 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -4,7 +4,13 @@ on: push: branches: - main + paths: + - 'terraform/**' + - 'lambda/**' pull_request: + paths: + - 'terraform/**' + - 'lambda/**' jobs: terraform: diff --git a/.gitignore b/.gitignore index 8b4298e..4862f36 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,30 @@ +# Terraform .terraform/ +*.tfstate +*.tfstate.* +*.tfvars + +# Python +*.pyc +*.pyo +*.pyd +__pycache__/ +*.py[cod] +*$py.class +.Python +.venv/ +venv/ +ENV/ +env/ +*.egg-info/ +.pytest_cache/ +.mypy_cache/ + +# Build artifacts +*.zip + +# Local environment +.envrc + +# OS .DS_Store diff --git a/README.md b/README.md index f75796a..6391186 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,34 @@ -# Operation Code Infra -Platform infrastructure for the [Operation Code site](https://operationcode.org/). +# Operation Code Infrastructure -## Setup +Terraform-managed AWS infrastructure for [Operation Code](https://operationcode.org/). -### Operation Code's ECS Cluster -Greetings! Much of Operation Code's web site runs in an AWS ECS cluster. These instructions will guide you through setting up access to our cluster so you can run rails console, tail logs, and more! +## Overview + +ECS cluster running containerized services on EC2 spot instances, fronted by an Application Load Balancer. + +### Active Services +- **Python Backend** (prod/staging) - `backend.operationcode.org`, `api.operationcode.org` +- **Pybot** (prod) - Slack integration bot at `pybot.operationcode.org` + +### Stack +- **Region:** us-east-2 +- **Compute:** ECS with Fargate + spot instances +- **Routing:** ALB with host-based routing +- **Logs:** CloudWatch (7-day retention) +- **State:** S3 backend + +## Structure +``` +terraform/ +├── ecs.tf # ECS cluster config +├── apps.tf # Service definitions +├── alb.tf # Load balancer +├── asg.tf # Auto-scaling groups +├── python_backend/ # Backend service module +└── pybot/ # Pybot service module +``` + +## License +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -## Licensing - [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) Operation Code Infra is under the [MIT License](/LICENSE). diff --git a/lambda/ses_email_forwarder/.gitignore b/lambda/ses_email_forwarder/.gitignore new file mode 100644 index 0000000..0e60372 --- /dev/null +++ b/lambda/ses_email_forwarder/.gitignore @@ -0,0 +1,17 @@ +# Python dependencies installed for Lambda packaging +python/ + +# Virtual environments +.venv/ +venv/ + +# Python cache +__pycache__/ +*.pyc + +# Pytest +.pytest_cache/ + +# IDE +.vscode/ +.idea/ diff --git a/lambda/ses_email_forwarder/README.md b/lambda/ses_email_forwarder/README.md new file mode 100644 index 0000000..c290238 --- /dev/null +++ b/lambda/ses_email_forwarder/README.md @@ -0,0 +1,73 @@ +# SES Email Forwarder Lambda Function + +This Lambda function forwards emails received by AWS SES to personal email addresses based on alias mappings stored in Airtable. + +## Overview + +When a donor with recurring donations receives a custom email alias (e.g., `john@coders.operationcode.org`), this Lambda function: +1. Receives the email via SES +2. Checks Airtable for the alias mapping +3. Validates the donor's status is "active" +4. Forwards the email to the donor's personal email address + +## Environment Variables + +- `EMAIL_BUCKET` - S3 bucket name where SES stores incoming emails +- `AIRTABLE_SECRET_NAME` - Name of the secret in AWS Secrets Manager containing Airtable credentials +- `FORWARD_FROM_EMAIL` - Email address to use as the "From" address (e.g., noreply@coders.operationcode.org) +- `AWS_SES_REGION` - AWS region for SES (us-east-1) +- `ENVIRONMENT` - Environment name for Sentry (prod/staging) + +## Secrets Manager Schema + +The secret referenced by `AIRTABLE_SECRET_NAME` must contain: +```json +{ + "airtable_api_key": "patXXXXXXXXXXXXXX", + "airtable_base_id": "appXXXXXXXXXXXXXX", + "airtable_table_name": "Email Aliases", + "sentry_dsn": "https://xxxxx@oxxxxx.ingest.sentry.io/xxxxx" +} +``` + +## Local Testing + +```bash +# Create virtual environment +python3 -m venv venv +source venv/bin/activate + +# Install dependencies +pip install -r requirements.txt +pip install pytest moto urllib3 + +# Run tests +pytest tests/ -v +``` + +## Architecture + +- **Region**: us-east-1 (required for SES email receiving) +- **Runtime**: Python 3.12 +- **Memory**: 256 MB +- **Timeout**: 30 seconds +- **Architecture**: ARM64 (Graviton) + +## Email Flow + +1. Email sent to `alias@coders.operationcode.org` +2. SES receives email and stores it in S3 +3. SES invokes Lambda function +4. Lambda: + - Retrieves email from S3 + - Queries Airtable for alias mapping + - Validates donor status is "active" + - Rewrites headers (From, Reply-To) + - Sends email via SES to personal email +5. Original sender receives replies via Reply-To header + +## Error Handling + +Errors are logged to: +- CloudWatch Logs: `/aws/lambda/ses-email-forwarder` +- Sentry: For alerting and monitoring diff --git a/lambda/ses_email_forwarder/handler.py b/lambda/ses_email_forwarder/handler.py new file mode 100644 index 0000000..8c117bc --- /dev/null +++ b/lambda/ses_email_forwarder/handler.py @@ -0,0 +1,375 @@ +import boto3 +import email +import os +import json +import urllib.request +import urllib.error +import urllib.parse +from email import policy +from email.parser import BytesParser +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.mime.base import MIMEBase +from email import encoders +import sentry_sdk +from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration + +# Cache for secrets and config +_secrets_cache = None +_config_cache = None + +# AWS clients (initialized lazily) +_s3_client = None +_ses_client = None +_secrets_client = None + + +def get_config(): + """Get configuration from environment variables with caching.""" + global _config_cache + if _config_cache is None: + _config_cache = { + 'email_bucket': os.environ.get('EMAIL_BUCKET', ''), + 'airtable_secret_name': os.environ.get('AIRTABLE_SECRET_NAME', ''), + 'forward_from_email': os.environ.get('FORWARD_FROM_EMAIL', ''), + 'aws_ses_region': os.environ.get('AWS_SES_REGION', 'us-east-1'), + 'environment': os.environ.get('ENVIRONMENT', 'production') + } + return _config_cache + + +def get_s3_client(): + """Get S3 client with lazy initialization.""" + global _s3_client + if _s3_client is None: + _s3_client = boto3.client('s3') + return _s3_client + + +def get_ses_client(): + """Get SES client with lazy initialization.""" + global _ses_client + if _ses_client is None: + config = get_config() + _ses_client = boto3.client('ses', region_name=config['aws_ses_region']) + return _ses_client + + +def get_secrets_client(): + """Get Secrets Manager client with lazy initialization.""" + global _secrets_client + if _secrets_client is None: + _secrets_client = boto3.client('secretsmanager', region_name='us-east-2') + return _secrets_client + + + + +def get_airtable_credentials(): + """ + Fetch Airtable credentials from Secrets Manager with caching. + + Returns: + dict: Contains airtable_api_key, airtable_base_id, airtable_table_name, sentry_dsn + """ + global _secrets_cache + if _secrets_cache is None: + config = get_config() + secret_name = config['airtable_secret_name'] + try: + secrets_client = get_secrets_client() + response = secrets_client.get_secret_value(SecretId=secret_name) + _secrets_cache = json.loads(response['SecretString']) + print(f"Successfully retrieved secrets from {secret_name}") + except Exception as e: + print(f"Error retrieving secrets from {secret_name}: {str(e)}") + raise + return _secrets_cache + + +# Initialize Sentry (deferred until first invocation to get DSN from Secrets Manager) +def init_sentry(): + """Initialize Sentry with DSN from Secrets Manager.""" + try: + config = get_config() + credentials = get_airtable_credentials() + sentry_dsn = credentials.get('sentry_dsn') + if sentry_dsn: + sentry_sdk.init( + dsn=sentry_dsn, + integrations=[AwsLambdaIntegration()], + traces_sample_rate=0.1, # 10% transaction sampling + environment=config['environment'] + ) + print("Sentry initialized successfully") + else: + print("Warning: No sentry_dsn found in secrets") + except Exception as e: + print(f"Warning: Failed to initialize Sentry: {str(e)}") + + +def lookup_alias_in_airtable(alias: str) -> dict | None: + """ + Query Airtable to find the mapping for a given alias. + Returns the record if found and active, None otherwise. + + Args: + alias: The email alias (local part before @) + + Returns: + dict or None: The Airtable record fields if found and active + """ + credentials = get_airtable_credentials() + airtable_api_key = credentials['airtable_api_key'] + airtable_base_id = credentials['airtable_base_id'] + airtable_table_name = credentials['airtable_table_name'] + + url = f"https://api.airtable.com/v0/{airtable_base_id}/{urllib.parse.quote(airtable_table_name)}" + + # Filter for exact alias match and active status + # Note: Airtable field names are case-sensitive + params = urllib.parse.urlencode({ + 'filterByFormula': f"AND({{Alias}} = '{alias}', {{Status}} = 'active')", + 'maxRecords': 1 + }) + + full_url = f"{url}?{params}" + + req = urllib.request.Request( + full_url, + headers={ + 'Authorization': f'Bearer {airtable_api_key}', + 'Content-Type': 'application/json' + } + ) + + try: + with urllib.request.urlopen(req) as response: + data = json.loads(response.read().decode()) + records = data.get('records', []) + if records: + print(f"Found active alias mapping for: {alias}") + return records[0]['fields'] + print(f"No active alias mapping found for: {alias}") + return None + except urllib.error.HTTPError as e: + error_body = e.read().decode() + print(f"Airtable API error: {e.code} - {error_body}") + sentry_sdk.capture_exception(e) + return None + except Exception as e: + print(f"Error querying Airtable: {str(e)}") + sentry_sdk.capture_exception(e) + return None + + +def get_email_from_s3(message_id: str) -> bytes: + """ + Retrieve the raw email from S3. + + Args: + message_id: The SES message ID (used as S3 key) + + Returns: + bytes: The raw email content + """ + try: + config = get_config() + s3_client = get_s3_client() + response = s3_client.get_object( + Bucket=config['email_bucket'], + Key=message_id + ) + return response['Body'].read() + except Exception as e: + print(f"Error retrieving email from S3: {str(e)}") + sentry_sdk.capture_exception(e) + raise + + +def forward_email(raw_email: bytes, forward_to: str, original_recipient: str) -> dict: + """ + Parse the original email and forward it to the destination address. + Rewrites headers to comply with SES requirements while preserving + the original sender information. + + Args: + raw_email: The raw email bytes from S3 + forward_to: The destination email address + original_recipient: The original recipient address (alias@coders.operationcode.org) + + Returns: + dict: SES send_raw_email response + """ + config = get_config() + + # Parse the original email + original_msg = BytesParser(policy=policy.default).parsebytes(raw_email) + + # Extract original headers + original_from = original_msg['From'] + original_subject = original_msg['Subject'] or '(no subject)' + original_to = original_msg['To'] + original_date = original_msg['Date'] + original_message_id = original_msg['Message-ID'] + + # Create new message + new_msg = MIMEMultipart('mixed') + + # Set headers for forwarded message + # SES requires From to be a verified identity + new_msg['From'] = config['forward_from_email'] + new_msg['To'] = forward_to + new_msg['Subject'] = original_subject + new_msg['Reply-To'] = original_from # Replies go to original sender + + # Add custom headers to preserve original info + new_msg['X-Original-From'] = original_from + new_msg['X-Original-To'] = original_recipient + new_msg['X-Forwarded-For'] = original_recipient + + # Handle multipart messages (with attachments) vs simple messages + if original_msg.is_multipart(): + # Copy all parts from original message + for part in original_msg.walk(): + content_type = part.get_content_type() + content_disposition = str(part.get('Content-Disposition', '')) + + if content_type == 'multipart/mixed' or content_type == 'multipart/alternative': + continue + + if 'attachment' in content_disposition: + # Handle attachments + new_part = MIMEBase(*content_type.split('/')) + new_part.set_payload(part.get_payload(decode=True)) + encoders.encode_base64(new_part) + new_part.add_header( + 'Content-Disposition', + 'attachment', + filename=part.get_filename() or 'attachment' + ) + new_msg.attach(new_part) + else: + # Handle body parts + payload = part.get_payload(decode=True) + if payload: + if content_type == 'text/plain': + new_msg.attach(MIMEText(payload.decode('utf-8', errors='replace'), 'plain')) + elif content_type == 'text/html': + new_msg.attach(MIMEText(payload.decode('utf-8', errors='replace'), 'html')) + else: + # Simple message without attachments + payload = original_msg.get_payload(decode=True) + if payload: + content_type = original_msg.get_content_type() + if content_type == 'text/html': + new_msg.attach(MIMEText(payload.decode('utf-8', errors='replace'), 'html')) + else: + new_msg.attach(MIMEText(payload.decode('utf-8', errors='replace'), 'plain')) + + # Send via SES + try: + ses_client = get_ses_client() + response = ses_client.send_raw_email( + Source=config['forward_from_email'], + Destinations=[forward_to], + RawMessage={'Data': new_msg.as_bytes()} + ) + return response + except Exception as e: + print(f"Error sending email via SES: {str(e)}") + sentry_sdk.capture_exception(e) + raise + + +def lambda_handler(event, context): + """ + Lambda handler for SES incoming email events. + + Event structure: + { + "Records": [{ + "eventSource": "aws:ses", + "eventVersion": "1.0", + "ses": { + "mail": { + "messageId": "...", + "source": "sender@example.com", + "destination": ["recipient@coders.operationcode.org"] + }, + "receipt": { + "recipients": ["recipient@coders.operationcode.org"], + ... + } + } + }] + } + + Args: + event: SES event containing email details + context: Lambda context object + + Returns: + dict: Response with statusCode and body + """ + # Initialize Sentry on first invocation + if _secrets_cache is None: + init_sentry() + + print(f"Received event: {json.dumps(event)}") + + for record in event.get('Records', []): + ses_data = record.get('ses', {}) + mail_data = ses_data.get('mail', {}) + + message_id = mail_data.get('messageId') + recipients = mail_data.get('destination', []) + source = mail_data.get('source', 'unknown') + + print(f"Processing message {message_id} from {source} to {recipients}") + + for recipient in recipients: + # Extract alias from recipient address + # e.g., "john482@coders.operationcode.org" -> "john482" + if '@' not in recipient: + print(f"Invalid recipient format: {recipient}") + continue + + alias = recipient.split('@')[0].lower() + print(f"Looking up alias: {alias}") + + # Query Airtable for the mapping + mapping = lookup_alias_in_airtable(alias) + + if not mapping: + print(f"No active mapping found for alias: {alias}") + # Silently drop emails to unknown aliases + continue + + forward_to = mapping.get('Email') + donor_name = mapping.get('Name', 'Member') + + if not forward_to: + print(f"No Email field in mapping for alias: {alias}") + continue + + print(f"Forwarding to: {forward_to} ({donor_name})") + + try: + # Get the raw email from S3 + raw_email = get_email_from_s3(message_id) + + # Forward it + response = forward_email(raw_email, forward_to, recipient) + print(f"Successfully forwarded. SES MessageId: {response.get('MessageId')}") + + except Exception as e: + error_msg = f"Error forwarding email for alias {alias}: {str(e)}" + print(error_msg) + sentry_sdk.capture_exception(e) + raise + + return { + 'statusCode': 200, + 'body': 'Processed' + } diff --git a/lambda/ses_email_forwarder/requirements.txt b/lambda/ses_email_forwarder/requirements.txt new file mode 100644 index 0000000..1b0134b --- /dev/null +++ b/lambda/ses_email_forwarder/requirements.txt @@ -0,0 +1,2 @@ +boto3>=1.34.0 # For local testing (included in Lambda runtime) +sentry-sdk>=1.40.0 # For error monitoring and alerting diff --git a/lambda/ses_email_forwarder/tests/__init__.py b/lambda/ses_email_forwarder/tests/__init__.py new file mode 100644 index 0000000..e8428be --- /dev/null +++ b/lambda/ses_email_forwarder/tests/__init__.py @@ -0,0 +1 @@ +# Tests for SES Email Forwarder Lambda function diff --git a/lambda/ses_email_forwarder/tests/fixtures/sample_ses_event.json b/lambda/ses_email_forwarder/tests/fixtures/sample_ses_event.json new file mode 100644 index 0000000..37e9124 --- /dev/null +++ b/lambda/ses_email_forwarder/tests/fixtures/sample_ses_event.json @@ -0,0 +1,74 @@ +{ + "Records": [ + { + "eventSource": "aws:ses", + "eventVersion": "1.0", + "ses": { + "mail": { + "timestamp": "2026-01-28T12:00:00.000Z", + "source": "sender@example.com", + "messageId": "abc123def456", + "destination": [ + "testuser@coders.operationcode.org" + ], + "headersTruncated": false, + "headers": [ + { + "name": "From", + "value": "sender@example.com" + }, + { + "name": "To", + "value": "testuser@coders.operationcode.org" + }, + { + "name": "Subject", + "value": "Test Email" + }, + { + "name": "MIME-Version", + "value": "1.0" + }, + { + "name": "Content-Type", + "value": "text/plain; charset=UTF-8" + } + ], + "commonHeaders": { + "from": [ + "sender@example.com" + ], + "to": [ + "testuser@coders.operationcode.org" + ], + "subject": "Test Email" + } + }, + "receipt": { + "timestamp": "2026-01-28T12:00:00.000Z", + "processingTimeMillis": 100, + "recipients": [ + "testuser@coders.operationcode.org" + ], + "spamVerdict": { + "status": "PASS" + }, + "virusVerdict": { + "status": "PASS" + }, + "spfVerdict": { + "status": "PASS" + }, + "dkimVerdict": { + "status": "PASS" + }, + "action": { + "type": "Lambda", + "functionArn": "arn:aws:lambda:us-east-1:123456789012:function:ses-email-forwarder", + "invocationType": "Event" + } + } + } + } + ] +} diff --git a/lambda/ses_email_forwarder/tests/test_handler.py b/lambda/ses_email_forwarder/tests/test_handler.py new file mode 100644 index 0000000..9f2941d --- /dev/null +++ b/lambda/ses_email_forwarder/tests/test_handler.py @@ -0,0 +1,300 @@ +import json +import os +import sys +import unittest +from unittest.mock import Mock, patch, MagicMock +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart +import base64 + +# Add parent directory to path for imports +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +import handler + + +class TestLambdaHandler(unittest.TestCase): + """Test suite for SES email forwarder Lambda function.""" + + def setUp(self): + """Set up test fixtures.""" + # Set required environment variables + os.environ['EMAIL_BUCKET'] = 'test-bucket' + os.environ['AIRTABLE_SECRET_NAME'] = 'test-secret' + os.environ['FORWARD_FROM_EMAIL'] = 'noreply@coders.operationcode.org' + os.environ['AWS_SES_REGION'] = 'us-east-1' + os.environ['ENVIRONMENT'] = 'test' + + # Reset caches + handler._secrets_cache = None + handler._config_cache = None + handler._s3_client = None + handler._ses_client = None + handler._secrets_client = None + + # Load sample SES event + fixture_path = os.path.join(os.path.dirname(__file__), 'fixtures', 'sample_ses_event.json') + with open(fixture_path, 'r') as f: + self.sample_event = json.load(f) + + def tearDown(self): + """Clean up after tests.""" + handler._secrets_cache = None + handler._config_cache = None + handler._s3_client = None + handler._ses_client = None + handler._secrets_client = None + + def test_get_airtable_credentials_caching(self): + """Test that credentials are cached after first retrieval.""" + mock_response = { + 'SecretString': json.dumps({ + 'airtable_api_key': 'test_key', + 'airtable_base_id': 'test_base', + 'airtable_table_name': 'Email Aliases', + 'sentry_dsn': 'https://test@test.ingest.sentry.io/test' + }) + } + + mock_secrets_client = Mock() + mock_secrets_client.get_secret_value.return_value = mock_response + + with patch.object(handler, 'get_secrets_client', return_value=mock_secrets_client): + # First call should hit Secrets Manager + creds1 = handler.get_airtable_credentials() + self.assertEqual(mock_secrets_client.get_secret_value.call_count, 1) + + # Second call should use cache + creds2 = handler.get_airtable_credentials() + self.assertEqual(mock_secrets_client.get_secret_value.call_count, 1) # Still 1, not 2 + + # Verify credentials + self.assertEqual(creds1['airtable_api_key'], 'test_key') + self.assertEqual(creds1, creds2) + + @patch('handler.urllib.request.urlopen') + def test_lookup_alias_active(self, mock_urlopen): + """Test looking up an active alias in Airtable.""" + # Mock Secrets Manager + with patch.object(handler, 'get_airtable_credentials', return_value={ + 'airtable_api_key': 'test_key', + 'airtable_base_id': 'test_base', + 'airtable_table_name': 'Email Aliases' + }): + # Mock Airtable API response + mock_response = MagicMock() + mock_response.read.return_value = json.dumps({ + 'records': [{ + 'id': 'rec123', + 'fields': { + 'Alias': 'testuser', + 'Email': 'test@example.com', + 'Name': 'Test User', + 'Status': 'active' + } + }] + }).encode() + mock_response.__enter__.return_value = mock_response + mock_urlopen.return_value = mock_response + + result = handler.lookup_alias_in_airtable('testuser') + + self.assertIsNotNone(result) + self.assertEqual(result['Email'], 'test@example.com') + self.assertEqual(result['Name'], 'Test User') + + @patch('handler.urllib.request.urlopen') + def test_lookup_alias_not_found(self, mock_urlopen): + """Test looking up a non-existent alias.""" + with patch.object(handler, 'get_airtable_credentials', return_value={ + 'airtable_api_key': 'test_key', + 'airtable_base_id': 'test_base', + 'airtable_table_name': 'Email Aliases' + }): + # Mock empty Airtable response + mock_response = MagicMock() + mock_response.read.return_value = json.dumps({'records': []}).encode() + mock_response.__enter__.return_value = mock_response + mock_urlopen.return_value = mock_response + + result = handler.lookup_alias_in_airtable('nonexistent') + + self.assertIsNone(result) + + def test_get_email_from_s3(self): + """Test retrieving email from S3.""" + mock_email_content = b"From: sender@example.com\nTo: test@coders.operationcode.org\n\nTest body" + + mock_response = { + 'Body': MagicMock() + } + mock_response['Body'].read.return_value = mock_email_content + + mock_s3_client = Mock() + mock_s3_client.get_object.return_value = mock_response + + with patch.object(handler, 'get_s3_client', return_value=mock_s3_client): + result = handler.get_email_from_s3('test-message-id') + + self.assertEqual(result, mock_email_content) + mock_s3_client.get_object.assert_called_once_with( + Bucket='test-bucket', + Key='test-message-id' + ) + + def test_forward_email_simple(self): + """Test forwarding a simple text email.""" + # Create a simple email + original_msg = MIMEText('Test email body', 'plain') + original_msg['From'] = 'sender@example.com' + original_msg['To'] = 'test@coders.operationcode.org' + original_msg['Subject'] = 'Test Subject' + raw_email = original_msg.as_bytes() + + mock_ses_response = {'MessageId': 'ses-msg-123'} + mock_ses_client = Mock() + mock_ses_client.send_raw_email.return_value = mock_ses_response + + with patch.object(handler, 'get_ses_client', return_value=mock_ses_client): + result = handler.forward_email(raw_email, 'recipient@example.com', 'test@coders.operationcode.org') + + self.assertEqual(result['MessageId'], 'ses-msg-123') + mock_ses_client.send_raw_email.assert_called_once() + + # Check the call arguments + call_args = mock_ses_client.send_raw_email.call_args + self.assertEqual(call_args[1]['Source'], 'noreply@coders.operationcode.org') + self.assertEqual(call_args[1]['Destinations'], ['recipient@example.com']) + + def test_forward_email_with_attachment(self): + """Test forwarding an email with attachment.""" + # Create email with attachment + msg = MIMEMultipart() + msg['From'] = 'sender@example.com' + msg['To'] = 'test@coders.operationcode.org' + msg['Subject'] = 'Test with Attachment' + + # Add body + msg.attach(MIMEText('Email body', 'plain')) + + # Add attachment + attachment = MIMEText('attachment content', 'plain') + attachment.add_header('Content-Disposition', 'attachment', filename='test.txt') + msg.attach(attachment) + + raw_email = msg.as_bytes() + + mock_ses_response = {'MessageId': 'ses-msg-456'} + mock_ses_client = Mock() + mock_ses_client.send_raw_email.return_value = mock_ses_response + + with patch.object(handler, 'get_ses_client', return_value=mock_ses_client): + result = handler.forward_email(raw_email, 'recipient@example.com', 'test@coders.operationcode.org') + + self.assertEqual(result['MessageId'], 'ses-msg-456') + + @patch('handler.init_sentry') + @patch('handler.get_email_from_s3') + @patch('handler.forward_email') + @patch('handler.lookup_alias_in_airtable') + def test_lambda_handler_success(self, mock_lookup, mock_forward, mock_get_email, mock_sentry): + """Test successful Lambda handler execution.""" + # Mock Airtable lookup + mock_lookup.return_value = { + 'Email': 'recipient@example.com', + 'Name': 'Test User', + 'Status': 'active' + } + + # Mock S3 email retrieval + mock_get_email.return_value = b"From: sender@example.com\nSubject: Test\n\nBody" + + # Mock SES send + mock_forward.return_value = {'MessageId': 'test-msg-id'} + + result = handler.lambda_handler(self.sample_event, None) + + self.assertEqual(result['statusCode'], 200) + self.assertEqual(result['body'], 'Processed') + mock_lookup.assert_called_once_with('testuser') + mock_get_email.assert_called_once() + mock_forward.assert_called_once() + + @patch('handler.init_sentry') + @patch('handler.lookup_alias_in_airtable') + def test_lambda_handler_inactive_alias(self, mock_lookup, mock_sentry): + """Test Lambda handler with inactive alias (should not forward).""" + # Mock Airtable lookup returning None (inactive or not found) + mock_lookup.return_value = None + + result = handler.lambda_handler(self.sample_event, None) + + self.assertEqual(result['statusCode'], 200) + self.assertEqual(result['body'], 'Processed') + mock_lookup.assert_called_once_with('testuser') + + @patch('handler.init_sentry') + @patch('handler.get_email_from_s3') + @patch('handler.lookup_alias_in_airtable') + def test_lambda_handler_missing_personal_email(self, mock_lookup, mock_get_email, mock_sentry): + """Test Lambda handler when Email is missing from mapping.""" + # Mock Airtable lookup with missing Email + mock_lookup.return_value = { + 'Name': 'Test User', + 'Status': 'active' + # Email is missing + } + + result = handler.lambda_handler(self.sample_event, None) + + self.assertEqual(result['statusCode'], 200) + self.assertEqual(result['body'], 'Processed') + # Should not attempt to get email from S3 + mock_get_email.assert_not_called() + + @patch('handler.init_sentry') + @patch('handler.get_email_from_s3') + @patch('handler.forward_email') + @patch('handler.lookup_alias_in_airtable') + @patch('handler.sentry_sdk') + def test_lambda_handler_forward_error(self, mock_sentry_sdk, mock_lookup, mock_forward, mock_get_email, mock_sentry): + """Test Lambda handler error handling when forwarding fails.""" + mock_lookup.return_value = { + 'Email': 'recipient@example.com', + 'Name': 'Test User', + 'Status': 'active' + } + + mock_get_email.return_value = b"test email" + mock_forward.side_effect = Exception("SES error") + + with self.assertRaises(Exception): + handler.lambda_handler(self.sample_event, None) + + # Verify Sentry was called to capture the exception + mock_sentry_sdk.capture_exception.assert_called() + + def test_lambda_handler_invalid_recipient_format(self): + """Test Lambda handler with invalid recipient format.""" + invalid_event = { + 'Records': [{ + 'eventSource': 'aws:ses', + 'ses': { + 'mail': { + 'messageId': 'test-msg', + 'source': 'sender@example.com', + 'destination': ['invalid-no-at-sign'] + } + } + }] + } + + with patch('handler.init_sentry'): + result = handler.lambda_handler(invalid_event, None) + + self.assertEqual(result['statusCode'], 200) + self.assertEqual(result['body'], 'Processed') + + +if __name__ == '__main__': + unittest.main() diff --git a/plans/ses-email-forwarding-guide.md b/plans/ses-email-forwarding-guide.md new file mode 100644 index 0000000..c313801 --- /dev/null +++ b/plans/ses-email-forwarding-guide.md @@ -0,0 +1,795 @@ +# AWS SES Email Forwarding System for Operation Code + +## Overview + +This document describes the architecture and implementation plan for an email forwarding system that: + +1. Allows donors who set up recurring donations to receive a custom email alias (e.g., `john@coders.operationcode.org`) +2. Forwards emails sent to that alias to the donor's personal email address +3. Stores alias mappings in Airtable (integrated with existing automation workflows) +4. Notifies a Slack channel when new aliases are created +5. Monitors for lapsed payments and alerts accordingly + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────────────┐ +│ Route 53 │ +│ MX record: coders.operationcode.org → inbound-smtp.us-east-1.amazonaws.com │ +│ TXT records: SPF, DKIM verification │ +└─────────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────────┐ +│ AWS SES (Email Receiving) │ +│ │ +│ Receipt Rule Set: "coders-email-forwarding" │ +│ ┌─────────────────────────────────────────────────────────────────────────┐ │ +│ │ Rule: "forward-to-members" │ │ +│ │ Recipients: coders.operationcode.org │ │ +│ │ Actions: │ │ +│ │ 1. Store in S3 (opcode-ses-incoming-emails bucket) │ │ +│ │ 2. Invoke Lambda (ses-email-forwarder) │ │ +│ └─────────────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────────┘ + │ + ┌─────────────┴─────────────┐ + ▼ ▼ +┌──────────────────────────────────┐ ┌──────────────────────────────────┐ +│ S3 Bucket │ │ Lambda: ses-email-forwarder │ +│ opcode-ses-incoming-emails/ │ │ │ +│ └── emails/{message-id} │ │ 1. Parse recipient (alias) │ +│ (raw email stored) │ │ 2. Query Airtable for mapping │ +│ │ │ 3. Fetch email from S3 │ +└──────────────────────────────────┘ │ 4. Rewrite headers │ + │ 5. Forward via SES │ + └──────────────────────────────────┘ + │ + ┌───────────────┴───────────────┐ + ▼ ▼ + ┌──────────────────┐ ┌──────────────┐ + │ Airtable │ │ SES Send │ + │ │ │ │ + │ Email Aliases │ │ Forward to │ + │ Base/Table │ │ personal │ + │ │ │ email │ + └──────────────────┘ └──────────────┘ +``` + +## Provisioning Flow (via Zapier) + +``` +┌─────────────────────────────────────────────────────────────────────────────────┐ +│ NEW DONOR PROVISIONING │ +├─────────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ Stripe Payment Link ──► Stripe Subscription Created ──► Zapier Trigger │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ Generate Alias │ │ +│ │ (firstname123) │ │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ Create Airtable │ │ +│ │ Record │ │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ Slack Notify │ │ +│ │ #new-members │ │ +│ └─────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────────────────┐ +│ LAPSED PAYMENT HANDLING │ +├─────────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ Stripe ──► invoice.payment_failed ──► Zapier Trigger │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ Find Airtable │ │ +│ │ Record by Email │ │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ Update Status │ │ +│ │ → "lapsed" │ │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ Slack Alert │ │ +│ │ #payment-issues │ │ +│ └─────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Component Specifications + +### 1. Route 53 DNS Records + +Add these records to the `operationcode.org` hosted zone for the `coders` subdomain: + +| Type | Name | Value | TTL | +|------|------|-------|-----| +| MX | coders.operationcode.org | `10 inbound-smtp.us-east-1.amazonaws.com` | 300 | +| TXT | coders.operationcode.org | `v=spf1 include:amazonses.com ~all` | 300 | +| CNAME | `{selector1}._domainkey.coders.operationcode.org` | `{provided by SES}` | 300 | +| CNAME | `{selector2}._domainkey.coders.operationcode.org` | `{provided by SES}` | 300 | +| CNAME | `{selector3}._domainkey.coders.operationcode.org` | `{provided by SES}` | 300 | + +> **Note:** The DKIM CNAME records will be provided by SES during domain verification. There will be 3 of them. + +--- + +### 2. Airtable Schema + +**Base Name:** `Operation Code Automation` (or existing base) + +**Table Name:** `Email Aliases` + +| Field Name | Field Type | Description | Example | +|------------|------------|-------------|---------| +| `alias` | Single line text (Primary) | The local part of the email | `john482` | +| `full_email` | Formula | `{alias} & "@coders.operationcode.org"` | `john482@coders.operationcode.org` | +| `personal_email` | Email | Donor's real email address | `john@gmail.com` | +| `donor_name` | Single line text | Full name | `John Smith` | +| `status` | Single select | Options: `active`, `lapsed`, `cancelled` | `active` | +| `stripe_customer_id` | Single line text | For payment tracking | `cus_ABC123` | +| `stripe_subscription_id` | Single line text | Subscription reference | `sub_XYZ789` | +| `last_payment_date` | Date | Last successful payment | `2026-01-15` | +| `created_at` | Created time | Auto-populated | `2026-01-01` | +| `notes` | Long text | Admin notes | | + +**Views to Create:** +- `Active Aliases` - Filter: status = "active" +- `Lapsed (30+ days)` - Filter: status = "lapsed" OR last_payment_date < 30 days ago +- `All Aliases` - No filter + +--- + +### 3. S3 Bucket + +**Bucket Name:** `opcode-ses-incoming-emails` + +**Configuration:** +- Region: `us-east-1` (must match SES region) +- Versioning: Disabled (optional, enable if you want email history) +- Encryption: SSE-S3 (default) +- Lifecycle Rule: Delete objects after 7 days (emails are ephemeral, just for forwarding) + +**Bucket Policy:** +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowSESPuts", + "Effect": "Allow", + "Principal": { + "Service": "ses.amazonaws.com" + }, + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::opcode-ses-incoming-emails/*", + "Condition": { + "StringEquals": { + "AWS:SourceAccount": "${AWS_ACCOUNT_ID}" + } + } + } + ] +} +``` + +--- + +### 4. Lambda Function: `ses-email-forwarder` + +**Runtime:** Python 3.12 +**Memory:** 256 MB +**Timeout:** 30 seconds +**Architecture:** arm64 (Graviton, cheaper) + +**Environment Variables:** + +| Variable | Value | +|----------|-------| +| `EMAIL_BUCKET` | `opcode-ses-incoming-emails` | +| `AIRTABLE_API_KEY` | `pat...` (Personal Access Token) | +| `AIRTABLE_BASE_ID` | `app...` (from Airtable URL) | +| `AIRTABLE_TABLE_NAME` | `Email Aliases` | +| `FORWARD_FROM_EMAIL` | `noreply@coders.operationcode.org` | +| `AWS_SES_REGION` | `us-east-1` | + +**IAM Role Policy:** +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject" + ], + "Resource": "arn:aws:s3:::opcode-ses-incoming-emails/*" + }, + { + "Effect": "Allow", + "Action": [ + "ses:SendRawEmail" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "arn:aws:logs:*:*:*" + } + ] +} +``` + +**Lambda Function Code:** + +```python +import boto3 +import email +import os +import json +import urllib.request +import urllib.error +from email import policy +from email.parser import BytesParser +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.mime.base import MIMEBase +from email import encoders + +# Configuration from environment variables +EMAIL_BUCKET = os.environ['EMAIL_BUCKET'] +AIRTABLE_API_KEY = os.environ['AIRTABLE_API_KEY'] +AIRTABLE_BASE_ID = os.environ['AIRTABLE_BASE_ID'] +AIRTABLE_TABLE_NAME = os.environ['AIRTABLE_TABLE_NAME'] +FORWARD_FROM_EMAIL = os.environ['FORWARD_FROM_EMAIL'] +AWS_SES_REGION = os.environ.get('AWS_SES_REGION', 'us-east-1') + +s3_client = boto3.client('s3') +ses_client = boto3.client('ses', region_name=AWS_SES_REGION) + + +def lookup_alias_in_airtable(alias: str) -> dict | None: + """ + Query Airtable to find the mapping for a given alias. + Returns the record if found and active, None otherwise. + """ + url = f"https://api.airtable.com/v0/{AIRTABLE_BASE_ID}/{urllib.parse.quote(AIRTABLE_TABLE_NAME)}" + + # Filter for exact alias match + params = urllib.parse.urlencode({ + 'filterByFormula': f"AND({{alias}} = '{alias}', {{status}} = 'active')", + 'maxRecords': 1 + }) + + full_url = f"{url}?{params}" + + req = urllib.request.Request( + full_url, + headers={ + 'Authorization': f'Bearer {AIRTABLE_API_KEY}', + 'Content-Type': 'application/json' + } + ) + + try: + with urllib.request.urlopen(req) as response: + data = json.loads(response.read().decode()) + records = data.get('records', []) + if records: + return records[0]['fields'] + return None + except urllib.error.HTTPError as e: + print(f"Airtable API error: {e.code} - {e.read().decode()}") + return None + + +def get_email_from_s3(message_id: str) -> bytes: + """Retrieve the raw email from S3.""" + response = s3_client.get_object( + Bucket=EMAIL_BUCKET, + Key=message_id + ) + return response['Body'].read() + + +def forward_email(raw_email: bytes, forward_to: str, original_recipient: str) -> dict: + """ + Parse the original email and forward it to the destination address. + Rewrites headers to comply with SES requirements while preserving + the original sender information. + """ + # Parse the original email + original_msg = BytesParser(policy=policy.default).parsebytes(raw_email) + + # Extract original headers + original_from = original_msg['From'] + original_subject = original_msg['Subject'] or '(no subject)' + original_to = original_msg['To'] + original_date = original_msg['Date'] + original_message_id = original_msg['Message-ID'] + + # Create new message + new_msg = MIMEMultipart('mixed') + + # Set headers for forwarded message + # SES requires From to be a verified identity + new_msg['From'] = FORWARD_FROM_EMAIL + new_msg['To'] = forward_to + new_msg['Subject'] = original_subject + new_msg['Reply-To'] = original_from # Replies go to original sender + + # Add custom headers to preserve original info + new_msg['X-Original-From'] = original_from + new_msg['X-Original-To'] = original_recipient + new_msg['X-Forwarded-For'] = original_recipient + + # Handle multipart messages (with attachments) vs simple messages + if original_msg.is_multipart(): + # Copy all parts from original message + for part in original_msg.walk(): + content_type = part.get_content_type() + content_disposition = str(part.get('Content-Disposition', '')) + + if content_type == 'multipart/mixed' or content_type == 'multipart/alternative': + continue + + if 'attachment' in content_disposition: + # Handle attachments + new_part = MIMEBase(*content_type.split('/')) + new_part.set_payload(part.get_payload(decode=True)) + encoders.encode_base64(new_part) + new_part.add_header( + 'Content-Disposition', + 'attachment', + filename=part.get_filename() or 'attachment' + ) + new_msg.attach(new_part) + else: + # Handle body parts + payload = part.get_payload(decode=True) + if payload: + if content_type == 'text/plain': + new_msg.attach(MIMEText(payload.decode('utf-8', errors='replace'), 'plain')) + elif content_type == 'text/html': + new_msg.attach(MIMEText(payload.decode('utf-8', errors='replace'), 'html')) + else: + # Simple message without attachments + payload = original_msg.get_payload(decode=True) + if payload: + content_type = original_msg.get_content_type() + if content_type == 'text/html': + new_msg.attach(MIMEText(payload.decode('utf-8', errors='replace'), 'html')) + else: + new_msg.attach(MIMEText(payload.decode('utf-8', errors='replace'), 'plain')) + + # Send via SES + response = ses_client.send_raw_email( + Source=FORWARD_FROM_EMAIL, + Destinations=[forward_to], + RawMessage={'Data': new_msg.as_bytes()} + ) + + return response + + +def handler(event, context): + """ + Lambda handler for SES incoming email events. + + Event structure: + { + "Records": [{ + "eventSource": "aws:ses", + "eventVersion": "1.0", + "ses": { + "mail": { + "messageId": "...", + "source": "sender@example.com", + "destination": ["recipient@coders.operationcode.org"] + }, + "receipt": { + "recipients": ["recipient@coders.operationcode.org"], + ... + } + } + }] + } + """ + print(f"Received event: {json.dumps(event)}") + + for record in event.get('Records', []): + ses_data = record.get('ses', {}) + mail_data = ses_data.get('mail', {}) + + message_id = mail_data.get('messageId') + recipients = mail_data.get('destination', []) + source = mail_data.get('source', 'unknown') + + print(f"Processing message {message_id} from {source} to {recipients}") + + for recipient in recipients: + # Extract alias from recipient address + # e.g., "john482@coders.operationcode.org" -> "john482" + if '@' not in recipient: + print(f"Invalid recipient format: {recipient}") + continue + + alias = recipient.split('@')[0].lower() + print(f"Looking up alias: {alias}") + + # Query Airtable for the mapping + mapping = lookup_alias_in_airtable(alias) + + if not mapping: + print(f"No active mapping found for alias: {alias}") + # Optionally: bounce the email or silently drop + continue + + forward_to = mapping.get('personal_email') + donor_name = mapping.get('donor_name', 'Member') + + if not forward_to: + print(f"No personal_email in mapping for alias: {alias}") + continue + + print(f"Forwarding to: {forward_to} ({donor_name})") + + try: + # Get the raw email from S3 + raw_email = get_email_from_s3(message_id) + + # Forward it + response = forward_email(raw_email, forward_to, recipient) + print(f"Successfully forwarded. SES MessageId: {response.get('MessageId')}") + + except Exception as e: + print(f"Error forwarding email: {str(e)}") + raise + + return { + 'statusCode': 200, + 'body': 'Processed' + } +``` + +**Required Python Packages:** +- None beyond standard library (boto3 is included in Lambda runtime) + +**Lambda Resource-based Policy (allow SES to invoke):** +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowSESInvoke", + "Effect": "Allow", + "Principal": { + "Service": "ses.amazonaws.com" + }, + "Action": "lambda:InvokeFunction", + "Resource": "arn:aws:lambda:us-east-1:${AWS_ACCOUNT_ID}:function:ses-email-forwarder", + "Condition": { + "StringEquals": { + "AWS:SourceAccount": "${AWS_ACCOUNT_ID}" + } + } + } + ] +} +``` + +--- + +### 5. SES Configuration + +#### Domain Identity Verification + +1. Go to SES Console → Identities → Create Identity +2. Select "Domain" +3. Enter: `coders.operationcode.org` +4. Enable "Easy DKIM" +5. SES will provide DNS records to add to Route 53 + +#### Receipt Rule Set + +**Rule Set Name:** `coders-email-forwarding` + +**Rule Configuration:** +- **Rule Name:** `forward-to-members` +- **Recipients:** `coders.operationcode.org` (catches all addresses on this subdomain) +- **Actions (in order):** + 1. **S3 Action:** + - Bucket: `opcode-ses-incoming-emails` + - Object key prefix: (leave empty) + 2. **Lambda Action:** + - Function: `ses-email-forwarder` + - Invocation type: `Event` (asynchronous) + +> **Important:** The rule set must be set as the "Active" rule set. + +#### Sending Authorization + +After domain verification, verify that `noreply@coders.operationcode.org` can send emails: +- The domain verification covers all addresses on that domain +- No additional email verification needed + +--- + +### 6. Zapier Zaps + +#### Zap 1: New Stripe Subscription → Create Email Alias + +**Trigger:** +- App: Stripe +- Event: New Subscription + +**Action 1: Code by Zapier (Generate Alias)** +```javascript +// Input: customer_name, customer_email from Stripe +const firstName = inputData.customer_name.split(' ')[0].toLowerCase(); +const randomSuffix = Math.floor(Math.random() * 900) + 100; // 3 digits +const alias = `${firstName}${randomSuffix}`; +return { alias: alias }; +``` + +**Action 2: Airtable - Create Record** +- Base: Operation Code Automation +- Table: Email Aliases +- Fields: + - alias: `{{alias from Step 2}}` + - personal_email: `{{Customer Email from Stripe}}` + - donor_name: `{{Customer Name from Stripe}}` + - status: `active` + - stripe_customer_id: `{{Customer ID from Stripe}}` + - stripe_subscription_id: `{{Subscription ID from Stripe}}` + - last_payment_date: `{{Current Date}}` + +**Action 3: Slack - Send Channel Message** +- Channel: `#coders-members` (or appropriate channel) +- Message: +``` +🎉 *New Coders Member!* +• Name: {{donor_name}} +• Email alias: {{alias}}@coders.operationcode.org +• Forwards to: {{personal_email}} +``` + +--- + +#### Zap 2: Stripe Payment Failed → Update Status & Alert + +**Trigger:** +- App: Stripe +- Event: Invoice Payment Failed + +**Action 1: Airtable - Find Record** +- Base: Operation Code Automation +- Table: Email Aliases +- Search Field: `stripe_customer_id` +- Search Value: `{{Customer ID from Stripe}}` + +**Action 2: Airtable - Update Record** (only if found) +- Record ID: `{{Record ID from Step 2}}` +- status: `lapsed` + +**Action 3: Slack - Send Channel Message** +- Channel: `#coders-admin` (or appropriate channel) +- Message: +``` +⚠️ *Payment Failed - Member Status Updated* +• Name: {{donor_name from Airtable}} +• Email alias: {{alias}}@coders.operationcode.org +• Personal email: {{personal_email}} +• Stripe Customer: {{Customer ID}} + +The member's email forwarding is still active but marked as lapsed. +``` + +--- + +#### Zap 3: Stripe Subscription Cancelled → Disable Forwarding + +**Trigger:** +- App: Stripe +- Event: Subscription Updated (filter for status = "canceled") + +**Action 1: Airtable - Find Record** +- Search by `stripe_subscription_id` + +**Action 2: Airtable - Update Record** +- status: `cancelled` + +**Action 3: Slack - Send Channel Message** +``` +📧 *Subscription Cancelled* +• Name: {{donor_name}} +• Email alias: {{alias}}@coders.operationcode.org (now inactive) +``` + +--- + +#### Zap 4 (Optional): Weekly Lapsed Member Report + +**Trigger:** +- App: Schedule by Zapier +- Event: Every Week on Monday + +**Action 1: Airtable - Find Records** +- View: `Lapsed (30+ days)` + +**Action 2: Slack - Send Channel Message** +``` +📊 *Weekly Lapsed Members Report* +{{Count}} members with lapsed payments: +{{List of names and aliases}} +``` + +--- + +## Cost Estimate (10-20 Users) + +### Assumptions +- 10-20 active email aliases +- Each user receives ~50 emails/month (500-1000 total incoming) +- Average email size: 50KB +- All emails are forwarded + +### Monthly Costs + +| Service | Usage | Unit Cost | Monthly Cost | +|---------|-------|-----------|--------------| +| **SES Receiving** | 1,000 emails | $0.10/1,000 | $0.10 | +| **SES Receiving (chunks)** | ~200 chunks (larger emails) | $0.09/1,000 | $0.02 | +| **SES Sending** | 1,000 emails (forwarded) | $0.10/1,000 | $0.10 | +| **SES Outbound Data** | ~50MB | $0.12/GB | $0.01 | +| **S3 Storage** | ~50MB (7-day retention) | $0.023/GB | ~$0.00 | +| **S3 Requests** | ~2,000 PUT/GET | $0.005/1,000 | $0.01 | +| **Lambda Invocations** | 1,000 | Free tier (1M/mo) | $0.00 | +| **Lambda Compute** | ~500 GB-seconds | Free tier (400K/mo) | $0.00 | +| **Route 53** | Hosted zone already exists | — | $0.00 | +| **Airtable** | Free tier or existing plan | — | $0.00 | + +### **Total Estimated Monthly Cost: $0.25 - $0.50** + +### Free Tier Coverage (First 12 Months) + +| Service | Free Tier Allowance | Your Usage | Status | +|---------|---------------------|------------|--------| +| SES | 3,000 messages/mo | ~2,000 | ✅ Covered | +| Lambda Requests | 1M/mo | ~1,000 | ✅ Covered | +| Lambda Compute | 400K GB-sec/mo | ~500 | ✅ Covered | +| S3 Storage | 5GB | ~50MB | ✅ Covered | + +**First 12 months: Essentially $0** +**After free tier expires: ~$0.25-0.50/month** + +--- + +## Implementation Checklist + +### Phase 1: AWS Infrastructure + +- [ ] **S3 Bucket** + - [ ] Create bucket `opcode-ses-incoming-emails` in us-east-1 + - [ ] Apply bucket policy for SES access + - [ ] Configure lifecycle rule (7-day expiration) + +- [ ] **SES Domain Verification** + - [ ] Add `coders.operationcode.org` as identity in SES + - [ ] Copy DKIM CNAME records + - [ ] Request production access (exit sandbox) if not already done + +- [ ] **Route 53 DNS Records** + - [ ] Add MX record for `coders` subdomain + - [ ] Add SPF TXT record + - [ ] Add DKIM CNAME records (3) + - [ ] Wait for verification (up to 72 hours, usually faster) + +- [ ] **Lambda Function** + - [ ] Create IAM role with required permissions + - [ ] Deploy `ses-email-forwarder` function + - [ ] Configure environment variables + - [ ] Add resource-based policy for SES invocation + +- [ ] **SES Receipt Rules** + - [ ] Create rule set `coders-email-forwarding` + - [ ] Create rule with S3 + Lambda actions + - [ ] Set rule set as active + +### Phase 2: Airtable Setup + +- [ ] Create `Email Aliases` table with schema above +- [ ] Create views: Active Aliases, Lapsed, All +- [ ] Generate Airtable Personal Access Token +- [ ] Test API access + +### Phase 3: Zapier Integration + +- [ ] Create Zap: Stripe Subscription → Airtable + Slack +- [ ] Create Zap: Stripe Payment Failed → Update Airtable + Slack +- [ ] Create Zap: Stripe Cancelled → Update Airtable + Slack +- [ ] (Optional) Create Zap: Weekly lapsed report + +### Phase 4: Testing + +- [ ] Create test record in Airtable manually +- [ ] Send test email to `test@coders.operationcode.org` +- [ ] Verify email arrives at destination +- [ ] Test with email containing attachment +- [ ] Test non-existent alias (should not forward) +- [ ] Test lapsed status (should not forward) +- [ ] Simulate Stripe subscription via test mode +- [ ] Verify full end-to-end flow + +--- + +## Troubleshooting + +### Email not being received by SES + +1. Check MX record propagation: `dig MX coders.operationcode.org` +2. Verify domain is verified in SES console +3. Check that receipt rule set is **active** + +### Email received but not forwarded + +1. Check CloudWatch Logs for Lambda function +2. Verify Airtable API key is valid +3. Check that alias exists in Airtable with status = "active" +4. Verify S3 bucket has the email object + +### Forwarded email going to spam + +1. Ensure SPF record is correct +2. Verify DKIM is passing (check email headers) +3. Consider adding DMARC record: + ``` + _dmarc.coders.operationcode.org TXT "v=DMARC1; p=none; rua=mailto:admin@operationcode.org" + ``` + +### Lambda timeout + +1. Increase timeout to 60 seconds +2. Check Airtable API response time +3. Check for large attachments (>10MB may fail) + +--- + +## Security Considerations + +1. **Airtable API Key:** Store in Lambda environment variables (encrypted at rest) +2. **S3 Bucket:** Not public, only SES can write, only Lambda can read +3. **Email Content:** Stored temporarily in S3, deleted after 7 days +4. **Spam Protection:** SES provides built-in spam/virus scanning +5. **Rate Limiting:** Consider CloudWatch alarm for unusual volume spikes + +--- + +## Future Enhancements + +1. **User Self-Service:** Allow donors to choose their own alias via a web form +2. **Alias Validation:** Check for duplicates before creating +3. **Email Analytics:** Track forwarding success/failure rates +4. **Bounce Handling:** Update Airtable if forwarding fails +5. **Custom Reply-From:** Allow sending FROM the alias (requires more SES config) diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index 8cc010f..00f2943 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -1,6 +1,26 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. +provider "registry.terraform.io/hashicorp/archive" { + version = "2.7.1" + constraints = ">= 2.0.0" + hashes = [ + "h1:A7EnRBVm4h9ryO9LwxYnKr4fy7ExPMwD5a1DsY7m1Y0=", + "zh:19881bb356a4a656a865f48aee70c0b8a03c35951b7799b6113883f67f196e8e", + "zh:2fcfbf6318dd514863268b09bbe19bfc958339c636bcbcc3664b45f2b8bf5cc6", + "zh:3323ab9a504ce0a115c28e64d0739369fe85151291a2ce480d51ccbb0c381ac5", + "zh:362674746fb3da3ab9bd4e70c75a3cdd9801a6cf258991102e2c46669cf68e19", + "zh:7140a46d748fdd12212161445c46bbbf30a3f4586c6ac97dd497f0c2565fe949", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:875e6ce78b10f73b1efc849bfcc7af3a28c83a52f878f503bb22776f71d79521", + "zh:b872c6ed24e38428d817ebfb214da69ea7eefc2c38e5a774db2ccd58e54d3a22", + "zh:cd6a44f731c1633ae5d37662af86e7b01ae4c96eb8b04144255824c3f350392d", + "zh:e0600f5e8da12710b0c52d6df0ba147a5486427c1a2cc78f31eea37a47ee1b07", + "zh:f21b2e2563bbb1e44e73557bcd6cdbc1ceb369d471049c40eb56cb84b6317a60", + "zh:f752829eba1cc04a479cf7ae7271526b402e206d5bcf1fcce9f535de5ff9e4e6", + ] +} + provider "registry.terraform.io/hashicorp/aws" { version = "5.100.0" constraints = ">= 3.29.0, >= 4.66.1, >= 5.0.0, >= 5.85.0, < 6.0.0" diff --git a/terraform/main.tf b/terraform/main.tf index c6e898a..0080caf 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -16,3 +16,11 @@ terraform { region = "us-east-2" } } + +# us-east-1 provider for SES + Lambda +# All SES-related resources (S3, Lambda, SES, CloudWatch) will use this +# Note: Default provider is defined in ecs.tf +provider "aws" { + alias = "us_east_1" + region = "us-east-1" +} diff --git a/terraform/route53.tf b/terraform/route53.tf new file mode 100644 index 0000000..9c037fe --- /dev/null +++ b/terraform/route53.tf @@ -0,0 +1,41 @@ +# Reference existing Route53 hosted zone +data "aws_route53_zone" "operationcode" { + name = "operationcode.org." +} + +# MX record for SES email receiving +resource "aws_route53_record" "coders_mx" { + zone_id = data.aws_route53_zone.operationcode.zone_id + name = "coders.operationcode.org" + type = "MX" + ttl = 300 + records = ["10 inbound-smtp.us-east-1.amazonaws.com"] +} + +# SPF record for email authentication +resource "aws_route53_record" "coders_spf" { + zone_id = data.aws_route53_zone.operationcode.zone_id + name = "coders.operationcode.org" + type = "TXT" + ttl = 300 + records = ["v=spf1 include:amazonses.com ~all"] +} + +# DKIM records (3 tokens from SES) +resource "aws_route53_record" "coders_dkim" { + count = 3 + zone_id = data.aws_route53_zone.operationcode.zone_id + name = "${module.ses_email_forwarder.ses_dkim_tokens[count.index]}._domainkey.coders.operationcode.org" + type = "CNAME" + ttl = 300 + records = ["${module.ses_email_forwarder.ses_dkim_tokens[count.index]}.dkim.amazonses.com"] +} + +# DMARC record for email policy +resource "aws_route53_record" "coders_dmarc" { + zone_id = data.aws_route53_zone.operationcode.zone_id + name = "_dmarc.coders.operationcode.org" + type = "TXT" + ttl = 300 + records = ["v=DMARC1; p=none; rua=mailto:admin@operationcode.org"] +} diff --git a/terraform/ses_email_forwarding.tf b/terraform/ses_email_forwarding.tf new file mode 100644 index 0000000..fe64c6d --- /dev/null +++ b/terraform/ses_email_forwarding.tf @@ -0,0 +1,32 @@ +# SES Email Forwarding Module +# Note: aws_caller_identity.current is defined in ecs.tf +# All resources in this module are deployed to us-east-1 +module "ses_email_forwarder" { + source = "./ses_email_forwarding" + + providers = { + aws = aws.us_east_1 # All resources in this module use us-east-1 + } + + domain = "coders.operationcode.org" + forward_from_email = "noreply@coders.operationcode.org" + environment = "prod" + account_id = data.aws_caller_identity.current.account_id + airtable_secret_name = "prod/ses_email_forwarder" # In us-east-2, cross-region access +} + +# Outputs +output "ses_email_forwarder_lambda_arn" { + description = "ARN of the SES email forwarder Lambda function" + value = module.ses_email_forwarder.lambda_function_arn +} + +output "ses_email_forwarder_bucket" { + description = "S3 bucket storing incoming emails" + value = module.ses_email_forwarder.s3_bucket_name +} + +output "ses_dkim_tokens" { + description = "DKIM tokens for DNS configuration" + value = module.ses_email_forwarder.ses_dkim_tokens +} diff --git a/terraform/ses_email_forwarding/data.tf b/terraform/ses_email_forwarding/data.tf new file mode 100644 index 0000000..7a1c5a3 --- /dev/null +++ b/terraform/ses_email_forwarding/data.tf @@ -0,0 +1,26 @@ +# Package Lambda code +data "archive_file" "lambda_zip" { + type = "zip" + source_dir = "${path.module}/../../lambda/ses_email_forwarder" + output_path = "${path.module}/lambda_function.zip" + + excludes = [ + "tests", + "tests/*", + "README.md", + "__pycache__", + "*.pyc", + ".pytest_cache", + ".venv", + ".venv/*", + "venv", + "venv/*" + ] +} + +# Reference Secrets Manager secret (in us-east-2) +# Note: We construct the ARN manually since the secret is in a different region +# Lambda in us-east-1 can access secrets in us-east-2 cross-region +locals { + secret_arn = "arn:aws:secretsmanager:us-east-2:${var.account_id}:secret:${var.airtable_secret_name}-*" +} diff --git a/terraform/ses_email_forwarding/main.tf b/terraform/ses_email_forwarding/main.tf new file mode 100644 index 0000000..647a2a9 --- /dev/null +++ b/terraform/ses_email_forwarding/main.tf @@ -0,0 +1,242 @@ +# S3 Bucket for storing incoming emails +resource "aws_s3_bucket" "incoming_emails" { + bucket = "opcode-ses-incoming-emails" + + tags = { + Name = "SES Incoming Emails" + Environment = var.environment + ManagedBy = "Terraform" + } +} + +# S3 Bucket lifecycle rule - delete emails after 7 days +resource "aws_s3_bucket_lifecycle_configuration" "incoming_emails" { + bucket = aws_s3_bucket.incoming_emails.id + + rule { + id = "delete-old-emails" + status = "Enabled" + + filter {} + + expiration { + days = 7 + } + } +} + +# S3 Bucket encryption +resource "aws_s3_bucket_server_side_encryption_configuration" "incoming_emails" { + bucket = aws_s3_bucket.incoming_emails.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +# S3 Bucket policy - allow SES to put objects +resource "aws_s3_bucket_policy" "incoming_emails" { + bucket = aws_s3_bucket.incoming_emails.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "AllowSESPuts" + Effect = "Allow" + Principal = { + Service = "ses.amazonaws.com" + } + Action = "s3:PutObject" + Resource = "${aws_s3_bucket.incoming_emails.arn}/*" + Condition = { + StringEquals = { + "AWS:SourceAccount" = var.account_id + } + } + } + ] + }) +} + +# IAM Role for Lambda +resource "aws_iam_role" "lambda_execution" { + name = "ses-email-forwarder-lambda-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "lambda.amazonaws.com" + } + } + ] + }) + + tags = { + Name = "SES Email Forwarder Lambda Role" + Environment = var.environment + ManagedBy = "Terraform" + } +} + +# IAM Policy for Lambda +resource "aws_iam_role_policy" "lambda_execution" { + name = "ses-email-forwarder-lambda-policy" + role = aws_iam_role.lambda_execution.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "S3GetObject" + Effect = "Allow" + Action = [ + "s3:GetObject" + ] + Resource = "${aws_s3_bucket.incoming_emails.arn}/*" + }, + { + Sid = "SESSendRawEmail" + Effect = "Allow" + Action = [ + "ses:SendRawEmail" + ] + Resource = "*" + }, + { + Sid = "SecretsManagerGetSecret" + Effect = "Allow" + Action = [ + "secretsmanager:GetSecretValue" + ] + Resource = local.secret_arn + }, + { + Sid = "CloudWatchLogs" + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = "arn:aws:logs:*:*:*" + } + ] + }) +} + +# Attach AWS managed policy for Lambda basic execution +resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { + role = aws_iam_role.lambda_execution.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} + +# CloudWatch Log Group for Lambda +resource "aws_cloudwatch_log_group" "lambda" { + name = "/aws/lambda/ses-email-forwarder" + retention_in_days = 14 + + tags = { + Name = "SES Email Forwarder Lambda Logs" + Environment = var.environment + ManagedBy = "Terraform" + } +} + +# Lambda Function +resource "aws_lambda_function" "ses_email_forwarder" { + filename = data.archive_file.lambda_zip.output_path + function_name = "ses-email-forwarder" + role = aws_iam_role.lambda_execution.arn + handler = "handler.lambda_handler" + source_code_hash = data.archive_file.lambda_zip.output_base64sha256 + runtime = "python3.12" + timeout = 30 + memory_size = 256 + architectures = ["arm64"] + layers = ["arn:aws:lambda:us-east-1:943013980633:layer:SentryPythonServerlessSDK:188"] + + environment { + variables = { + EMAIL_BUCKET = aws_s3_bucket.incoming_emails.id + AIRTABLE_SECRET_NAME = var.airtable_secret_name + FORWARD_FROM_EMAIL = var.forward_from_email + AWS_SES_REGION = "us-east-1" + ENVIRONMENT = var.environment + } + } + + depends_on = [ + aws_cloudwatch_log_group.lambda, + aws_iam_role_policy_attachment.lambda_basic_execution, + aws_iam_role_policy.lambda_execution + ] + + tags = { + Name = "SES Email Forwarder" + Environment = var.environment + ManagedBy = "Terraform" + } +} + +# Lambda Permission - allow SES to invoke +resource "aws_lambda_permission" "ses_invoke" { + statement_id = "AllowSESInvoke" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.ses_email_forwarder.function_name + principal = "ses.amazonaws.com" + source_account = var.account_id +} + +# SES Domain Identity +resource "aws_ses_domain_identity" "coders" { + domain = var.domain +} + +# SES Domain DKIM +resource "aws_ses_domain_dkim" "coders" { + domain = aws_ses_domain_identity.coders.domain +} + +# SES Receipt Rule Set +resource "aws_ses_receipt_rule_set" "main" { + rule_set_name = "coders-email-forwarding" +} + +# SES Receipt Rule +resource "aws_ses_receipt_rule" "forward" { + name = "forward-to-lambda" + rule_set_name = aws_ses_receipt_rule_set.main.rule_set_name + recipients = [var.domain] + enabled = true + scan_enabled = true + + # Action 1: Store email in S3 + s3_action { + bucket_name = aws_s3_bucket.incoming_emails.id + position = 1 + } + + # Action 2: Invoke Lambda + lambda_action { + function_arn = aws_lambda_function.ses_email_forwarder.arn + invocation_type = "Event" + position = 2 + } + + depends_on = [ + aws_s3_bucket_policy.incoming_emails, + aws_lambda_permission.ses_invoke + ] +} + +# Activate the receipt rule set +resource "aws_ses_active_receipt_rule_set" "main" { + rule_set_name = aws_ses_receipt_rule_set.main.rule_set_name +} diff --git a/terraform/ses_email_forwarding/outputs.tf b/terraform/ses_email_forwarding/outputs.tf new file mode 100644 index 0000000..4470193 --- /dev/null +++ b/terraform/ses_email_forwarding/outputs.tf @@ -0,0 +1,29 @@ +output "lambda_function_arn" { + description = "ARN of the SES email forwarder Lambda function" + value = aws_lambda_function.ses_email_forwarder.arn +} + +output "lambda_function_name" { + description = "Name of the Lambda function" + value = aws_lambda_function.ses_email_forwarder.function_name +} + +output "s3_bucket_name" { + description = "Name of the S3 bucket storing incoming emails" + value = aws_s3_bucket.incoming_emails.id +} + +output "ses_dkim_tokens" { + description = "DKIM tokens for DNS configuration" + value = aws_ses_domain_dkim.coders.dkim_tokens +} + +output "ses_receipt_rule_set_name" { + description = "Name of the SES receipt rule set" + value = aws_ses_receipt_rule_set.main.rule_set_name +} + +output "ses_domain_identity" { + description = "The domain identity verified in SES" + value = aws_ses_domain_identity.coders.domain +} diff --git a/terraform/ses_email_forwarding/variables.tf b/terraform/ses_email_forwarding/variables.tf new file mode 100644 index 0000000..ec49f16 --- /dev/null +++ b/terraform/ses_email_forwarding/variables.tf @@ -0,0 +1,24 @@ +variable "domain" { + description = "Domain for email forwarding (e.g., coders.operationcode.org)" + type = string +} + +variable "forward_from_email" { + description = "From address for forwarded emails (must be verified SES identity)" + type = string +} + +variable "environment" { + description = "Environment name (prod/staging)" + type = string +} + +variable "account_id" { + description = "AWS account ID" + type = string +} + +variable "airtable_secret_name" { + description = "Name of secret in Secrets Manager containing Airtable credentials" + type = string +} diff --git a/terraform/ses_email_forwarding/versions.tf b/terraform/ses_email_forwarding/versions.tf new file mode 100644 index 0000000..6d3da8c --- /dev/null +++ b/terraform/ses_email_forwarding/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + configuration_aliases = [aws] + } + archive = { + source = "hashicorp/archive" + version = ">= 2.0" + } + } +} From ffcb381456d0a9b6aa7cadec5bd19a55def042fc Mon Sep 17 00:00:00 2001 From: Irving Popovetsky Date: Thu, 29 Jan 2026 08:15:41 -0800 Subject: [PATCH 3/4] bounce and complaints handler --- .gitignore | 5 + lambda/ses_bounce_handler/README.md | 80 ++++ lambda/ses_bounce_handler/handler.py | 313 +++++++++++++++ lambda/ses_bounce_handler/requirements.txt | 2 + lambda/ses_bounce_handler/tests/__init__.py | 1 + .../tests/fixtures/sample_bounce_event.json | 18 + .../fixtures/sample_complaint_event.json | 10 + .../ses_bounce_handler/tests/test_handler.py | 361 ++++++++++++++++++ lambda/ses_email_forwarder/handler.py | 3 +- .../ses_email_forwarding/bounce_handling.tf | 231 +++++++++++ terraform/ses_email_forwarding/data.tf | 6 + terraform/ses_email_forwarding/outputs.tf | 21 + 12 files changed, 1050 insertions(+), 1 deletion(-) create mode 100644 lambda/ses_bounce_handler/README.md create mode 100644 lambda/ses_bounce_handler/handler.py create mode 100644 lambda/ses_bounce_handler/requirements.txt create mode 100644 lambda/ses_bounce_handler/tests/__init__.py create mode 100644 lambda/ses_bounce_handler/tests/fixtures/sample_bounce_event.json create mode 100644 lambda/ses_bounce_handler/tests/fixtures/sample_complaint_event.json create mode 100644 lambda/ses_bounce_handler/tests/test_handler.py create mode 100644 terraform/ses_email_forwarding/bounce_handling.tf diff --git a/.gitignore b/.gitignore index 4862f36..75cc097 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ *.tfstate *.tfstate.* *.tfvars +tfplan +*.tfplan # Python *.pyc @@ -19,6 +21,9 @@ env/ *.egg-info/ .pytest_cache/ .mypy_cache/ +.coverage +.coverage.* +htmlcov/ # Build artifacts *.zip diff --git a/lambda/ses_bounce_handler/README.md b/lambda/ses_bounce_handler/README.md new file mode 100644 index 0000000..09b292f --- /dev/null +++ b/lambda/ses_bounce_handler/README.md @@ -0,0 +1,80 @@ +# SES Bounce and Complaint Handler + +Lambda function that processes SES bounce and complaint notifications to maintain email sender reputation and automatically disable problematic email aliases. + +## Overview + +This Lambda function: +- Receives bounce and complaint notifications from SES via SNS topics +- Queries Airtable to find the affected email alias +- Updates bounce/complaint counts and timestamps +- Automatically disables aliases for permanent bounces and spam complaints + +## Architecture + +``` +SES Email → Bounce/Complaint → SNS Topic → Lambda → Airtable Update +``` + +## Environment Variables + +| Variable | Description | Example | +|----------|-------------|---------| +| `AIRTABLE_SECRET_NAME` | Name of secret in AWS Secrets Manager | `operation-code-automation` | +| `ENVIRONMENT` | Environment name for Sentry tagging | `prod` | + +## Secrets Manager + +The function expects the following fields in the secret: +- `airtable_api_key` - Airtable API key +- `airtable_base_id` - Airtable base ID (e.g., `appXXXXXXXXXXXXXX`) +- `airtable_table_name` - Table name (e.g., `Email Aliases`) +- `sentry_dsn` - Sentry DSN for error monitoring (optional) + +## Bounce Handling Logic + +| Bounce Type | Action | Rationale | +|-------------|--------|-----------| +| **Permanent** | Set `Status = "bouncing"` immediately | Invalid email - stop forwarding | +| **Transient** | Increment `bounce_count`, keep active | Temporary issue - allow retry | + +## Complaint Handling Logic + +All complaints immediately set `Status = "bouncing"` to protect sender reputation. + +## Airtable Fields Updated + +- `bounce_count` (Number) - Total bounce events +- `last_bounce_date` (Date) - Most recent bounce +- `last_bounce_type` (Single Select) - Permanent, Transient, or Undetermined +- `complaint_count` (Number) - Total complaint events +- `last_complaint_date` (Date) - Most recent complaint +- `Status` (Single Select) - Set to "bouncing" for permanent bounces and complaints + +## Testing + +Run unit tests: +```bash +cd lambda/ses_bounce_handler +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt +pip install pytest moto urllib3 +pytest tests/ -v +``` + +Test with SES Mailbox Simulator: +- `bounce@simulator.amazonses.com` - Permanent bounce +- `ooto@simulator.amazonses.com` - Transient bounce +- `complaint@simulator.amazonses.com` - Spam complaint + +## Monitoring + +CloudWatch Logs: `/aws/lambda/ses-bounce-handler` + +Sentry errors are automatically captured and reported. + +## Dependencies + +- boto3 - AWS SDK +- sentry-sdk - Error monitoring diff --git a/lambda/ses_bounce_handler/handler.py b/lambda/ses_bounce_handler/handler.py new file mode 100644 index 0000000..8f2b8c0 --- /dev/null +++ b/lambda/ses_bounce_handler/handler.py @@ -0,0 +1,313 @@ +import boto3 +import os +import json +import urllib.request +import urllib.error +import urllib.parse +import sentry_sdk +from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration + +# Cache for secrets and config +_secrets_cache = None +_config_cache = None + +# AWS clients (initialized lazily) +_secrets_client = None + + +def get_config(): + """Get configuration from environment variables with caching.""" + global _config_cache + if _config_cache is None: + _config_cache = { + 'airtable_secret_name': os.environ.get('AIRTABLE_SECRET_NAME', ''), + 'environment': os.environ.get('ENVIRONMENT', 'production') + } + return _config_cache + + +def get_secrets_client(): + """Get Secrets Manager client with lazy initialization.""" + global _secrets_client + if _secrets_client is None: + _secrets_client = boto3.client('secretsmanager', region_name='us-east-2') + return _secrets_client + + +def get_airtable_credentials(): + """ + Fetch Airtable credentials from Secrets Manager with caching. + + Returns: + dict: Contains airtable_api_key, airtable_base_id, airtable_table_name, sentry_dsn + """ + global _secrets_cache + if _secrets_cache is None: + config = get_config() + secret_name = config['airtable_secret_name'] + try: + secrets_client = get_secrets_client() + response = secrets_client.get_secret_value(SecretId=secret_name) + _secrets_cache = json.loads(response['SecretString']) + print(f"Successfully retrieved secrets from {secret_name}") + except Exception as e: + print(f"Error retrieving secrets from {secret_name}: {str(e)}") + raise + return _secrets_cache + + +def init_sentry(): + """Initialize Sentry with DSN from Secrets Manager.""" + try: + config = get_config() + credentials = get_airtable_credentials() + sentry_dsn = credentials.get('sentry_dsn') + if sentry_dsn: + sentry_sdk.init( + dsn=sentry_dsn, + integrations=[AwsLambdaIntegration()], + environment=config['environment'], + traces_sample_rate=0.1 + ) + print("Sentry initialized successfully") + else: + print("No Sentry DSN found in secrets") + except Exception as e: + print(f"Error initializing Sentry: {str(e)}") + + +def find_airtable_record_by_email(email): + """ + Query Airtable to find record with matching Email field. + + Args: + email: Email address to search for + + Returns: + dict with 'id' and 'fields', or None if not found + """ + credentials = get_airtable_credentials() + base_id = credentials['airtable_base_id'] + table_name = credentials['airtable_table_name'] + api_key = credentials['airtable_api_key'] + + # URL encode the filter formula + filter_formula = f"{{Email}}='{email}'" + encoded_formula = urllib.parse.quote(filter_formula) + url = f"https://api.airtable.com/v0/{base_id}/{urllib.parse.quote(table_name)}?filterByFormula={encoded_formula}" + + try: + # Create request with authorization header + req = urllib.request.Request(url) + req.add_header('Authorization', f'Bearer {api_key}') + + with urllib.request.urlopen(req) as response: + data = json.loads(response.read().decode('utf-8')) + records = data.get('records', []) + + if records: + print(f"Found Airtable record for {email}: {records[0]['id']}") + return records[0] + else: + print(f"No Airtable record found for {email}") + return None + + except urllib.error.HTTPError as e: + error_body = e.read().decode('utf-8') + print(f"HTTP error querying Airtable for {email}: {e.code} - {error_body}") + raise + except Exception as e: + print(f"Error querying Airtable for {email}: {str(e)}") + raise + + +def update_airtable_record(record_id, updates): + """ + Update Airtable record using PATCH API. + + Args: + record_id: Airtable record ID (starts with 'rec') + updates: Dict of field names to new values + """ + credentials = get_airtable_credentials() + base_id = credentials['airtable_base_id'] + table_name = credentials['airtable_table_name'] + api_key = credentials['airtable_api_key'] + + url = f"https://api.airtable.com/v0/{base_id}/{urllib.parse.quote(table_name)}/{record_id}" + + # Prepare the request body + body = { + 'fields': updates + } + json_data = json.dumps(body).encode('utf-8') + + try: + # Create PATCH request with authorization header + req = urllib.request.Request(url, data=json_data, method='PATCH') + req.add_header('Authorization', f'Bearer {api_key}') + req.add_header('Content-Type', 'application/json') + + with urllib.request.urlopen(req) as response: + result = json.loads(response.read().decode('utf-8')) + print(f"Successfully updated Airtable record {record_id}") + return result + + except urllib.error.HTTPError as e: + error_body = e.read().decode('utf-8') + print(f"HTTP error updating Airtable record {record_id}: {e.code} - {error_body}") + raise + except Exception as e: + print(f"Error updating Airtable record {record_id}: {str(e)}") + raise + + +def handle_bounce(message): + """ + Process SES bounce notification. + + SNS message structure: + { + "notificationType": "Bounce", + "bounce": { + "bounceType": "Permanent|Transient|Undetermined", + "bounceSubType": "General|NoEmail|MailboxFull|...", + "bouncedRecipients": [ + {"emailAddress": "user@example.com", "status": "5.1.1", ...} + ], + "timestamp": "2026-01-28T12:00:00.000Z" + }, + "mail": { + "messageId": "...", + "source": "noreply@coders.operationcode.org", + "destination": ["user@example.com"] + } + } + """ + bounce = message['bounce'] + bounce_type = bounce['bounceType'] + bounce_timestamp = bounce['timestamp'] + + for recipient in bounce['bouncedRecipients']: + email_address = recipient['emailAddress'] + + # Find Airtable record + record = find_airtable_record_by_email(email_address) + + if not record: + print(f"No Airtable record found for {email_address}") + continue + + # Get current bounce_count (default to 0 if field doesn't exist) + current_fields = record.get('fields', {}) + current_count = current_fields.get('bounce_count', 0) + + # Prepare updates + updates = { + 'bounce_count': current_count + 1, + 'last_bounce_date': bounce_timestamp[:10], # YYYY-MM-DD + 'last_bounce_type': bounce_type + } + + # Permanent bounces disable the alias immediately + if bounce_type == "Permanent": + updates['Status'] = "bouncing" + print(f"Permanent bounce for {email_address} - disabling alias") + else: + print(f"Transient bounce for {email_address} - incrementing counter") + + # Update Airtable + update_airtable_record(record['id'], updates) + + +def handle_complaint(message): + """ + Process SES complaint notification. + + SNS message structure: + { + "notificationType": "Complaint", + "complaint": { + "complainedRecipients": [ + {"emailAddress": "user@example.com"} + ], + "timestamp": "2026-01-28T12:00:00.000Z", + "complaintFeedbackType": "abuse|auth-failure|fraud|..." + }, + "mail": {...} + } + """ + complaint = message['complaint'] + complaint_timestamp = complaint['timestamp'] + + for recipient in complaint['complainedRecipients']: + email_address = recipient['emailAddress'] + + # Find Airtable record + record = find_airtable_record_by_email(email_address) + + if not record: + print(f"No Airtable record found for {email_address}") + continue + + # Get current count + current_fields = record.get('fields', {}) + current_count = current_fields.get('complaint_count', 0) + + # Prepare updates - complaints disable immediately + updates = { + 'complaint_count': current_count + 1, + 'last_complaint_date': complaint_timestamp[:10], + 'Status': "bouncing" # Disable on first complaint (reputation!) + } + + print(f"Complaint received for {email_address} - disabling alias") + + # Update Airtable + update_airtable_record(record['id'], updates) + + +def lambda_handler(event, context): + """ + Main Lambda handler for SNS notifications from SES. + + Event structure (SNS wrapper around SES notification): + { + "Records": [ + { + "EventSource": "aws:sns", + "Sns": { + "Message": "{...SES notification JSON...}", + "Subject": "Amazon SES Bounce Notification", + "TopicArn": "arn:aws:sns:us-east-1:...:ses-email-bounces" + } + } + ] + } + """ + # Initialize Sentry on first invocation + init_sentry() + + try: + for record in event['Records']: + # Parse SNS message + sns_message = record['Sns']['Message'] + message = json.loads(sns_message) + + notification_type = message.get('notificationType') + + print(f"Processing {notification_type} notification") + + if notification_type == 'Bounce': + handle_bounce(message) + elif notification_type == 'Complaint': + handle_complaint(message) + else: + print(f"Unknown notification type: {notification_type}") + + return {'statusCode': 200, 'body': 'Success'} + + except Exception as e: + print(f"Error processing notification: {str(e)}") + sentry_sdk.capture_exception(e) + raise # Re-raise to trigger Lambda retry diff --git a/lambda/ses_bounce_handler/requirements.txt b/lambda/ses_bounce_handler/requirements.txt new file mode 100644 index 0000000..7d1d79a --- /dev/null +++ b/lambda/ses_bounce_handler/requirements.txt @@ -0,0 +1,2 @@ +boto3>=1.34.0 +sentry-sdk>=1.40.0 diff --git a/lambda/ses_bounce_handler/tests/__init__.py b/lambda/ses_bounce_handler/tests/__init__.py new file mode 100644 index 0000000..b9dfd7f --- /dev/null +++ b/lambda/ses_bounce_handler/tests/__init__.py @@ -0,0 +1 @@ +# Test package for ses_bounce_handler diff --git a/lambda/ses_bounce_handler/tests/fixtures/sample_bounce_event.json b/lambda/ses_bounce_handler/tests/fixtures/sample_bounce_event.json new file mode 100644 index 0000000..c60d4e2 --- /dev/null +++ b/lambda/ses_bounce_handler/tests/fixtures/sample_bounce_event.json @@ -0,0 +1,18 @@ +{ + "Records": [ + { + "EventSource": "aws:sns", + "EventVersion": "1.0", + "EventSubscriptionArn": "arn:aws:sns:us-east-1:123456789012:ses-email-bounces:12345678-1234-1234-1234-123456789012", + "Sns": { + "Type": "Notification", + "MessageId": "12345678-1234-1234-1234-123456789012", + "TopicArn": "arn:aws:sns:us-east-1:123456789012:ses-email-bounces", + "Subject": "Amazon SES Email Receiving Notification", + "Message": "{\"notificationType\":\"Bounce\",\"bounce\":{\"bounceType\":\"Permanent\",\"bounceSubType\":\"General\",\"bouncedRecipients\":[{\"emailAddress\":\"test@example.com\",\"action\":\"failed\",\"status\":\"5.1.1\",\"diagnosticCode\":\"smtp; 550 5.1.1 user unknown\"}],\"timestamp\":\"2026-01-28T12:00:00.000Z\",\"feedbackId\":\"0000014c5b30d4d0-1234abcd-1234-1234-1234-123456789012-000000\",\"reportingMTA\":\"dns; email.amazonses.com\"},\"mail\":{\"timestamp\":\"2026-01-28T12:00:00.000Z\",\"source\":\"noreply@coders.operationcode.org\",\"sourceArn\":\"arn:aws:ses:us-east-1:123456789012:identity/coders.operationcode.org\",\"messageId\":\"0000014c5b30d4d0-abcd1234-1234-1234-1234-123456789012-000000\",\"destination\":[\"test@example.com\"]}}", + "Timestamp": "2026-01-28T12:00:00.000Z", + "SignatureVersion": "1" + } + } + ] +} diff --git a/lambda/ses_bounce_handler/tests/fixtures/sample_complaint_event.json b/lambda/ses_bounce_handler/tests/fixtures/sample_complaint_event.json new file mode 100644 index 0000000..4fbbb62 --- /dev/null +++ b/lambda/ses_bounce_handler/tests/fixtures/sample_complaint_event.json @@ -0,0 +1,10 @@ +{ + "Records": [ + { + "EventSource": "aws:sns", + "Sns": { + "Message": "{\"notificationType\":\"Complaint\",\"complaint\":{\"complainedRecipients\":[{\"emailAddress\":\"test@example.com\"}],\"timestamp\":\"2026-01-28T12:00:00.000Z\",\"feedbackId\":\"0000014c5b30d4d0-1234abcd-1234-1234-1234-123456789012-000000\",\"complaintFeedbackType\":\"abuse\"},\"mail\":{\"timestamp\":\"2026-01-28T12:00:00.000Z\",\"source\":\"noreply@coders.operationcode.org\",\"messageId\":\"0000014c5b30d4d0-abcd1234-1234-1234-1234-123456789012-000000\",\"destination\":[\"test@example.com\"]}}" + } + } + ] +} diff --git a/lambda/ses_bounce_handler/tests/test_handler.py b/lambda/ses_bounce_handler/tests/test_handler.py new file mode 100644 index 0000000..fa3f84c --- /dev/null +++ b/lambda/ses_bounce_handler/tests/test_handler.py @@ -0,0 +1,361 @@ +import unittest +from unittest.mock import patch, MagicMock +import json +import sys +import os + +# Add parent directory to path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +import handler + + +class TestBounceHandler(unittest.TestCase): + + def setUp(self): + """Reset global caches before each test""" + handler._config_cache = None + handler._secrets_cache = None + handler._secrets_client = None + + @patch.dict(os.environ, { + 'AIRTABLE_SECRET_NAME': 'test-secret', + 'ENVIRONMENT': 'test' + }) + def test_get_config(self): + """Test configuration loading from environment variables""" + config = handler.get_config() + self.assertEqual(config['airtable_secret_name'], 'test-secret') + self.assertEqual(config['environment'], 'test') + + @patch('handler.urllib.request.urlopen') + @patch('handler.get_airtable_credentials') + def test_find_record_by_email_found(self, mock_creds, mock_urlopen): + """Test finding Airtable record by email address""" + # Mock credentials + mock_creds.return_value = { + 'airtable_api_key': 'test_key', + 'airtable_base_id': 'appTEST123', + 'airtable_table_name': 'Email Aliases' + } + + # Mock Airtable API response + mock_response = MagicMock() + mock_response.read.return_value = json.dumps({ + 'records': [ + { + 'id': 'recABC123', + 'fields': { + 'Alias': 'testuser', + 'Email': 'test@example.com', + 'Status': 'active', + 'bounce_count': 0 + } + } + ] + }).encode('utf-8') + mock_urlopen.return_value.__enter__.return_value = mock_response + + # Test + record = handler.find_airtable_record_by_email('test@example.com') + + # Assert + self.assertIsNotNone(record) + self.assertEqual(record['id'], 'recABC123') + self.assertEqual(record['fields']['Email'], 'test@example.com') + + @patch('handler.urllib.request.urlopen') + @patch('handler.get_airtable_credentials') + def test_find_record_by_email_not_found(self, mock_creds, mock_urlopen): + """Test finding Airtable record when email doesn't exist""" + mock_creds.return_value = { + 'airtable_api_key': 'test_key', + 'airtable_base_id': 'appTEST123', + 'airtable_table_name': 'Email Aliases' + } + + # Mock empty response + mock_response = MagicMock() + mock_response.read.return_value = json.dumps({ + 'records': [] + }).encode('utf-8') + mock_urlopen.return_value.__enter__.return_value = mock_response + + # Test + record = handler.find_airtable_record_by_email('nonexistent@example.com') + + # Assert + self.assertIsNone(record) + + @patch('handler.urllib.request.urlopen') + @patch('handler.get_airtable_credentials') + def test_update_airtable_record(self, mock_creds, mock_urlopen): + """Test updating Airtable record""" + mock_creds.return_value = { + 'airtable_api_key': 'test_key', + 'airtable_base_id': 'appTEST123', + 'airtable_table_name': 'Email Aliases' + } + + # Mock successful update response + mock_response = MagicMock() + mock_response.read.return_value = json.dumps({ + 'id': 'recABC123', + 'fields': { + 'bounce_count': 1, + 'Status': 'bouncing' + } + }).encode('utf-8') + mock_urlopen.return_value.__enter__.return_value = mock_response + + # Test + updates = {'bounce_count': 1, 'Status': 'bouncing'} + result = handler.update_airtable_record('recABC123', updates) + + # Assert + self.assertIsNotNone(result) + self.assertEqual(result['id'], 'recABC123') + + @patch('handler.find_airtable_record_by_email') + @patch('handler.update_airtable_record') + def test_handle_permanent_bounce(self, mock_update, mock_find): + """Test permanent bounce disables alias""" + # Mock Airtable record + mock_find.return_value = { + 'id': 'recABC123', + 'fields': { + 'Alias': 'testuser', + 'Email': 'test@example.com', + 'Status': 'active', + 'bounce_count': 0 + } + } + + # Bounce message + message = { + 'notificationType': 'Bounce', + 'bounce': { + 'bounceType': 'Permanent', + 'bouncedRecipients': [ + {'emailAddress': 'test@example.com'} + ], + 'timestamp': '2026-01-28T12:00:00.000Z' + } + } + + # Test + handler.handle_bounce(message) + + # Assert + mock_update.assert_called_once() + call_args = mock_update.call_args[0] + self.assertEqual(call_args[0], 'recABC123') + updates = call_args[1] + self.assertEqual(updates['Status'], 'bouncing') + self.assertEqual(updates['bounce_count'], 1) + self.assertEqual(updates['last_bounce_type'], 'Permanent') + self.assertEqual(updates['last_bounce_date'], '2026-01-28') + + @patch('handler.find_airtable_record_by_email') + @patch('handler.update_airtable_record') + def test_handle_transient_bounce(self, mock_update, mock_find): + """Test transient bounce increments counter but keeps active""" + mock_find.return_value = { + 'id': 'recABC123', + 'fields': { + 'Alias': 'testuser', + 'Email': 'test@example.com', + 'Status': 'active', + 'bounce_count': 2 + } + } + + message = { + 'notificationType': 'Bounce', + 'bounce': { + 'bounceType': 'Transient', + 'bouncedRecipients': [ + {'emailAddress': 'test@example.com'} + ], + 'timestamp': '2026-01-28T12:00:00.000Z' + } + } + + handler.handle_bounce(message) + + call_args = mock_update.call_args[0] + updates = call_args[1] + # Transient bounce should NOT change status + self.assertNotIn('Status', updates) + self.assertEqual(updates['bounce_count'], 3) + self.assertEqual(updates['last_bounce_type'], 'Transient') + + @patch('handler.find_airtable_record_by_email') + @patch('handler.update_airtable_record') + def test_handle_bounce_no_record(self, mock_update, mock_find): + """Test bounce handling when no Airtable record exists""" + mock_find.return_value = None + + message = { + 'notificationType': 'Bounce', + 'bounce': { + 'bounceType': 'Permanent', + 'bouncedRecipients': [ + {'emailAddress': 'unknown@example.com'} + ], + 'timestamp': '2026-01-28T12:00:00.000Z' + } + } + + # Test - should not raise exception + handler.handle_bounce(message) + + # Assert - update should not be called + mock_update.assert_not_called() + + @patch('handler.find_airtable_record_by_email') + @patch('handler.update_airtable_record') + def test_handle_complaint(self, mock_update, mock_find): + """Test complaint disables alias immediately""" + mock_find.return_value = { + 'id': 'recABC123', + 'fields': { + 'Alias': 'testuser', + 'Email': 'test@example.com', + 'Status': 'active', + 'complaint_count': 0 + } + } + + message = { + 'notificationType': 'Complaint', + 'complaint': { + 'complainedRecipients': [ + {'emailAddress': 'test@example.com'} + ], + 'timestamp': '2026-01-28T12:00:00.000Z' + } + } + + handler.handle_complaint(message) + + call_args = mock_update.call_args[0] + updates = call_args[1] + self.assertEqual(updates['Status'], 'bouncing') + self.assertEqual(updates['complaint_count'], 1) + self.assertEqual(updates['last_complaint_date'], '2026-01-28') + + @patch('handler.find_airtable_record_by_email') + @patch('handler.update_airtable_record') + def test_handle_multiple_recipients(self, mock_update, mock_find): + """Test handling bounce with multiple recipients""" + mock_find.side_effect = [ + { + 'id': 'recABC123', + 'fields': {'Email': 'test1@example.com', 'bounce_count': 0} + }, + { + 'id': 'recDEF456', + 'fields': {'Email': 'test2@example.com', 'bounce_count': 1} + } + ] + + message = { + 'notificationType': 'Bounce', + 'bounce': { + 'bounceType': 'Permanent', + 'bouncedRecipients': [ + {'emailAddress': 'test1@example.com'}, + {'emailAddress': 'test2@example.com'} + ], + 'timestamp': '2026-01-28T12:00:00.000Z' + } + } + + handler.handle_bounce(message) + + # Assert both records were updated + self.assertEqual(mock_update.call_count, 2) + + @patch('handler.init_sentry') + @patch('handler.handle_bounce') + def test_lambda_handler_bounce(self, mock_handle_bounce, mock_init_sentry): + """Test Lambda handler with bounce notification""" + event = { + 'Records': [ + { + 'EventSource': 'aws:sns', + 'Sns': { + 'Message': json.dumps({ + 'notificationType': 'Bounce', + 'bounce': { + 'bounceType': 'Permanent', + 'bouncedRecipients': [ + {'emailAddress': 'test@example.com'} + ], + 'timestamp': '2026-01-28T12:00:00.000Z' + } + }) + } + } + ] + } + + result = handler.lambda_handler(event, None) + + self.assertEqual(result['statusCode'], 200) + mock_handle_bounce.assert_called_once() + mock_init_sentry.assert_called_once() + + @patch('handler.init_sentry') + @patch('handler.handle_complaint') + def test_lambda_handler_complaint(self, mock_handle_complaint, mock_init_sentry): + """Test Lambda handler with complaint notification""" + event = { + 'Records': [ + { + 'EventSource': 'aws:sns', + 'Sns': { + 'Message': json.dumps({ + 'notificationType': 'Complaint', + 'complaint': { + 'complainedRecipients': [ + {'emailAddress': 'test@example.com'} + ], + 'timestamp': '2026-01-28T12:00:00.000Z' + } + }) + } + } + ] + } + + result = handler.lambda_handler(event, None) + + self.assertEqual(result['statusCode'], 200) + mock_handle_complaint.assert_called_once() + + @patch('handler.init_sentry') + def test_lambda_handler_unknown_notification(self, mock_init_sentry): + """Test Lambda handler with unknown notification type""" + event = { + 'Records': [ + { + 'EventSource': 'aws:sns', + 'Sns': { + 'Message': json.dumps({ + 'notificationType': 'Unknown', + }) + } + } + ] + } + + result = handler.lambda_handler(event, None) + + # Should still return success but log unknown type + self.assertEqual(result['statusCode'], 200) + + +if __name__ == '__main__': + unittest.main() diff --git a/lambda/ses_email_forwarder/handler.py b/lambda/ses_email_forwarder/handler.py index 8c117bc..1866981 100644 --- a/lambda/ses_email_forwarder/handler.py +++ b/lambda/ses_email_forwarder/handler.py @@ -273,7 +273,8 @@ def forward_email(raw_email: bytes, forward_to: str, original_recipient: str) -> response = ses_client.send_raw_email( Source=config['forward_from_email'], Destinations=[forward_to], - RawMessage={'Data': new_msg.as_bytes()} + RawMessage={'Data': new_msg.as_bytes()}, + ConfigurationSetName='coders-email-forwarding-config' ) return response except Exception as e: diff --git a/terraform/ses_email_forwarding/bounce_handling.tf b/terraform/ses_email_forwarding/bounce_handling.tf new file mode 100644 index 0000000..3e998d5 --- /dev/null +++ b/terraform/ses_email_forwarding/bounce_handling.tf @@ -0,0 +1,231 @@ +# ============================================================================ +# SNS Topics for Bounce and Complaint Notifications +# ============================================================================ + +resource "aws_sns_topic" "ses_bounces" { + name = "ses-email-bounces" + + tags = { + Name = "SES Email Bounces" + Environment = var.environment + ManagedBy = "Terraform" + } +} + +resource "aws_sns_topic" "ses_complaints" { + name = "ses-email-complaints" + + tags = { + Name = "SES Email Complaints" + Environment = var.environment + ManagedBy = "Terraform" + } +} + +# ============================================================================ +# Lambda Function for Bounce and Complaint Handling +# ============================================================================ + +resource "aws_lambda_function" "bounce_handler" { + filename = data.archive_file.bounce_lambda_zip.output_path + function_name = "ses-bounce-handler" + role = aws_iam_role.bounce_lambda_execution.arn + handler = "handler.lambda_handler" + source_code_hash = data.archive_file.bounce_lambda_zip.output_base64sha256 + runtime = "python3.12" + timeout = 30 + memory_size = 256 + architectures = ["arm64"] + + # Sentry layer for error monitoring + layers = ["arn:aws:lambda:us-east-1:943013980633:layer:SentryPythonServerlessSDK:188"] + + environment { + variables = { + AIRTABLE_SECRET_NAME = var.airtable_secret_name + ENVIRONMENT = var.environment + } + } + + depends_on = [ + aws_cloudwatch_log_group.bounce_lambda, + aws_iam_role_policy_attachment.bounce_lambda_basic_execution, + aws_iam_role_policy.bounce_lambda_execution + ] + + tags = { + Name = "SES Bounce Handler" + Environment = var.environment + ManagedBy = "Terraform" + } +} + +# CloudWatch Log Group +resource "aws_cloudwatch_log_group" "bounce_lambda" { + name = "/aws/lambda/ses-bounce-handler" + retention_in_days = 14 + + tags = { + Name = "SES Bounce Handler Lambda Logs" + Environment = var.environment + ManagedBy = "Terraform" + } +} + +# ============================================================================ +# IAM Role and Policies for Bounce Handler Lambda +# ============================================================================ + +resource "aws_iam_role" "bounce_lambda_execution" { + name = "ses-bounce-handler-lambda-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "lambda.amazonaws.com" + } + } + ] + }) + + tags = { + Name = "SES Bounce Handler Lambda Role" + Environment = var.environment + ManagedBy = "Terraform" + } +} + +resource "aws_iam_role_policy" "bounce_lambda_execution" { + name = "ses-bounce-handler-lambda-policy" + role = aws_iam_role.bounce_lambda_execution.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "SecretsManagerGetSecret" + Effect = "Allow" + Action = [ + "secretsmanager:GetSecretValue" + ] + Resource = local.secret_arn + }, + { + Sid = "CloudWatchLogs" + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = "arn:aws:logs:*:*:*" + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "bounce_lambda_basic_execution" { + role = aws_iam_role.bounce_lambda_execution.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} + +# ============================================================================ +# SNS Subscriptions - Connect SNS Topics to Lambda +# ============================================================================ + +resource "aws_sns_topic_subscription" "bounces_to_lambda" { + topic_arn = aws_sns_topic.ses_bounces.arn + protocol = "lambda" + endpoint = aws_lambda_function.bounce_handler.arn +} + +resource "aws_sns_topic_subscription" "complaints_to_lambda" { + topic_arn = aws_sns_topic.ses_complaints.arn + protocol = "lambda" + endpoint = aws_lambda_function.bounce_handler.arn +} + +# ============================================================================ +# Lambda Permissions - Allow SNS to Invoke Lambda +# ============================================================================ + +resource "aws_lambda_permission" "sns_bounces_invoke" { + statement_id = "AllowSNSBouncesInvoke" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.bounce_handler.function_name + principal = "sns.amazonaws.com" + source_arn = aws_sns_topic.ses_bounces.arn +} + +resource "aws_lambda_permission" "sns_complaints_invoke" { + statement_id = "AllowSNSComplaintsInvoke" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.bounce_handler.function_name + principal = "sns.amazonaws.com" + source_arn = aws_sns_topic.ses_complaints.arn +} + +# ============================================================================ +# SES Configuration Set - Routes Bounce/Complaint Events to SNS +# ============================================================================ + +resource "aws_ses_configuration_set" "main" { + name = "coders-email-forwarding-config" + + reputation_metrics_enabled = true +} + +resource "aws_ses_event_destination" "bounces" { + name = "bounce-notifications" + configuration_set_name = aws_ses_configuration_set.main.name + enabled = true + matching_types = ["bounce"] + + sns_destination { + topic_arn = aws_sns_topic.ses_bounces.arn + } +} + +resource "aws_ses_event_destination" "complaints" { + name = "complaint-notifications" + configuration_set_name = aws_ses_configuration_set.main.name + enabled = true + matching_types = ["complaint"] + + sns_destination { + topic_arn = aws_sns_topic.ses_complaints.arn + } +} + +# ============================================================================ +# Data Source - Package Bounce Handler Lambda Code +# ============================================================================ + +data "archive_file" "bounce_lambda_zip" { + type = "zip" + source_dir = "${path.module}/../../lambda/ses_bounce_handler" + output_path = "${path.module}/bounce_handler.zip" + + excludes = [ + "tests", + "tests/*", + "README.md", + "__pycache__", + "__pycache__/*", + "*.pyc", + ".pytest_cache", + ".pytest_cache/*", + ".coverage", + ".coverage.*", + "htmlcov", + "htmlcov/*", + ".venv", + ".venv/*", + "venv", + "venv/*" + ] +} diff --git a/terraform/ses_email_forwarding/data.tf b/terraform/ses_email_forwarding/data.tf index 7a1c5a3..2f36104 100644 --- a/terraform/ses_email_forwarding/data.tf +++ b/terraform/ses_email_forwarding/data.tf @@ -9,8 +9,14 @@ data "archive_file" "lambda_zip" { "tests/*", "README.md", "__pycache__", + "__pycache__/*", "*.pyc", ".pytest_cache", + ".pytest_cache/*", + ".coverage", + ".coverage.*", + "htmlcov", + "htmlcov/*", ".venv", ".venv/*", "venv", diff --git a/terraform/ses_email_forwarding/outputs.tf b/terraform/ses_email_forwarding/outputs.tf index 4470193..b7675c6 100644 --- a/terraform/ses_email_forwarding/outputs.tf +++ b/terraform/ses_email_forwarding/outputs.tf @@ -27,3 +27,24 @@ output "ses_domain_identity" { description = "The domain identity verified in SES" value = aws_ses_domain_identity.coders.domain } + +# Bounce handling outputs +output "bounce_handler_lambda_arn" { + description = "ARN of the bounce handler Lambda function" + value = aws_lambda_function.bounce_handler.arn +} + +output "sns_bounce_topic_arn" { + description = "ARN of the SNS topic for bounce notifications" + value = aws_sns_topic.ses_bounces.arn +} + +output "sns_complaint_topic_arn" { + description = "ARN of the SNS topic for complaint notifications" + value = aws_sns_topic.ses_complaints.arn +} + +output "ses_configuration_set_name" { + description = "Name of the SES configuration set" + value = aws_ses_configuration_set.main.name +} From 8fb55bb79256c422304310fb018af71c4a125c06 Mon Sep 17 00:00:00 2001 From: Irving Popovetsky Date: Thu, 29 Jan 2026 08:30:48 -0800 Subject: [PATCH 4/4] fix issue found in integration testing --- lambda/ses_bounce_handler/handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lambda/ses_bounce_handler/handler.py b/lambda/ses_bounce_handler/handler.py index 8f2b8c0..a7179e4 100644 --- a/lambda/ses_bounce_handler/handler.py +++ b/lambda/ses_bounce_handler/handler.py @@ -294,7 +294,9 @@ def lambda_handler(event, context): sns_message = record['Sns']['Message'] message = json.loads(sns_message) - notification_type = message.get('notificationType') + # SES Configuration Set Event Destinations use 'eventType' + # Direct SES notifications use 'notificationType' + notification_type = message.get('eventType') or message.get('notificationType') print(f"Processing {notification_type} notification")