Add initial version of wiremock LocalStack extension

.github/workflows/test.yml

@@ -2,6 +2,15 @@ name: LocalStack TypeDB Extension Tests
 
 on:
   pull_request:
+    branches:
+      - main
+    paths:
+      - 'localstack-wiremock/**'
+  push:
+    branches:
+      - main
+    paths:
+      - 'localstack-wiremock/**'
   workflow_dispatch:
 
 env:

README.md

@@ -4,4 +4,4 @@ This repo contains miscellaneous utilities for LocalStack.
 
 ## License
 
-The code in this repo is available under the Apache 2.0 license.
+The code in this repo is available under the Apache 2.0 license. (unless stated otherwise)

localstack-wiremock/.gitignore

@@ -0,0 +1,8 @@
+.venv
+dist
+build
+**/*.egg-info
+.eggs
+.terraform*
+terraform.tfstate*
+*.zip

localstack-wiremock/Makefile

@@ -0,0 +1,48 @@
+VENV_BIN = python3 -m venv
+VENV_DIR ?= .venv
+VENV_ACTIVATE = $(VENV_DIR)/bin/activate
+VENV_RUN = . $(VENV_ACTIVATE)
+
+usage:          ## Shows usage for this Makefile
+	@cat Makefile | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}'
+
+venv: $(VENV_ACTIVATE)
+
+$(VENV_ACTIVATE): pyproject.toml
+	test -d .venv || $(VENV_BIN) .venv
+	$(VENV_RUN); pip install --upgrade pip setuptools plux
+	$(VENV_RUN); pip install -e .[dev]
+	touch $(VENV_DIR)/bin/activate
+
+clean:
+	rm -rf .venv/
+	rm -rf build/
+	rm -rf .eggs/
+	rm -rf *.egg-info/
+
+install: venv   ## Install dependencies
+	$(VENV_RUN); python -m plux entrypoints
+
+dist: venv      ## Create distribution
+	$(VENV_RUN); python -m build
+
+publish: clean-dist venv dist   ## Publish extension to pypi
+	$(VENV_RUN); pip install --upgrade twine; twine upload dist/*
+
+entrypoints: venv   # Generate plugin entrypoints for Python package
+	$(VENV_RUN); python -m plux entrypoints
+
+format:		    ## Run ruff to format the whole codebase
+	$(VENV_RUN); python -m ruff format .; python -m ruff check --output-format=full --fix .
+
+test:		    ## Run integration tests (requires LocalStack running with the Extension installed)
+	$(VENV_RUN); pytest tests $(PYTEST_ARGS)
+
+sample:		    ## Deploy sample app
+	bin/create-stubs.sh
+	(cd sample-app; tflocal init; tflocal apply)
+
+clean-dist: clean
+	rm -rf dist/
+
+.PHONY: clean clean-dist dist install publish usage venv format test

localstack-wiremock/README.md

@@ -0,0 +1,41 @@
+WireMock on LocalStack
+===============================
+
+This repo contains a [LocalStack Extension](https://github.com/localstack/localstack-extensions) that facilitates developing WireMock-based applications locally.
+
+## Prerequisites
+
+* Docker
+* LocalStack Pro (free trial available)
+* `localstack` CLI
+* `make`
+
+## Install from GitHub repository
+
+This extension can be installed directly from this Github repo via:
+
+```bash
+localstack extensions install "git+https://github.com/whummer/localstack-utils.git#egg=localstack-wiremock&subdirectory=localstack-wiremock"
+```
+
+## Install local development version
+
+To install the extension into localstack in developer mode, you will need Python 3.11, and create a virtual environment in the extensions project.
+
+In the newly generated project, simply run
+
+```bash
+make install
+```
+
+Then, to enable the extension for LocalStack, run
+
+```bash
+localstack extensions dev enable .
+```
+
+You can then start LocalStack with `EXTENSION_DEV_MODE=1` to load all enabled extensions:
+
+```bash
+EXTENSION_DEV_MODE=1 localstack start
+```

localstack-wiremock/bin/create-stubs.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+echo "Downloading WireMock stub definitions..."
+
+# Define the URL for the stub definitions and the temporary file path
+STUBS_URL="https://library.wiremock.org/catalog/api/p/personio.de/personio-de-personnel/personio.de-personnel-stubs.json"
+TMP_STUBS_FILE="/tmp/personio-stubs.json"
+
+# Define the WireMock server URL
+WIREMOCK_URL="http://localhost:8080"
+
+# Download the stub definitions
+curl -s -o "$TMP_STUBS_FILE" "$STUBS_URL"
+
+echo "Download complete. Stubs saved to $TMP_STUBS_FILE"
+echo "Importing stubs into WireMock..."
+
+# Send a POST request to WireMock's import endpoint with the downloaded file
+curl -v -X POST -H "Content-Type: application/json" --data-binary "@$TMP_STUBS_FILE" "$WIREMOCK_URL/__admin/mappings/import"
+
+echo ""
+echo "WireMock stub import request sent."

localstack-wiremock/localstack_wiremock/__init__.py

@@ -0,0 +1 @@
+name = "localstack_wiremock"

localstack-wiremock/localstack_wiremock/extension.py

@@ -0,0 +1,16 @@
+from localstack_wiremock.utils.docker import ProxiedDockerContainerExtension
+
+
+class WireMockExtension(ProxiedDockerContainerExtension):
+    name = "localstack-wiremock"
+
+    HOST = "wiremock.<domain>"
+    # name of the Docker image to spin up
+    DOCKER_IMAGE = "wiremock/wiremock"
+
+    def __init__(self):
+        super().__init__(
+            image_name=self.DOCKER_IMAGE,
+            container_ports=[8080],
+            host=self.HOST,
+        )

localstack-wiremock/localstack_wiremock/utils/docker.py

@@ -0,0 +1,183 @@
+import re
+import logging
+from functools import cache
+from typing import Callable
+import requests
+
+from localstack_wiremock.utils.h2_proxy import (
+    apply_http2_patches_for_grpc_support,
+    ProxyRequestMatcher,
+)
+from localstack.utils.docker_utils import DOCKER_CLIENT
+from localstack.extensions.api import Extension, http
+from localstack.http import Request
+from localstack.utils.container_utils.container_client import PortMappings
+from localstack.utils.net import get_addressable_container_host
+from localstack.utils.sync import retry
+from rolo import route
+from rolo.proxy import Proxy
+from rolo.routing import RuleAdapter, WithHost
+
+LOG = logging.getLogger(__name__)
+logging.basicConfig()
+
+# TODO: merge utils with code in TypeDB extension over time ...
+
+
+class ProxiedDockerContainerExtension(Extension, ProxyRequestMatcher):
+    name: str
+    """Name of this extension"""
+    image_name: str
+    """Docker image name"""
+    container_name: str | None
+    """Name of the Docker container spun up by the extension"""
+    container_ports: list[int]
+    """List of network ports of the Docker container spun up by the extension"""
+    host: str | None
+    """
+    Optional host on which to expose the container endpoints.
+    Can be either a static hostname, or a pattern like `<regex("(.+\.)?"):subdomain>myext.<domain>`
+    """
+    path: str | None
+    """Optional path on which to expose the container endpoints."""
+    command: list[str] | None
+    """Optional command (and flags) to execute in the container."""
+
+    request_to_port_router: Callable[[Request], int] | None
+    """Callable that returns the target port for a given request, for routing purposes"""
+    http2_ports: list[int] | None
+    """List of ports for which HTTP2 proxy forwarding into the container should be enabled."""
+
+    def __init__(
+        self,
+        image_name: str,
+        container_ports: list[int],
+        host: str | None = None,
+        path: str | None = None,
+        container_name: str | None = None,
+        command: list[str] | None = None,
+        request_to_port_router: Callable[[Request], int] | None = None,
+        http2_ports: list[int] | None = None,
+    ):
+        self.image_name = image_name
+        self.container_ports = container_ports
+        self.host = host
+        self.path = path
+        self.container_name = container_name
+        self.command = command
+        self.request_to_port_router = request_to_port_router
+        self.http2_ports = http2_ports
+
+    def update_gateway_routes(self, router: http.Router[http.RouteHandler]):
+        if self.path:
+            raise NotImplementedError(
+                "Path-based routing not yet implemented for this extension"
+            )
+        self.start_container()
+        # add resource for HTTP/1.1 requests
+        resource = RuleAdapter(ProxyResource(self))
+        if self.host:
+            resource = WithHost(self.host, [resource])
+        router.add(resource)
+
+        # apply patches to serve HTTP/2 requests
+        for port in self.http2_ports or []:
+            apply_http2_patches_for_grpc_support(
+                get_addressable_container_host(), port, self
+            )
+
+    def on_platform_shutdown(self):
+        self._remove_container()
+
+    def _get_container_name(self) -> str:
+        if self.container_name:
+            return self.container_name
+        name = f"ls-ext-{self.name}"
+        name = re.sub(r"\W", "-", name)
+        return name
+
+    @cache
+    def start_container(self) -> None:
+        container_name = self._get_container_name()
+        LOG.debug("Starting extension container %s", container_name)
+
+        ports = PortMappings()
+        for port in self.container_ports:
+            ports.add(port)
+
+        kwargs = {}
+        if self.command:
+            kwargs["command"] = self.command
+
+        try:
+            DOCKER_CLIENT.run_container(
+                self.image_name,
+                detach=True,
+                remove=True,
+                name=container_name,
+                ports=ports,
+                **kwargs,
+            )
+        except Exception as e:
+            LOG.debug("Failed to start container %s: %s", container_name, e)
+            raise
+
+        main_port = self.container_ports[0]
+        container_host = get_addressable_container_host()
+
+        def _ping_endpoint():
+            # TODO: allow defining a custom healthcheck endpoint ...
+            response = requests.get(f"http://{container_host}:{main_port}/")
+            assert response.ok
+
+        try:
+            retry(_ping_endpoint, retries=40, sleep=1)
+        except Exception as e:
+            LOG.info("Failed to connect to container %s: %s", container_name, e)
+            self._remove_container()
+            raise
+
+        LOG.debug("Successfully started extension container %s", container_name)
+
+    def _remove_container(self):
+        container_name = self._get_container_name()
+        LOG.debug("Stopping extension container %s", container_name)
+        DOCKER_CLIENT.remove_container(
+            container_name, force=True, check_existence=False
+        )
+
+
+class ProxyResource:
+    """
+    Simple proxy resource that forwards incoming requests from the
+    LocalStack Gateway to the target Docker container.
+    """
+
+    extension: ProxiedDockerContainerExtension
+
+    def __init__(self, extension: ProxiedDockerContainerExtension):
+        self.extension = extension
+
+    @route("/<path:path>")
+    def index(self, request: Request, path: str, *args, **kwargs):
+        return self._proxy_request(request, forward_path=f"/{path}")
+
+    def _proxy_request(self, request: Request, forward_path: str, *args, **kwargs):
+        self.extension.start_container()
+
+        port = self.extension.container_ports[0]
+        container_host = get_addressable_container_host()
+        base_url = f"http://{container_host}:{port}"
+        proxy = Proxy(forward_base_url=base_url)
+
+        # update content length (may have changed due to content compression)
+        if request.method not in ("GET", "OPTIONS"):
+            request.headers["Content-Length"] = str(len(request.data))
+
+        # make sure we're forwarding the correct Host header
+        request.headers["Host"] = f"localhost:{port}"
+
+        # forward the request to the target
+        result = proxy.forward(request, forward_path=forward_path)
+
+        return result

localstack-wiremock/pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+requires = ["setuptools", "wheel", "plux>=1.3.1"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "localstack-wiremock"
+version = "0.1.0"
+description = "LocalStack Extension: WireMock on LocalStack"
+readme = {file = "README.md", content-type = "text/markdown; charset=UTF-8"}
+requires-python = ">=3.9"
+authors = [
+    { name = "LocalStack + WireMock team"}
+]
+keywords = ["LocalStack", "WireMock"]
+classifiers = []
+dependencies = [
+    "httpx",
+    "h2",
+    "priority",
+]
+
+[project.urls]
+Homepage = "https://github.com/whummer/localstack-utils"
+
+[project.optional-dependencies]
+dev = [
+    "boto3",
+    "build",
+    "jsonpatch",
+    "localstack",
+    "pytest",
+    "rolo",
+    "ruff",
+    "twisted"
+]
+
+[project.entry-points."localstack.extensions"]
+localstack_wiremock = "localstack_wiremock.extension:WireMockExtension"