From 6f6af91be86755479eed54c192f109c40ddc1620 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Tue, 8 Oct 2024 14:28:52 +0800 Subject: [PATCH 1/2] Support set python version when build python runner image --- .ci/tests/integration-oauth2/e2e.yaml | 2 +- .../e2e_with_downloader.yaml | 2 +- .ci/tests/integration/e2e.yaml | 2 +- .ci/tests/integration/e2e_with_tls.yaml | 2 +- images/build.sh | 3 +- .../pulsarctl.Dockerfile | 31 +++++++++++++++++-- 6 files changed, 35 insertions(+), 7 deletions(-) diff --git a/.ci/tests/integration-oauth2/e2e.yaml b/.ci/tests/integration-oauth2/e2e.yaml index a7e8660c1..4003041ed 100644 --- a/.ci/tests/integration-oauth2/e2e.yaml +++ b/.ci/tests/integration-oauth2/e2e.yaml @@ -6,7 +6,7 @@ setup: - name: build images command: | chmod +x images/build.sh images/samples/build.sh - PULSAR_IMAGE_TAG=3.2.2.1 PULSAR_IMAGE=streamnative/sn-platform KIND_PUSH=true images/build.sh + PULSAR_IMAGE_TAG=3.2.2.1 PULSAR_IMAGE=streamnative/sn-platform PYTHON_VERSION=3.10 KIND_PUSH=true images/build.sh PULSAR_IMAGE_TAG=3.2.2.1 KIND_PUSH=true images/samples/build.sh - name: install helm diff --git a/.ci/tests/integration-oauth2/e2e_with_downloader.yaml b/.ci/tests/integration-oauth2/e2e_with_downloader.yaml index 486885ed5..b6833c076 100644 --- a/.ci/tests/integration-oauth2/e2e_with_downloader.yaml +++ b/.ci/tests/integration-oauth2/e2e_with_downloader.yaml @@ -6,7 +6,7 @@ setup: - name: build images command: | chmod +x images/build.sh images/samples/build.sh - PULSAR_IMAGE_TAG=3.2.2.1 PULSAR_IMAGE=streamnative/sn-platform KIND_PUSH=true images/build.sh + PULSAR_IMAGE_TAG=3.2.2.1 PULSAR_IMAGE=streamnative/sn-platform PYTHON_VERSION=3.9 KIND_PUSH=true images/build.sh PULSAR_IMAGE_TAG=3.2.2.1 KIND_PUSH=true images/samples/build.sh - name: install helm diff --git a/.ci/tests/integration/e2e.yaml b/.ci/tests/integration/e2e.yaml index ed4020639..0cde7f739 100644 --- a/.ci/tests/integration/e2e.yaml +++ b/.ci/tests/integration/e2e.yaml @@ -6,7 +6,7 @@ setup: - name: build images command: | chmod +x images/build.sh images/samples/build.sh - PULSAR_IMAGE_TAG=3.2.2.1 PULSAR_IMAGE=streamnative/sn-platform KIND_PUSH=true images/build.sh + PULSAR_IMAGE_TAG=3.2.2.1 PULSAR_IMAGE=streamnative/sn-platform PYTHON_VERSION=3.12 KIND_PUSH=true images/build.sh PULSAR_IMAGE_TAG=3.2.2.1 KIND_PUSH=true images/samples/build.sh - name: install helm diff --git a/.ci/tests/integration/e2e_with_tls.yaml b/.ci/tests/integration/e2e_with_tls.yaml index e1844caf3..2182ba2d7 100644 --- a/.ci/tests/integration/e2e_with_tls.yaml +++ b/.ci/tests/integration/e2e_with_tls.yaml @@ -6,7 +6,7 @@ setup: - name: build images command: | chmod +x images/build.sh images/samples/build.sh - PULSAR_IMAGE_TAG=3.2.2.1 PULSAR_IMAGE=streamnative/sn-platform KIND_PUSH=true images/build.sh + PULSAR_IMAGE_TAG=3.2.2.1 PULSAR_IMAGE=streamnative/sn-platform PYTHON_VERSION=3.11 KIND_PUSH=true images/build.sh PULSAR_IMAGE_TAG=3.2.2.1 KIND_PUSH=true images/samples/build.sh - name: install helm diff --git a/images/build.sh b/images/build.sh index d292f24d4..1a5dc6502 100755 --- a/images/build.sh +++ b/images/build.sh @@ -21,6 +21,7 @@ set -e PULSAR_IMAGE=${PULSAR_IMAGE:-"streamnative/sn-platform"} PULSAR_IMAGE_TAG=${PULSAR_IMAGE_TAG:-"2.7.1"} +PYTHON_VERSION=${PYTHON_VERSION:-"3.12"} DOCKER_REPO=${DOCKER_REPO:-"streamnative"} RUNNER_BASE="pulsar-functions-runner-base" PULSARCTL_RUNNER_BASE="pulsar-functions-pulsarctl-runner-base" @@ -48,7 +49,7 @@ docker tag ${PULSARCTL_JAVA_RUNNER} "${DOCKER_REPO}"/${PULSARCTL_JAVA_RUNNER}:"$ echo "build python runner" docker build --platform linux/amd64 -t ${PYTHON_RUNNER} images/pulsar-functions-python-runner --build-arg PULSAR_IMAGE="$PULSAR_IMAGE" --build-arg PULSAR_IMAGE_TAG="$PULSAR_IMAGE_TAG" --progress=plain -docker build --platform linux/amd64 -t ${PULSARCTL_PYTHON_RUNNER} images/pulsar-functions-python-runner -f images/pulsar-functions-python-runner/pulsarctl.Dockerfile --build-arg PULSAR_IMAGE="$PULSAR_IMAGE" --build-arg PULSAR_IMAGE_TAG="$PULSAR_IMAGE_TAG" --progress=plain +docker build --platform linux/amd64 -t ${PULSARCTL_PYTHON_RUNNER} images/pulsar-functions-python-runner -f images/pulsar-functions-python-runner/pulsarctl.Dockerfile --build-arg PULSAR_IMAGE="$PULSAR_IMAGE" --build-arg PULSAR_IMAGE_TAG="$PULSAR_IMAGE_TAG" --build-arg PYTHON_VERSION="$PYTHON_VERSION" --progress=plain docker tag ${PYTHON_RUNNER} "${DOCKER_REPO}"/${PYTHON_RUNNER}:"${RUNNER_TAG}" docker tag ${PULSARCTL_PYTHON_RUNNER} "${DOCKER_REPO}"/${PULSARCTL_PYTHON_RUNNER}:"${RUNNER_TAG}" diff --git a/images/pulsar-functions-python-runner/pulsarctl.Dockerfile b/images/pulsar-functions-python-runner/pulsarctl.Dockerfile index d42f542a0..182b2a299 100644 --- a/images/pulsar-functions-python-runner/pulsarctl.Dockerfile +++ b/images/pulsar-functions-python-runner/pulsarctl.Dockerfile @@ -1,7 +1,34 @@ ARG PULSAR_IMAGE ARG PULSAR_IMAGE_TAG +ARG PYTHON_VERSION=3.12 FROM ${PULSAR_IMAGE}:${PULSAR_IMAGE_TAG} as pulsar -FROM pulsar-functions-pulsarctl-runner-base:latest + +FROM python:${PYTHON_VERSION}-alpine + +ENV GID=10001 +ENV UID=10000 +ENV USER=pulsar +RUN addgroup -g $GID pulsar +RUN adduser -u $UID -G pulsar -D -g '' $USER + +RUN mkdir -p /pulsar/bin/ \ + && mkdir -p /pulsar/lib/ \ + && mkdir -p /pulsar/conf/ \ + && mkdir -p /pulsar/instances/ \ + && mkdir -p /pulsar/connectors/ \ + && mkdir -p /pulsar/logs/ \ + && mkdir -p /pulsar/tmp/ \ + && mkdir -p /pulsar/examples/ \ + && chown -R $UID:$GID /pulsar \ + && chmod -R g=u /pulsar \ + && apk update && apk add --no-cache wget bash \ + && wget https://github.com/streamnative/pulsarctl/releases/latest/download/pulsarctl-amd64-linux.tar.gz -P /pulsar/bin/ \ + && tar -xzf /pulsar/bin/pulsarctl-amd64-linux.tar.gz -C /pulsar/bin/ \ + && rm -rf /pulsar/bin/pulsarctl-amd64-linux.tar.gz \ + && chmod +x /pulsar/bin/pulsarctl-amd64-linux/pulsarctl \ + && ln -s /pulsar/bin/pulsarctl-amd64-linux/pulsarctl /usr/local/bin/pulsarctl + +WORKDIR /pulsar COPY --from=pulsar --chown=$UID:$GID /pulsar/bin /pulsar/bin COPY --from=pulsar --chown=$UID:$GID /pulsar/instances/python-instance /pulsar/instances/python-instance @@ -20,7 +47,7 @@ ENV PULSAR_CLIENT_PYTHON_VERSION=3.5.0 COPY --from=pulsar --chown=$UID:$GID /pulsar/README /pulsar/cpp-clien* /pulsar/cpp-client/ RUN apk update \ - && apk add --no-cache python3 python3-dev tk-dev curl ca-certificates\ + && apk add --no-cache tk-dev curl ca-certificates\ && mkdir -p /etc/pki/tls/certs && cp /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt \ && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ && mv /usr/lib/python3.*/EXTERNALLY-MANAGED /tmp/EXTERNALLY-MANAGED.old \ From 95009a74bc2b630de8b47f29a680afe227c8baec Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Fri, 22 Nov 2024 09:07:43 +0800 Subject: [PATCH 2/2] Update ci --- .ci/tests/integration/e2e.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/tests/integration/e2e.yaml b/.ci/tests/integration/e2e.yaml index 0cde7f739..ed4020639 100644 --- a/.ci/tests/integration/e2e.yaml +++ b/.ci/tests/integration/e2e.yaml @@ -6,7 +6,7 @@ setup: - name: build images command: | chmod +x images/build.sh images/samples/build.sh - PULSAR_IMAGE_TAG=3.2.2.1 PULSAR_IMAGE=streamnative/sn-platform PYTHON_VERSION=3.12 KIND_PUSH=true images/build.sh + PULSAR_IMAGE_TAG=3.2.2.1 PULSAR_IMAGE=streamnative/sn-platform KIND_PUSH=true images/build.sh PULSAR_IMAGE_TAG=3.2.2.1 KIND_PUSH=true images/samples/build.sh - name: install helm