From 847920b770dc47578fb24295fd901530dbf3332d Mon Sep 17 00:00:00 2001 From: Joris Decombe Date: Thu, 21 May 2026 17:34:33 +1200 Subject: [PATCH] Add Kubernetes exec support to devcontainer exec Enable `devcontainer exec` to run commands inside Kubernetes pods via `kubectl exec`, as an alternative to the existing Docker exec path. New CLI flags for the `exec` command: --kubectl-path kubectl CLI path (default: kubectl) --k8s-context Kubernetes context (from kubeconfig) --k8s-kubeconfig Path to kubeconfig file (for custom CAs) --k8s-namespace Target pod namespace (required with --k8s-pod) --k8s-pod Target pod name --k8s-container Target container name (required with --k8s-pod) Since kubectl exec doesn't support Docker's -u (user), -e (env), or -w (cwd) flags natively, these are handled by wrapping commands in shell invocations with proper POSIX quoting. A fast path avoids shell wrapping when none of these are needed (e.g., for the shell server). User switching for non-root remoteUser uses `su -s /bin/sh -c` (no login shell, matching Docker's -u behaviour). The user parameter is validated against a strict regex to prevent shell injection. Environment probing uses probeRemoteEnv to capture the actual runtime environment inside the container, correctly resolving K8s valueFrom env vars (ConfigMaps, Secrets, Downward API) that aren't visible in the static pod spec. Relates to: devcontainers/spec#672, microsoft/vscode-remote-release#6413 --- src/spec-node/devContainersSpecCLI.ts | 68 ++++++++- src/spec-node/featuresCLI/utils.ts | 6 + src/spec-node/kubernetesContainer.ts | 58 ++++++++ src/spec-shutdown/kubeUtils.ts | 202 ++++++++++++++++++++++++++ src/test/cli.kubernetes.test.ts | 90 ++++++++++++ 5 files changed, 423 insertions(+), 1 deletion(-) create mode 100644 src/spec-node/kubernetesContainer.ts create mode 100644 src/spec-shutdown/kubeUtils.ts create mode 100644 src/test/cli.kubernetes.test.ts diff --git a/src/spec-node/devContainersSpecCLI.ts b/src/spec-node/devContainersSpecCLI.ts index 832e9603f..f3dfcd480 100644 --- a/src/spec-node/devContainersSpecCLI.ts +++ b/src/spec-node/devContainersSpecCLI.ts @@ -17,6 +17,8 @@ import { Log, LogDimensions, LogLevel, makeLog, mapLogLevel } from '../spec-util import { probeRemoteEnv, runLifecycleHooks, runRemoteCommand, UserEnvProbe, setupInContainer } from '../spec-common/injectHeadless'; import { extendImage } from './containerFeatures'; import { dockerCLI, DockerCLIParameters, dockerPtyCLI, inspectContainer } from '../spec-shutdown/dockerUtils'; +import { KubeCLIParameters } from '../spec-shutdown/kubeUtils'; +import { createK8sContainerProperties } from './kubernetesContainer'; import { buildAndExtendDockerCompose, dockerComposeCLIConfig, getDefaultImageName, getProjectName, readDockerComposeConfig, readVersionPrefix } from './dockerCompose'; import { DevContainerFromDockerComposeConfig, DevContainerFromDockerfileConfig, getDockerComposeFilePaths } from '../spec-configuration/configuration'; import { workspaceFromPath } from '../spec-utils/workspaces'; @@ -1268,6 +1270,12 @@ function execOptions(y: Argv) { 'default-user-env-probe': { choices: ['none' as 'none', 'loginInteractiveShell' as 'loginInteractiveShell', 'interactiveShell' as 'interactiveShell', 'loginShell' as 'loginShell'], default: defaultDefaultUserEnvProbe, description: 'Default value for the devcontainer.json\'s "userEnvProbe".' }, 'remote-env': { type: 'string', description: 'Remote environment variables of the format name=value. These will be added when executing the user commands.' }, 'skip-feature-auto-mapping': { type: 'boolean', default: false, hidden: true, description: 'Temporary option for testing.' }, + 'kubectl-path': { type: 'string', default: 'kubectl', description: 'kubectl CLI path.' }, + 'k8s-context': { type: 'string', description: 'Kubernetes context to use (from kubeconfig).' }, + 'k8s-kubeconfig': { type: 'string', description: 'Path to kubeconfig file (for custom CA certificates or non-default configs).' }, + 'k8s-namespace': { type: 'string', description: 'Kubernetes namespace of the target pod.' }, + 'k8s-pod': { type: 'string', description: 'Kubernetes pod name to exec into.' }, + 'k8s-container': { type: 'string', description: 'Kubernetes container name within the pod.' }, }) .positional('cmd', { type: 'string', @@ -1288,7 +1296,15 @@ function execOptions(y: Argv) { if (remoteEnvs?.some(remoteEnv => !/.+=.*/.test(remoteEnv))) { throw new Error('Unmatched argument format: remote-env must match ='); } - if (!argv['container-id'] && !idLabels?.length && !argv['workspace-folder']) { + const isK8s = !!(argv['k8s-pod']); + if (isK8s) { + if (!argv['k8s-namespace']) { + throw new Error('--k8s-namespace is required when using --k8s-pod'); + } + if (!argv['k8s-container']) { + throw new Error('--k8s-container is required when using --k8s-pod'); + } + } else if (!argv['container-id'] && !idLabels?.length && !argv['workspace-folder']) { argv['workspace-folder'] = process.cwd(); } return true; @@ -1330,6 +1346,12 @@ export async function doExec({ 'default-user-env-probe': defaultUserEnvProbe, 'remote-env': addRemoteEnv, 'skip-feature-auto-mapping': skipFeatureAutoMapping, + 'kubectl-path': kubectlPath, + 'k8s-context': k8sContext, + 'k8s-kubeconfig': k8sKubeconfig, + 'k8s-namespace': k8sNamespace, + 'k8s-pod': k8sPod, + 'k8s-container': k8sContainer, _: restArgs, }: ExecArgs & { _?: string[] }) { const disposables: (() => Promise | undefined)[] = []; @@ -1387,6 +1409,50 @@ export async function doExec({ const { common } = params; const { cliHost } = common; output = common.output; + + // Kubernetes exec path — bypass Docker container discovery entirely. + if (k8sPod && k8sNamespace && k8sContainer) { + const kubeParams: KubeCLIParameters = { + cliHost, + kubectlCLI: kubectlPath || 'kubectl', + context: k8sContext, + kubeconfig: k8sKubeconfig, + namespace: k8sNamespace, + pod: k8sPod, + container: k8sContainer, + env: cliHost.env, + output, + }; + + // Optionally load devcontainer.json for remoteUser/remoteEnv/workspaceFolder. + const workspace = workspaceFolder ? workspaceFromPath(cliHost.path, workspaceFolder) : undefined; + const configPath = configFile ? configFile : workspace + ? (await getDevContainerConfigPathIn(cliHost, workspace.configFolderPath) + || (overrideConfigFile ? getDefaultDevContainerConfigPath(cliHost, workspace.configFolderPath) : undefined)) + : overrideConfigFile; + const configs = configPath && await readDevContainerConfigFile(cliHost, workspace, configPath, params.mountWorkspaceGitRoot, params.mountGitWorktreeCommonDir, output, undefined, overrideConfigFile) || undefined; + + const remoteUser = configs?.config.config.remoteUser; + const remoteWorkspaceFolder = configs?.workspaceConfig.workspaceFolder || configs?.config.config.workspaceFolder; + + const containerProperties = await createK8sContainerProperties(common, kubeParams, remoteWorkspaceFolder, remoteUser); + + // Probe remote environment (shell init scripts, userEnvProbe setting) + // and merge with devcontainer.json remoteEnv + CLI --remote-env. + const k8sConfig = { + ...(configs?.config.config || {}), + remoteEnv: { ...(configs?.config.config.remoteEnv || {}), ...envListToObj(addRemoteEnvs) }, + }; + const remoteEnv = probeRemoteEnv(common, containerProperties, k8sConfig); + const remoteCwd = containerProperties.remoteWorkspaceFolder || containerProperties.homeFolder; + await runRemoteCommand({ ...common, output, stdin: process.stdin, ...(logFormat !== 'json' ? { stdout: process.stdout, stderr: process.stderr } : {}) }, containerProperties, restArgs || [], remoteCwd, { remoteEnv: await remoteEnv, pty: isTTY, print: 'continuous' }); + return { + code: 0, + dispose, + }; + } + + // Docker exec path. const workspace = workspaceFolder ? workspaceFromPath(cliHost.path, workspaceFolder) : undefined; const configPath = configFile ? configFile : workspace ? (await getDevContainerConfigPathIn(cliHost, workspace.configFolderPath) diff --git a/src/spec-node/featuresCLI/utils.ts b/src/spec-node/featuresCLI/utils.ts index c981a4c2e..b774a077f 100644 --- a/src/spec-node/featuresCLI/utils.ts +++ b/src/spec-node/featuresCLI/utils.ts @@ -44,6 +44,12 @@ export const staticExecParams = { 'log-level': 'info' as 'info', 'log-format': 'text' as 'text', 'default-user-env-probe': 'loginInteractiveShell' as 'loginInteractiveShell', + 'kubectl-path': 'kubectl', + 'k8s-context': undefined, + 'k8s-kubeconfig': undefined, + 'k8s-namespace': undefined, + 'k8s-pod': undefined, + 'k8s-container': undefined, }; export interface LaunchResult { diff --git a/src/spec-node/kubernetesContainer.ts b/src/spec-node/kubernetesContainer.ts new file mode 100644 index 000000000..a31e4dcfc --- /dev/null +++ b/src/spec-node/kubernetesContainer.ts @@ -0,0 +1,58 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ResolverParameters, getContainerProperties, ContainerProperties } from '../spec-common/injectHeadless'; +import { KubeCLIParameters, inspectPod, kubectlExecFunction, kubectlPtyExecFunction } from '../spec-shutdown/kubeUtils'; + +export function parseContainerUser(containerUser: string): { user: string | undefined; group: string | undefined } { + const [, user, , group] = /([^:]*)(:(.*))?/.exec(containerUser) as (string | undefined)[]; + return { user: (user === '0' ? 'root' : user) || undefined, group }; +} + +export async function createK8sContainerProperties( + params: ResolverParameters, + kubeParams: KubeCLIParameters, + remoteWorkspaceFolder: string | undefined, + remoteUser: string | undefined, +): Promise { + const inspecting = 'Inspecting pod'; + const start = params.output.start(inspecting); + const podInfo = await inspectPod(kubeParams); + params.output.stop(inspecting, start); + + const containerUser = remoteUser || podInfo.containerUser || 'root'; + const { user, group } = parseContainerUser(containerUser); + + // Use parsed user (not raw containerUser) because su only accepts + // usernames, not the user:group format that Docker's -u flag supports. + const remoteExec = kubectlExecFunction(kubeParams, user); + const remotePtyExec = await kubectlPtyExecFunction(kubeParams, user, params.loadNativeModule, params.allowInheritTTY); + + // Only provide remoteExecAsRoot if the container already runs as root. + // In K8s, switching to root via su/runuser fails when runAsNonRoot is set + // or the container lacks privilege escalation tools. + const remoteExecAsRoot = user === 'root' + ? remoteExec + : undefined; + + return getContainerProperties({ + params, + createdAt: podInfo.createdAt, + startedAt: podInfo.startedAt, + remoteWorkspaceFolder, + containerUser: user, + containerGroup: group, + // We pass an empty env here rather than undefined. The shell server + // launched by getContainerProperties will probe the actual runtime + // environment (resolving valueFrom refs) when probeRemoteEnv runs. + // Passing undefined would also probe env but can cause hangs when + // the shell server's PATH probe interacts with kubectl exec wrapping. + containerEnv: {}, + remoteExec, + remotePtyExec, + remoteExecAsRoot, + rootShellServer: undefined, + }); +} diff --git a/src/spec-shutdown/kubeUtils.ts b/src/spec-shutdown/kubeUtils.ts new file mode 100644 index 000000000..63f47e3b7 --- /dev/null +++ b/src/spec-shutdown/kubeUtils.ts @@ -0,0 +1,202 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CLIHost, runCommandNoPty, ExecFunction, ExecParameters, Exec, PtyExecFunction, PtyExec, PtyExecParameters, plainExecAsPtyExec } from '../spec-common/commonUtils'; +import * as ptyType from 'node-pty'; +import { Log, LogEvent, makeLog } from '../spec-utils/log'; +import { escapeRegExCharacters } from '../spec-utils/strings'; + +export interface KubeCLIParameters { + cliHost: CLIHost; + kubectlCLI: string; + context: string | undefined; + kubeconfig: string | undefined; + namespace: string; + pod: string; + container: string; + env: NodeJS.ProcessEnv; + output: Log; +} + +export interface PodDetails { + name: string; + namespace: string; + createdAt: string; + startedAt: string; + containerUser: string; +} + +export async function inspectPod(params: KubeCLIParameters): Promise { + const result = await kubectlCLI(params, 'get', 'pod', params.pod, + '-n', params.namespace, + '-o', 'json', + ); + const pod = JSON.parse(result.stdout.toString()); + const containerSpec = pod.spec?.containers?.find((c: { name: string }) => c.name === params.container) + || pod.spec?.containers?.[0]; + const containerStatus = pod.status?.containerStatuses?.find((c: { name: string }) => c.name === params.container) + || pod.status?.containerStatuses?.[0]; + + const securityContext = containerSpec?.securityContext || pod.spec?.securityContext || {}; + const runAsUser = securityContext.runAsUser; + const containerUser = runAsUser ? String(runAsUser) : 'root'; + + // Pod spec env only contains static values — valueFrom refs (ConfigMaps, + // Secrets, Downward API) are resolved by the kubelet at runtime and aren't + // visible here. We deliberately omit containerEnv so getContainerProperties + // probes the actual runtime environment via the shell server. + + return { + name: pod.metadata.name, + namespace: pod.metadata.namespace, + createdAt: pod.metadata.creationTimestamp || '', + startedAt: containerStatus?.state?.running?.startedAt || pod.metadata.creationTimestamp || '', + containerUser, + }; +} + +/** + * kubectl exec doesn't support -u (user), -e (env), or -w (cwd) flags + * like `docker exec` does. When env/cwd/user switching is needed, we wrap + * the target command in a shell invocation. When none of these are needed, + * we pass the command through directly to avoid unnecessary shell layers + * (important for interactive shells used by the shell server). + * + * For non-root users, we use `su -s /bin/sh -c` (no login shell, + * matching Docker's `-u` behaviour). + */ +function buildWrappedCommand(user: string | undefined, params: ExecParameters | PtyExecParameters): { cmd: string; args: string[] } { + const { env, cwd, cmd, args } = params; + + const hasEnv = env && Object.keys(env).length > 0; + const hasCwd = !!cwd; + const needsUserSwitch = !!(user && user !== 'root'); + + // Fast path: no wrapping needed when there's nothing to set up. + if (!hasEnv && !hasCwd && !needsUserSwitch) { + return { cmd, args: args || [] }; + } + + const parts: string[] = []; + + if (hasEnv) { + for (const key of Object.keys(env!)) { + if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) { + parts.push(`export ${key}=${shellQuote(env![key] ?? '')};`); + } + } + } + + if (hasCwd) { + parts.push(`cd ${shellQuote(cwd!)};`); + } + + parts.push(`exec ${shellQuote(cmd)}`); + if (args) { + parts.push(...args.map(shellQuote)); + } + + const script = parts.join(' '); + + if (needsUserSwitch) { + if (!/^[a-zA-Z0-9_][\w.-]*$/.test(user!)) { + throw new Error(`Invalid container user: ${user}`); + } + return { cmd: 'su', args: ['-s', '/bin/sh', user!, '-c', script] }; + } + + return { cmd: '/bin/sh', args: ['-c', script] }; +} + +function shellQuote(s: string): string { + const sanitised = s.replace(/\0/g, ''); + if (/^[a-zA-Z0-9_./:=-]+$/.test(sanitised)) { + return sanitised; + } + return `'${sanitised.replace(/'/g, `'\\''`)}'`; +} + +function toKubectlExecArgs(params: KubeCLIParameters, user: string | undefined, execParams: ExecParameters | PtyExecParameters, pty: boolean): { argsPrefix: string[]; args: string[] } { + const kubectlArgs = [...globalKubeArgs(params), 'exec', '-i']; + if (pty) { + kubectlArgs.push('-t'); + } + kubectlArgs.push(params.pod, '-n', params.namespace, '-c', params.container, '--'); + + const argsPrefix = kubectlArgs.slice(); + + const wrapped = buildWrappedCommand(user, execParams); + kubectlArgs.push(wrapped.cmd, ...wrapped.args); + + return { argsPrefix, args: kubectlArgs }; +} + +export function kubectlExecFunction(params: KubeCLIParameters, user: string | undefined, allocatePtyIfPossible = false): ExecFunction { + return async function (execParams: ExecParameters): Promise { + const canAllocatePty = allocatePtyIfPossible && process.stdin.isTTY && execParams.stdio?.[0] === 'inherit'; + const { argsPrefix, args: execArgs } = toKubectlExecArgs(params, user, execParams, canAllocatePty); + return params.cliHost.exec({ + cmd: params.kubectlCLI, + args: execArgs, + env: params.env, + stdio: execParams.stdio, + output: replacingKubectlExecLog(execParams.output, params.kubectlCLI, argsPrefix), + }); + }; +} + +export async function kubectlPtyExecFunction(params: KubeCLIParameters, user: string | undefined, loadNativeModule: (moduleName: string) => Promise, allowInheritTTY: boolean): Promise { + const pty = await loadNativeModule('node-pty'); + if (!pty) { + const plain = kubectlExecFunction(params, user, true); + return plainExecAsPtyExec(plain, allowInheritTTY); + } + + return async function (execParams: PtyExecParameters): Promise { + const { argsPrefix, args: execArgs } = toKubectlExecArgs(params, user, execParams, true); + return params.cliHost.ptyExec({ + cmd: params.kubectlCLI, + args: execArgs, + env: params.env, + output: replacingKubectlExecLog(execParams.output, params.kubectlCLI, argsPrefix), + }); + }; +} + +function replacingKubectlExecLog(original: Log, cmd: string, args: string[]) { + const search = `Run: ${cmd} ${(args || []).join(' ').replace(/\n.*/g, '')}`; + const searchR = new RegExp(escapeRegExCharacters(search), 'g'); + return makeLog({ + ...original, + get dimensions() { + return original.dimensions; + }, + event: (e: LogEvent) => original.event('text' in e ? { + ...e, + text: e.text.replace(searchR, 'Run in container:'), + } : e), + }); +} + +function globalKubeArgs(params: KubeCLIParameters): string[] { + const args: string[] = []; + if (params.kubeconfig) { + args.push('--kubeconfig', params.kubeconfig); + } + if (params.context) { + args.push('--context', params.context); + } + return args; +} + +async function kubectlCLI(params: KubeCLIParameters, ...args: string[]) { + return runCommandNoPty({ + exec: params.cliHost.exec, + cmd: params.kubectlCLI, + args: [...globalKubeArgs(params), ...args], + env: params.env, + output: params.output, + }); +} diff --git a/src/test/cli.kubernetes.test.ts b/src/test/cli.kubernetes.test.ts new file mode 100644 index 000000000..ef12baf86 --- /dev/null +++ b/src/test/cli.kubernetes.test.ts @@ -0,0 +1,90 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import * as path from 'path'; +import { shellExec } from './testUtils'; + +const pkg = require('../../package.json'); + +const tmp = path.relative(process.cwd(), path.join(__dirname, 'tmp')); +const cli = `npx --prefix ${tmp} devcontainer`; + +async function installCLI() { + await shellExec(`rm -rf ${tmp}/node_modules`); + await shellExec(`mkdir -p ${tmp}`); + await shellExec(`npm --prefix ${tmp} install devcontainers-cli-${pkg.version}.tgz`); +} + +// Validation tests — no K8s cluster required. +describe('Dev Containers CLI - Kubernetes flag validation', function () { + this.timeout('120s'); + + before('Install', installCLI); + + it('should reject missing --k8s-namespace', async function () { + try { + await shellExec(`${cli} exec --k8s-pod some-pod --k8s-container some-container -- echo test`); + assert.fail('Should have thrown'); + } catch (err: any) { + assert.ok(err.stderr.includes('--k8s-namespace is required'), `Expected namespace error, got: ${err.stderr}`); + } + }); + + it('should reject missing --k8s-container', async function () { + try { + await shellExec(`${cli} exec --k8s-pod some-pod --k8s-namespace some-ns -- echo test`); + assert.fail('Should have thrown'); + } catch (err: any) { + assert.ok(err.stderr.includes('--k8s-container is required'), `Expected container error, got: ${err.stderr}`); + } + }); +}); + +// Integration tests — require a running K8s cluster. +// Set DEVCONTAINER_TEST_K8S_NAMESPACE, DEVCONTAINER_TEST_K8S_POD, and +// DEVCONTAINER_TEST_K8S_CONTAINER environment variables to run these. +describe('Dev Containers CLI - Kubernetes exec', function () { + this.timeout('120s'); + + const k8sNamespace = process.env.DEVCONTAINER_TEST_K8S_NAMESPACE; + const k8sPod = process.env.DEVCONTAINER_TEST_K8S_POD; + const k8sContainer = process.env.DEVCONTAINER_TEST_K8S_CONTAINER; + + before('Install and check K8s prerequisites', async function () { + if (!k8sNamespace || !k8sPod || !k8sContainer) { + this.skip(); + return; + } + await installCLI(); + }); + + it('should exec a simple command in a K8s pod', async function () { + if (!k8sNamespace || !k8sPod || !k8sContainer) { + this.skip(); + return; + } + const res = await shellExec(`${cli} exec --k8s-namespace ${k8sNamespace} --k8s-pod ${k8sPod} --k8s-container ${k8sContainer} -- echo hello`); + assert.ok(res.stdout.includes('hello'), `Expected "hello" in stdout, got: ${res.stdout}`); + }); + + it('should exec whoami in a K8s pod', async function () { + if (!k8sNamespace || !k8sPod || !k8sContainer) { + this.skip(); + return; + } + const res = await shellExec(`${cli} exec --k8s-namespace ${k8sNamespace} --k8s-pod ${k8sPod} --k8s-container ${k8sContainer} -- whoami`); + assert.ok(res.stdout.trim().length > 0, 'Expected non-empty whoami output'); + }); + + it('should pass remote-env to K8s exec', async function () { + if (!k8sNamespace || !k8sPod || !k8sContainer) { + this.skip(); + return; + } + const res = await shellExec(`${cli} exec --k8s-namespace ${k8sNamespace} --k8s-pod ${k8sPod} --k8s-container ${k8sContainer} --remote-env TEST_VAR=hello123 -- sh -c 'echo $TEST_VAR'`); + assert.ok(res.stdout.includes('hello123'), `Expected "hello123" in stdout, got: ${res.stdout}`); + }); +});