Skip to content

Commit 26abcbc

Browse files
aledbfaledbf
committed
Add containerd integration documentation and implement file injection support via annotations
1 parent 386f7c0 commit 26abcbc

4 files changed

Lines changed: 404 additions & 60 deletions

File tree

‎docs/containerd.md‎

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,299 @@
1+
# Containerd Integration
2+
3+
This document describes how to use the spinbox runtime with containerd to run containers inside QEMU/KVM virtual machines.
4+
5+
## Overview
6+
7+
Spinbox is a containerd shim runtime (`io.containerd.spinbox.v1`) that executes containers inside lightweight QEMU/KVM virtual machines. Each container runs in its own isolated VM, providing strong security boundaries.
8+
9+
## Quick Start
10+
11+
### Using ctr CLI
12+
13+
```bash
14+
# Basic container in a VM
15+
ctr run --rm --runtime io.containerd.spinbox.v1 \
16+
docker.io/library/alpine:latest \
17+
my-container \
18+
/bin/sh -c "echo hello from VM"
19+
20+
# With custom files injected into the VM
21+
ctr run --rm --runtime io.containerd.spinbox.v1 \
22+
--label 'io.spin.extras.files={"/usr/local/bin/myapp":{"source":"/host/path/myapp","mode":493}}' \
23+
docker.io/library/alpine:latest \
24+
my-container \
25+
/usr/local/bin/myapp
26+
```
27+
28+
### Using Go Client
29+
30+
```go
31+
package main
32+
33+
import (
34+
"context"
35+
36+
"github.com/containerd/containerd/v2/client"
37+
"github.com/containerd/containerd/v2/pkg/cio"
38+
"github.com/containerd/containerd/v2/pkg/namespaces"
39+
"github.com/containerd/containerd/v2/pkg/oci"
40+
)
41+
42+
func main() {
43+
ctx := namespaces.WithNamespace(context.Background(), "default")
44+
45+
c, err := client.New("/run/containerd/containerd.sock")
46+
if err != nil {
47+
panic(err)
48+
}
49+
defer c.Close()
50+
51+
image, err := c.GetImage(ctx, "docker.io/library/alpine:latest")
52+
if err != nil {
53+
panic(err)
54+
}
55+
56+
container, err := c.NewContainer(ctx, "my-vm-container",
57+
client.WithImage(image),
58+
client.WithNewSnapshot("my-vm-snapshot", image),
59+
client.WithNewSpec(
60+
oci.WithImageConfig(image),
61+
oci.WithProcessArgs("/bin/sh", "-c", "echo hello"),
62+
),
63+
// Use spinbox runtime
64+
client.WithRuntime("io.containerd.spinbox.v1", nil),
65+
)
66+
if err != nil {
67+
panic(err)
68+
}
69+
defer container.Delete(ctx, client.WithSnapshotCleanup)
70+
71+
task, err := container.NewTask(ctx, cio.NewCreator(cio.WithStdio))
72+
if err != nil {
73+
panic(err)
74+
}
75+
defer task.Delete(ctx)
76+
77+
if err := task.Start(ctx); err != nil {
78+
panic(err)
79+
}
80+
81+
exitCh, _ := task.Wait(ctx)
82+
status := <-exitCh
83+
fmt.Printf("Exited with code: %d\n", status.ExitCode())
84+
}
85+
```
86+
87+
## Injecting Files into the VM
88+
89+
The `io.spin.extras.files` annotation allows injecting arbitrary files from the host into the VM. Files are specified as a JSON object mapping destination paths (in the VM) to source configurations.
90+
91+
### Annotation Format
92+
93+
```json
94+
{
95+
"<destPath>": {
96+
"source": "<hostPath>",
97+
"mode": <permissions>
98+
}
99+
}
100+
```
101+
102+
| Field | Description |
103+
|-------|-------------|
104+
| `destPath` (key) | Absolute path where the file will be placed inside the VM |
105+
| `source` | Absolute path to the source file on the host |
106+
| `mode` | File permissions in decimal (493 = 0755, 420 = 0644) |
107+
108+
### Example: Single File
109+
110+
```bash
111+
ctr run --rm --runtime io.containerd.spinbox.v1 \
112+
--label 'io.spin.extras.files={"/usr/local/bin/myapp":{"source":"/opt/bin/myapp","mode":493}}' \
113+
docker.io/library/alpine:latest \
114+
my-container \
115+
/usr/local/bin/myapp
116+
```
117+
118+
### Example: Multiple Files
119+
120+
```bash
121+
ctr run --rm --runtime io.containerd.spinbox.v1 \
122+
--label 'io.spin.extras.files={
123+
"/usr/local/bin/app":{"source":"/host/bin/app","mode":493},
124+
"/etc/app/config.json":{"source":"/host/configs/app.json","mode":420},
125+
"/usr/local/bin/helper":{"source":"/host/bin/helper","mode":493}
126+
}' \
127+
docker.io/library/alpine:latest \
128+
my-container \
129+
/usr/local/bin/app
130+
```
131+
132+
### Go Client Example
133+
134+
```go
135+
import "encoding/json"
136+
137+
// Define files to inject
138+
extraFiles := map[string]struct {
139+
Source string `json:"source"`
140+
Mode int64 `json:"mode"`
141+
}{
142+
"/usr/local/bin/myapp": {Source: "/host/path/to/myapp", Mode: 0755},
143+
"/etc/myapp/config.json": {Source: "/host/configs/app.json", Mode: 0644},
144+
}
145+
extraFilesJSON, _ := json.Marshal(extraFiles)
146+
147+
container, err := c.NewContainer(ctx, "my-container",
148+
client.WithImage(image),
149+
client.WithNewSnapshot("snapshot", image),
150+
client.WithNewSpec(oci.WithImageConfig(image)),
151+
client.WithRuntime("io.containerd.spinbox.v1", nil),
152+
client.WithContainerLabels(map[string]string{
153+
"io.spin.extras.files": string(extraFilesJSON),
154+
}),
155+
)
156+
```
157+
158+
### How It Works
159+
160+
1. The shim parses the `io.spin.extras.files` annotation
161+
2. Creates a tar archive containing all specified files
162+
3. Caches the tar by content hash (for efficiency)
163+
4. Attaches the tar as a read-only virtio-blk device to the VM
164+
5. Guest extracts files to their destination paths on boot
165+
6. Extraction is idempotent (runs once per boot)
166+
167+
## Supervisor Agent
168+
169+
The supervisor agent provides additional capabilities inside the VM for integration with external control planes. To use the supervisor:
170+
171+
1. Inject the supervisor binary via `io.spin.extras.files`
172+
2. Enable supervisor and provide runtime configuration via annotations
173+
174+
### Required Setup
175+
176+
The supervisor binary must be placed at `/run/spin-stack/spin-supervisor` inside the VM:
177+
178+
```json
179+
{
180+
"/run/spin-stack/spin-supervisor": {
181+
"source": "/usr/share/spin-stack/bin/spin-supervisor",
182+
"mode": 493
183+
}
184+
}
185+
```
186+
187+
### Supervisor Annotations
188+
189+
| Annotation | Required | Description |
190+
|------------|----------|-------------|
191+
| `io.spin.supervisor.enabled` | Yes | Set to `"true"` to enable supervisor |
192+
| `io.spin.supervisor.workspace_id` | Yes | Workspace UUID |
193+
| `io.spin.supervisor.secret` | Yes | 64-character hex-encoded HMAC secret |
194+
| `io.spin.supervisor.control_plane` | Yes | Control plane URL |
195+
| `io.spin.supervisor.metadata_addr` | No | Metadata service address (default: `169.254.169.254:80`) |
196+
197+
### Complete Supervisor Example
198+
199+
```bash
200+
ctr run --rm --runtime io.containerd.spinbox.v1 \
201+
--label 'io.spin.extras.files={"/run/spin-stack/spin-supervisor":{"source":"/usr/share/spin-stack/bin/spin-supervisor","mode":493}}' \
202+
--label io.spin.supervisor.enabled=true \
203+
--label io.spin.supervisor.workspace_id=550e8400-e29b-41d4-a716-446655440000 \
204+
--label io.spin.supervisor.secret=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef \
205+
--label io.spin.supervisor.control_plane=https://control.example.com:443 \
206+
docker.io/library/alpine:latest \
207+
my-container \
208+
/bin/sh
209+
```
210+
211+
### Go Client with Supervisor
212+
213+
```go
214+
import "encoding/json"
215+
216+
// Files including supervisor binary
217+
extraFiles := map[string]struct {
218+
Source string `json:"source"`
219+
Mode int64 `json:"mode"`
220+
}{
221+
// Supervisor binary - required path
222+
"/run/spin-stack/spin-supervisor": {
223+
Source: "/usr/share/spin-stack/bin/spin-supervisor",
224+
Mode: 0755,
225+
},
226+
// Additional application files
227+
"/usr/local/bin/myapp": {
228+
Source: "/host/path/to/myapp",
229+
Mode: 0755,
230+
},
231+
}
232+
extraFilesJSON, _ := json.Marshal(extraFiles)
233+
234+
container, err := c.NewContainer(ctx, "my-container",
235+
client.WithImage(image),
236+
client.WithNewSnapshot("snapshot", image),
237+
client.WithNewSpec(
238+
oci.WithImageConfig(image),
239+
oci.WithProcessArgs("/usr/local/bin/myapp"),
240+
),
241+
client.WithRuntime("io.containerd.spinbox.v1", nil),
242+
client.WithContainerLabels(map[string]string{
243+
// Inject files
244+
"io.spin.extras.files": string(extraFilesJSON),
245+
// Supervisor configuration
246+
"io.spin.supervisor.enabled": "true",
247+
"io.spin.supervisor.workspace_id": "550e8400-e29b-41d4-a716-446655440000",
248+
"io.spin.supervisor.secret": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
249+
"io.spin.supervisor.control_plane": "https://control.example.com:443",
250+
}),
251+
)
252+
```
253+
254+
## Annotations Reference
255+
256+
### File Injection
257+
258+
| Annotation | Format | Description |
259+
|------------|--------|-------------|
260+
| `io.spin.extras.files` | JSON object | Files to inject into VM (see format above) |
261+
262+
### Supervisor Configuration
263+
264+
| Annotation | Format | Description |
265+
|------------|--------|-------------|
266+
| `io.spin.supervisor.enabled` | `"true"` | Enable supervisor agent |
267+
| `io.spin.supervisor.workspace_id` | UUID string | Workspace identifier |
268+
| `io.spin.supervisor.secret` | 64-char hex | HMAC secret for authentication |
269+
| `io.spin.supervisor.control_plane` | URL | Control plane endpoint |
270+
| `io.spin.supervisor.metadata_addr` | `host:port` | Metadata service (default: `169.254.169.254:80`) |
271+
272+
## File Permission Reference
273+
274+
Common permission values (decimal):
275+
276+
| Octal | Decimal | Description |
277+
|-------|---------|-------------|
278+
| 0755 | 493 | Executable (rwxr-xr-x) |
279+
| 0644 | 420 | Regular file (rw-r--r--) |
280+
| 0600 | 384 | Private file (rw-------) |
281+
| 0700 | 448 | Private executable (rwx------) |
282+
283+
## Troubleshooting
284+
285+
### Files Not Appearing in VM
286+
287+
1. Verify source paths exist on the host
288+
2. Check paths are absolute (both source and destination)
289+
3. Review shim logs: `journalctl -u containerd`
290+
291+
### Supervisor Not Starting
292+
293+
1. Ensure binary is at exact path: `/run/spin-stack/spin-supervisor`
294+
2. Verify all required annotations are set
295+
3. Check supervisor logs inside VM: `/var/log/spin-supervisor.log`
296+
297+
### Force Re-extraction
298+
299+
Add `spin.extras_force=1` to kernel cmdline to force file re-extraction on boot (useful for debugging).

‎internal/shim/extras/annotations.go‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
//go:build linux
2+
3+
package extras
4+
5+
import (
6+
"encoding/json"
7+
"fmt"
8+
"os"
9+
"path/filepath"
10+
11+
"github.com/opencontainers/runtime-spec/specs-go"
12+
)
13+
14+
// Annotation keys for extras configuration.
15+
const (
16+
// AnnotationFiles is a JSON object mapping destination paths to source configurations.
17+
// Format: {"<destPath>": {"source": "<sourcePath>", "mode": <mode>}, ...}
18+
// Example: {"/usr/local/bin/myapp": {"source": "/host/path/myapp", "mode": 493}}
19+
// Note: mode is decimal (493 = 0755 octal)
20+
AnnotationFiles = "io.spin.extras.files"
21+
)
22+
23+
// FileConfig represents a single file configuration from annotations.
24+
type FileConfig struct {
25+
Source string `json:"source"` // Source path on host
26+
Mode int64 `json:"mode"` // File permissions (decimal, e.g., 493 for 0755)
27+
}
28+
29+
// FromAnnotations parses extra files configuration from OCI spec annotations.
30+
// Returns nil if no extras are configured.
31+
func FromAnnotations(spec *specs.Spec) ([]File, error) {
32+
if spec == nil || spec.Annotations == nil {
33+
return nil, nil
34+
}
35+
36+
filesJSON, ok := spec.Annotations[AnnotationFiles]
37+
if !ok || filesJSON == "" {
38+
return nil, nil
39+
}
40+
41+
// Parse JSON: map of destPath -> FileConfig
42+
var filesMap map[string]FileConfig
43+
if err := json.Unmarshal([]byte(filesJSON), &filesMap); err != nil {
44+
return nil, fmt.Errorf("parse %s annotation: %w", AnnotationFiles, err)
45+
}
46+
47+
var files []File
48+
for destPath, cfg := range filesMap {
49+
// Validate destination path is absolute
50+
if !filepath.IsAbs(destPath) {
51+
return nil, fmt.Errorf("extras: destination path must be absolute: %s", destPath)
52+
}
53+
54+
// Validate source path exists
55+
cleanSource := filepath.Clean(cfg.Source)
56+
if !filepath.IsAbs(cleanSource) {
57+
return nil, fmt.Errorf("extras: source path must be absolute: %s", cfg.Source)
58+
}
59+
if _, err := os.Stat(cleanSource); err != nil {
60+
return nil, fmt.Errorf("extras: source file not found %s: %w", cleanSource, err)
61+
}
62+
63+
// Default mode to 0644 if not specified
64+
mode := cfg.Mode
65+
if mode == 0 {
66+
mode = 0644
67+
}
68+
69+
files = append(files, NewFile(destPath, cleanSource, mode))
70+
}
71+
72+
return files, nil
73+
}

0 commit comments

Comments
 (0)