Skip to content

FreeBSD guest: port init.c, make libkrun adjustments for processing kernel cmdline #480

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nohajc wants to merge 2 commits into
base: main
Choose a base branch
from
Open

FreeBSD guest: port init.c, make libkrun adjustments for processing kernel cmdline #480

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c401244
init.c: port to FreeBSD
nohajc Dec 3, 2025
a4232cb
krun_set_exec now also preserves raw argv (passed as base64-encoded v…
nohajc Dec 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,4 @@ examples/consoles
examples/rootfs_fedora
test-prefix
/linux-sysroot
/freebsd-sysroot
1 change: 1 addition & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

53 changes: 50 additions & 3 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,10 +81,14 @@ ifeq ($(TIMESYNC),1)
INIT_DEFS += -D__TIMESYNC__
endif

CLANG = /usr/bin/clang

OS = $(shell uname -s)
ARCH = $(shell uname -m)
DEBIAN_DIST ?= bookworm
ROOTFS_DIR = linux-sysroot
FREEBSD_VERSION ?= 14.3-RELEASE
FREEBSD_ROOTFS_DIR = freebsd-sysroot

KRUN_BINARY_Linux = libkrun$(VARIANT).so.$(FULL_VERSION)
KRUN_SONAME_Linux = libkrun$(VARIANT).so.$(ABI_VERSION)
Expand Down Expand Up @@ -121,7 +125,7 @@ else
SYSROOT_TARGET =
endif
# Cross-compile on macOS with the LLVM linker (brew install lld)
CC_LINUX=/usr/bin/clang -target $(ARCH)-linux-gnu -fuse-ld=lld -Wl,-strip-debug --sysroot $(SYSROOT_LINUX) -Wno-c23-extensions
CC_LINUX=$(CLANG) -target $(ARCH)-linux-gnu -fuse-ld=lld -Wl,-strip-debug --sysroot $(SYSROOT_LINUX) -Wno-c23-extensions
else
# Build on Linux host
CC_LINUX=$(CC)
Expand All @@ -134,6 +138,28 @@ $(INIT_BINARY): $(INIT_SRC) $(SYSROOT_TARGET)
$(CC_LINUX) -O2 -static -Wall $(INIT_DEFS) -o $@ $(INIT_SRC) $(INIT_DEFS)
endif

ifeq ($(OS),Darwin)
# If SYSROOT_BSD is not set and we're on macOS, generate sysroot automatically
ifeq ($(SYSROOT_BSD),)
SYSROOT_BSD = $(FREEBSD_ROOTFS_DIR)
SYSROOT_BSD_TARGET = $(FREEBSD_ROOTFS_DIR)/.sysroot_ready
else
SYSROOT_BSD_TARGET =
endif
# Cross-compile on macOS with the LLVM linker (brew install lld)
CC_BSD=$(CLANG) -target $(ARCH)-unknown-freebsd -fuse-ld=lld -stdlib=libc++ -Wl,-strip-debug --sysroot $(SYSROOT_BSD)
else
# Build on FreeBSD host
CC_BSD=$(CC)
SYSROOT_BSD_TARGET =
endif

ifeq ($(BUILD_BSD_INIT),1)
INIT_BINARY_BSD = init/init-freebsd
$(INIT_BINARY_BSD): $(INIT_SRC) $(SYSROOT_BSD_TARGET)
$(CC_BSD) -std=c23 -O2 -static -Wall $(INIT_DEFS) -lutil -o $@ $(INIT_SRC) $(INIT_DEFS)
endif

NITRO_INIT_BINARY= init/nitro/init
$(NITRO_INIT_BINARY): $(NITRO_INIT_SRC)
$(CC) -O2 -static -Wall $(NITRO_INIT_LD_FLAGS) -o $@ $(NITRO_INIT_SRC) $(NITRO_INIT_LD_FLAGS)
Expand Down Expand Up @@ -166,11 +192,27 @@ $(PACKAGES_FILE):
@mkdir -p $(ROOTFS_TMP)
@curl -fL -o $@ https://deb.debian.org/debian/dists/$(DEBIAN_DIST)/main/binary-$(ARCH)/Packages.xz

# FreeBSD sysroot preparation rules for cross-compilation on macOS
FREEBSD_BASE_TXZ = $(FREEBSD_ROOTFS_DIR)/base.txz

.INTERMEDIATE: $(FREEBSD_BASE_TXZ)

$(FREEBSD_ROOTFS_DIR)/.sysroot_ready: $(FREEBSD_BASE_TXZ)
@echo "Extracting FreeBSD base to $(FREEBSD_ROOTFS_DIR)..."
@cd $(FREEBSD_ROOTFS_DIR) && tar xJf base.txz 2>/dev/null || true
@touch $@

$(FREEBSD_BASE_TXZ):
@echo "Downloading FreeBSD $(FREEBSD_VERSION) base for $(ARCH)..."
@mkdir -p $(FREEBSD_ROOTFS_DIR)
@curl -fL -o $@ https://download.freebsd.org/releases/$(ARCH)/$(FREEBSD_VERSION)/base.txz

clean-sysroot:
rm -rf $(ROOTFS_DIR)
rm -rf $(FREEBSD_ROOTFS_DIR)


$(LIBRARY_RELEASE_$(OS)): $(INIT_BINARY)
$(LIBRARY_RELEASE_$(OS)): $(INIT_BINARY) $(INIT_BINARY_BSD)
cargo build --release $(FEATURE_FLAGS)
ifeq ($(SEV),1)
mv target/release/libkrun.so target/release/$(KRUN_BASE_$(OS))
Expand All @@ -189,7 +231,7 @@ endif
endif
cp target/release/$(KRUN_BASE_$(OS)) $(LIBRARY_RELEASE_$(OS))

$(LIBRARY_DEBUG_$(OS)): $(INIT_BINARY)
$(LIBRARY_DEBUG_$(OS)): $(INIT_BINARY) $(INIT_BINARY_BSD)
cargo build $(FEATURE_FLAGS)
ifeq ($(SEV),1)
mv target/debug/libkrun.so target/debug/$(KRUN_BASE_$(OS))
Expand Down Expand Up @@ -221,7 +263,12 @@ install: libkrun.pc
cd $(DESTDIR)$(PREFIX)/$(LIBDIR_$(OS))/ ; ln -sf $(KRUN_BINARY_$(OS)) $(KRUN_SONAME_$(OS)) ; ln -sf $(KRUN_SONAME_$(OS)) $(KRUN_BASE_$(OS))

clean:
ifeq ($(BUILD_INIT),1)
rm -f $(INIT_BINARY)
endif
ifeq ($(BUILD_BSD_INIT),1)
rm -f $(INIT_BINARY_BSD)
endif
cargo clean
rm -rf test-prefix
cd tests; cargo clean
Expand Down
156 changes: 150 additions & 6 deletions init/init.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,21 @@
#include <sys/resource.h>
#include <sys/socket.h>
#include <sys/stat.h>
#if __FreeBSD__
Copy link
Contributor

@valpackett valpackett Dec 9, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hm, why are all of them just if and not ifdef?

Copy link
Contributor Author

@nohajc nohajc Dec 9, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No particular reason. Both should work. This way the variable must evaluate to true not just "be defined".

#include <kenv.h>
#include <libutil.h>
#include <sys/mount.h>
#include <sys/param.h>
#else
#include <sys/statfs.h>
#endif
#include <sys/time.h>
#include <sys/types.h>
#include <sys/wait.h>

#if __linux__
#include <linux/vm_sockets.h>
#endif

#include "jsmn.h"

Expand Down Expand Up @@ -50,6 +59,92 @@ static char *snp_get_luks_passphrase(char *, char *, char *, int *);

char DEFAULT_KRUN_INIT[] = "/bin/sh";

#if __FreeBSD__

#define b64_ntop __b64_ntop
#define b64_pton __b64_pton
/* There are no header files for these functions. */

int b64_ntop(unsigned char const *src, size_t srclength, char *target,
size_t targsize);
int b64_pton(char const *src, unsigned char *target, size_t targsize);

static char *get_kenv(const char *name)
{
static char kenv_value[KENV_MVALLEN + 1];
if (kenv(KENV_GET, name, kenv_value, KENV_MVALLEN + 1) < 0) {
return NULL;
}
return kenv_value;
}

static int get_krun_init_argv_flat(char *buf, int buf_len)
{
int len_total = 0;
int len_part;
char *buf_ptr = buf;
char name_buf[32];
int idx = 0;

while (true) {
snprintf(name_buf, sizeof(name_buf), "KRUN_INIT_ARGV%d", idx++);
char *argv_b64 = get_kenv(name_buf);
if (!argv_b64) {
break;
}
len_part =
b64_pton(argv_b64, (unsigned char *)buf_ptr, buf_len - len_total);
buf_ptr += len_part;
len_total += len_part;
}

return len_total;
}

#define getenv get_kenv

#define _PATH_CONSOLE "/dev/console"
#define _PATH_DEVNULL "/dev/null"
#define _PATH_INITLOG "/init.log"
/*
* Start a session and allocate a controlling terminal.
* Only called by children of init after forking.
*/
static void open_console(void)
{
int fd;

/*
* Try to open /dev/console. Open the device with O_NONBLOCK to
* prevent potential blocking on a carrier.
*/
revoke(_PATH_CONSOLE);
if ((fd = open(_PATH_CONSOLE, O_RDWR | O_NONBLOCK)) != -1) {
(void)fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) & ~O_NONBLOCK);
if (login_tty(fd) == 0)
return;
close(fd);
}

/* No luck. Log output to file if possible. */
if ((fd = open(_PATH_DEVNULL, O_RDWR)) == -1) {
_exit(1);
}
if (fd != STDIN_FILENO) {
dup2(fd, STDIN_FILENO);
close(fd);
}
fd = open(_PATH_INITLOG, O_WRONLY | O_APPEND | O_CREAT, 0644);
if (fd == -1)
dup2(STDIN_FILENO, STDOUT_FILENO);
else if (fd != STDOUT_FILENO) {
dup2(fd, STDOUT_FILENO);
close(fd);
}
dup2(STDOUT_FILENO, STDERR_FILENO);
}
#endif

static void set_rlimits(const char *rlimits)
{
unsigned long long int lim_id, lim_cur, lim_max;
Expand Down Expand Up @@ -395,6 +490,7 @@ static int chroot_luks()

static int mount_filesystems()
{
#if __linux__
char *const DIRS_LEVEL1[] = {"/dev", "/proc", "/sys"};
char *const DIRS_LEVEL2[] = {"/dev/pts", "/dev/shm"};
int i;
Expand Down Expand Up @@ -451,7 +547,7 @@ static int mount_filesystems()

/* May fail if already exists and that's fine. */
symlink("/proc/self/fd", "/dev/fd");

#endif
return 0;
}

Expand Down Expand Up @@ -1003,6 +1099,7 @@ void set_exit_code(int code)
close(fd);
}

#if __linux__
int try_mount(const char *source, const char *target, const char *fstype,
unsigned long mountflags, const void *data)
{
Expand Down Expand Up @@ -1037,11 +1134,19 @@ int try_mount(const char *source, const char *target, const char *fstype,

return mount_status;
}
#endif

char *clone_str(const char *str)
{
if (str == NULL) {
return NULL;
}
return strdup(str);
}

int main(int argc, char **argv)
{
struct ifreq ifr;
int fd;
int sockfd;
int status;
int saved_errno;
Expand All @@ -1051,14 +1156,21 @@ int main(int argc, char **argv)
char *krun_home;
char *krun_term;
char *krun_init;
#if __linux__
int fd;
char *krun_root;
char *krun_root_fstype;
char *krun_root_options;
#endif
char *env_init_pid1;
char *config_workdir, *env_workdir;
char *rlimits;
char **config_argv, **exec_argv;

#if __FreeBSD__
open_console();
#endif

#ifdef TDX
if (mkdir("/tmp", 0755) < 0 && errno != EEXIST) {
perror("mkdir(/tmp)");
Expand Down Expand Up @@ -1092,21 +1204,25 @@ int main(int argc, char **argv)
exit(-2);
}

krun_root = getenv("KRUN_BLOCK_ROOT_DEVICE");
#if __linux__
krun_root = clone_str(getenv("KRUN_BLOCK_ROOT_DEVICE"));
Copy link
Contributor Author

@nohajc nohajc Dec 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixing a potential issue here:

The string pointed to by the return value of getenv() may be statically allocated, and can be modified by a subsequent call to getenv(), putenv(3), setenv(3), or unsetenv(3).

https://linux.die.net/man/3/getenv

if (krun_root) {
if (mkdir("/newroot", 0755) < 0 && errno != EEXIST) {
perror("mkdir(/newroot)");
exit(-1);
}

krun_root_fstype = getenv("KRUN_BLOCK_ROOT_FSTYPE");
krun_root_options = getenv("KRUN_BLOCK_ROOT_OPTIONS");
krun_root_fstype = clone_str(getenv("KRUN_BLOCK_ROOT_FSTYPE"));
krun_root_options = clone_str(getenv("KRUN_BLOCK_ROOT_OPTIONS"));

if (try_mount(krun_root, "/newroot", krun_root_fstype, 0,
krun_root_options) < 0) {
perror("mount KRUN_BLOCK_ROOT_DEVICE");
exit(-1);
}
free(krun_root);
free(krun_root_fstype);
free(krun_root_options);

chdir("/newroot");

Expand Down Expand Up @@ -1137,10 +1253,15 @@ int main(int argc, char **argv)
perror("Couldn't set shared propagation on the root mount");
exit(-1);
}
#endif

setsid();
ioctl(0, TIOCSCTTY, 1);

#if __FreeBSD__
setlogin("root");
#endif

sockfd = socket(AF_INET, SOCK_DGRAM, 0);
if (sockfd >= 0) {
memset(&ifr, 0, sizeof ifr);
Expand Down Expand Up @@ -1184,16 +1305,35 @@ int main(int argc, char **argv)
chdir(config_workdir);
}

#if __FreeBSD__
exec_argv = malloc(MAX_ARGS * sizeof(char *));
#else
exec_argv = argv;
#endif
krun_init = getenv("KRUN_INIT");
if (krun_init) {
exec_argv[0] = krun_init;
exec_argv[0] = clone_str(krun_init);
} else if (config_argv) {
exec_argv = config_argv;
} else {
exec_argv[0] = &DEFAULT_KRUN_INIT[0];
}

#if __FreeBSD__
int i = 1;
static char argv_flat[512];
Copy link
Collaborator

@slp slp Dec 15, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you go for the unified solution for both Linux and FreeBSD, please make this a value that can be "reallocatable" so we aren't limited to 512 bytes.

int argv_flat_len = get_krun_init_argv_flat(argv_flat, sizeof(argv_flat));
Copy link
Collaborator

@slp slp Dec 15, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Given than most of the VMM context works fine with both Linux and FreeBSD without any changes, it's kind of a bummer having to make a difference for argument passing. Please consider implementing this unconditionally, so it's used on Linux too.

Most use cases should be using with /.krun_config.json, but for those that aren't, I think it's better to have a single path.

Copy link
Contributor Author

@nohajc nohajc Dec 15, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree it would be better to unify this. At the same time, I didn't want to do any disruptive changes for Linux, especially before release.

So, would it be OK to merge it like this (as phase 1) and then unify in a separate PR?

Or do you prefer to do it properly in one go? I can also live with this not being included in 1.17.


int j = 0;
while (j < argv_flat_len) {
exec_argv[i++] = &argv_flat[j];
for (; j < argv_flat_len && argv_flat[j] != 0; j++) {
}
j++;
}
exec_argv[i] = NULL;
#endif

env_init_pid1 = getenv("KRUN_INIT_PID1");
if (env_init_pid1 && *env_init_pid1 == '1') {
init_pid1 = true;
Expand All @@ -1219,9 +1359,13 @@ int main(int argc, char **argv)
}
if (child == 0) { // child
exec_init:
#if __FreeBSD__
open_console();
#else
if (setup_redirects() < 0) {
exit(125);
}
#endif
if (execvp(exec_argv[0], exec_argv) < 0) {
saved_errno = errno;
printf("Couldn't execute '%s' inside the vm: %s\n", exec_argv[0],
Expand Down
Loading
Loading