Part 3 · 1 chapters · ~8 min

A Container Runtime in Go

Writing a minimal runtime: re-executing with clone flags, UTS, PID, mount, network and user namespaces, cgroup placement, mounting /proc, pivot_root into an extracted image, executing the command, and what runc adds on top (OCI config, seccomp, capabilities, hooks).

About a hundred lines

code

// box.go (Linux only; run as root or with user namespaces), after Liz Rice's "containers from scratch"
package main

import ("os"; "os/exec"; "path/filepath"; "strconv"; "syscall")

func main() {
	switch os.Args[1] {
	case "run":   run()
	case "child": child()
	}
}

func run() {
	cmd := exec.Command("/proc/self/exe", append([]string{"child"}, os.Args[2:]...)...)
	cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Cloneflags:   syscall.CLONE_NEWUTS | syscall.CLONE_NEWPID | syscall.CLONE_NEWNS | syscall.CLONE_NEWNET,
		Unshareflags: syscall.CLONE_NEWNS,
	}
	must(cmd.Start())
	cg := "/sys/fs/cgroup/box"
	os.MkdirAll(cg, 0755)
	os.WriteFile(filepath.Join(cg, "memory.max"), []byte("256M"), 0644)
	os.WriteFile(filepath.Join(cg, "pids.max"), []byte("64"), 0644)
	os.WriteFile(filepath.Join(cg, "cgroup.procs"), []byte(strconv.Itoa(cmd.Process.Pid)), 0644)
	must(cmd.Wait())
}

func child() {
	must(syscall.Sethostname([]byte("box")))
	root := "/var/lib/box/rootfs"                         // e.g. an extracted alpine minirootfs
	must(syscall.Mount(root, root, "", syscall.MS_BIND|syscall.MS_REC, ""))
	os.MkdirAll(root+"/.old", 0700)
	must(syscall.PivotRoot(root, root+"/.old"))
	must(os.Chdir("/"))
	must(syscall.Unmount("/.old", syscall.MNT_DETACH))
	must(syscall.Mount("proc", "/proc", "proc", 0, ""))
	must(syscall.Exec(os.Args[2], os.Args[2:], os.Environ()))
}

func must(err error) { if err != nil { panic(err) } }

// go build -o box . && sudo ./box run /bin/sh   → hostname: box; ps: PID 1 is sh; ls /: alpine's files

Missing for production: network setup (veth into a bridge), user namespaces for rootless, seccomp and capability dropping (part 6), an OCI config format, and lifecycle management. That is what runc implements.

A CONTAINER RUNTIME IN GO

what `run` does, step by step

swipe the figure sideways, or tap expand for full screen

1/4

re-exec

Go cannot safely fork a running multi-threaded runtime, so the binary re-executes itself (/proc/self/exe) with a "child" argument and namespace flags in SysProcAttr.Cloneflags.

re-exec /proc/self/exeGo needs this trick