feat(9pfs volumes): Add support for 9pfs shared volumes

Implement a method of mounting local paths inside unikernels
(Unikraft only at this moment) based on a dedicated annotation.
This can be used in conjunction with NFS shared volumes in a Kubernetes
cluster, when a NFS provider has been configured.

The new annotation is called "com.urunc.unikernel.ninePFSMntPoint" and
it holds the value of mount point inside the unikernel, or in other
words, the value of "mountPath" specified by the volumeMount field of
the Kubernetes template.

The recommended usage is by setting it in the Pod specification of the
Kubernetes template.

In Unikontainer Exec method, urunc will search the list of mount points
passed from containerd and find a match for the mount point specified by
the annotation. Once found, urunc will be able to determine the local
path on the worker node of the PersistenVolume or PersistentVolumeClaim
and instruct qemu to mount it via 9pfs provider into the unikernel.

Signed-off-by: Ciprian Barbu <ciprian.barbu@thalesgroup.com>

Author: Ciprian Barbu

pkg/unikontainers/config.go

@@ -36,15 +36,16 @@ var ErrEmptyAnnotations = errors.New("spec annotations are empty")
 // Urunc specific annotations
 // ALways keep it in sync with the struct UnikernelConfig struct
 const (
-	annotType          = "com.urunc.unikernel.unikernelType"
-	annotVersion       = "com.urunc.unikernel.unikernelVersion"
-	annotBinary        = "com.urunc.unikernel.binary"
-	annotCmdLine       = "com.urunc.unikernel.cmdline"
-	annotHypervisor    = "com.urunc.unikernel.hypervisor"
-	annotInitrd        = "com.urunc.unikernel.initrd"
-	annotBlock         = "com.urunc.unikernel.block"
-	annotBlockMntPoint = "com.urunc.unikernel.blkMntPoint"
-	annotUseDMBlock    = "com.urunc.unikernel.useDMBlock"
+	annotType            = "com.urunc.unikernel.unikernelType"
+	annotVersion         = "com.urunc.unikernel.unikernelVersion"
+	annotBinary          = "com.urunc.unikernel.binary"
+	annotCmdLine         = "com.urunc.unikernel.cmdline"
+	annotHypervisor      = "com.urunc.unikernel.hypervisor"
+	annotInitrd          = "com.urunc.unikernel.initrd"
+	annotBlock           = "com.urunc.unikernel.block"
+	annotBlockMntPoint   = "com.urunc.unikernel.blkMntPoint"
+	annotUseDMBlock      = "com.urunc.unikernel.useDMBlock"
+	annotNinePFSMntPoint = "com.urunc.unikernel.ninePFSMntPoint"
 )
 
 // A UnikernelConfig struct holds the info provided by bima image on how to execute our unikernel
@@ -58,6 +59,7 @@ type UnikernelConfig struct {
 	Block            string `json:"com.urunc.unikernel.block,omitempty"`
 	BlkMntPoint      string `json:"com.urunc.unikernel.blkMntPoint,omitempty"`
 	UseDMBlock       string `json:"com.urunc.unikernel.useDMBlock"`
+	NinePFSMntPoint  string `json:"com.urunc.unikernel.ninePFSMntPoint,omitempty"`
 }
 
 // GetUnikernelConfig tries to get the Unikernel config from the bundle annotations.
@@ -104,6 +106,7 @@ func getConfigFromSpec(spec *specs.Spec) (*UnikernelConfig, error) {
 	block := spec.Annotations[annotBlock]
 	blkMntPoint := spec.Annotations[annotBlockMntPoint]
 	useDMBlock := spec.Annotations[annotUseDMBlock]
+	ninePFSMntPoint := spec.Annotations[annotNinePFSMntPoint]
 
 	Log.WithFields(logrus.Fields{
 		"unikernelType":    unikernelType,
@@ -115,6 +118,7 @@ func getConfigFromSpec(spec *specs.Spec) (*UnikernelConfig, error) {
 		"block":            block,
 		"blkMntPoint":      blkMntPoint,
 		"useDMBlock":       useDMBlock,
+		"ninePFSMntPoint":  ninePFSMntPoint,
 	}).Info("urunc annotations")
 
 	// TODO: We need to use a better check to see if annotations were empty
@@ -132,6 +136,7 @@ func getConfigFromSpec(spec *specs.Spec) (*UnikernelConfig, error) {
 		Block:            block,
 		BlkMntPoint:      blkMntPoint,
 		UseDMBlock:       useDMBlock,
+		NinePFSMntPoint:  ninePFSMntPoint,
 	}, nil
 }
 
@@ -171,6 +176,7 @@ func getConfigFromJSON(jsonFilePath string) (*UnikernelConfig, error) {
 		"block":            conf.Block,
 		"blkMntPoint":      conf.BlkMntPoint,
 		"useDMBlock":       conf.UseDMBlock,
+		"ninePFSMntPoint":  conf.NinePFSMntPoint,
 	}).Info(uruncJSONFilename + " annotations")
 	return &conf, nil
 }
@@ -231,6 +237,12 @@ func (c *UnikernelConfig) decode() error {
 	}
 	c.UseDMBlock = string(decoded)
 
+	decoded, err = base64.StdEncoding.DecodeString(c.NinePFSMntPoint)
+	if err != nil {
+		return fmt.Errorf("failed to decode ninePFSMntPoint: %v", err)
+	}
+	c.NinePFSMntPoint = string(decoded)
+
 	return nil
 }
 

pkg/unikontainers/hypervisors/qemu.go

@@ -95,6 +95,12 @@ func (q *Qemu) Execve(args ExecArgs, ukernel unikernels.Unikernel) error {
 	if args.InitrdPath != "" {
 		cmdString += " -initrd " + args.InitrdPath
 	}
+
+	if len(args.NinePFSVols) > 0 {
+		cmdString += " -fsdev local,id=p9idfs1,path=" + args.NinePFSVols[0].Src + ",security_model=mapped-xattr"
+		cmdString += " -device virtio-9p-pci,fsdev=p9idfs1,mount_tag=fs1"
+	}
+
 	cmdString += ukernel.MonitorCli(qemuString)
 	exArgs := strings.Split(cmdString, " ")
 	exArgs = append(exArgs, "-append", args.Command)

pkg/unikontainers/hypervisors/vmm.go

@@ -25,20 +25,27 @@ import (
 
 const DefaultMemory uint64 = 256 // The default memory for every hypervisor: 256 MB
 
+// MountSpec defines a specification for mounting a path from the host on the guest
+type MountSpec struct {
+	Src string
+	Dst string
+}
+
 // ExecArgs holds the data required by Execve to start the VMM
 // FIXME: add extra fields if required by additional VMM's
 type ExecArgs struct {
-	Container     string   // The container ID
-	UnikernelPath string   // The path of the unikernel inside rootfs
-	TapDevice     string   // The TAP device name
-	BlockDevice   string   // The block device path
-	InitrdPath    string   // The path to the initrd of the unikernel
-	Command       string   // The unikernel's command line
-	IPAddress     string   // The IP address of the TAP device
-	GuestMAC      string   // The MAC address of the guest network device
-	Seccomp       bool     // Enable or disable seccomp filters for the VMM
-	MemSizeB      uint64   // The size of the memory provided to the VM in bytes
-	Environment   []string // Environment
+	Container     string      // The container ID
+	UnikernelPath string      // The path of the unikernel inside rootfs
+	TapDevice     string      // The TAP device name
+	BlockDevice   string      // The block device path
+	InitrdPath    string      // The path to the initrd of the unikernel
+	Command       string      // The unikernel's command line
+	IPAddress     string      // The IP address of the TAP device
+	GuestMAC      string      // The MAC address of the guest network device
+	Seccomp       bool        // Enable or disable seccomp filters for the VMM
+	MemSizeB      uint64      // The size of the memory provided to the VM in bytes
+	Environment   []string    // Environment
+	NinePFSVols   []MountSpec // Spec for mounting volumes via 9pfs, (source, destination)
 }
 
 type VmmType string

pkg/unikontainers/unikernels/unikernel.go

@@ -38,6 +38,7 @@ type UnikernelParams struct {
 	RootFSType       string   // The rootfs type of the Unikernel
 	BlockMntPoint    string   // The mount point for the block device
 	Version          string   // The version of the unikernel
+	NinePFSMntPoint  string   // Mount point inside unikernel of the 9pfs theshared volume
 }
 
 var ErrNotSupportedUnikernel = errors.New("unikernel is not supported")

pkg/unikontainers/unikernels/unikraft.go

@@ -44,7 +44,7 @@ type UnikraftNet struct {
 }
 
 type UnikraftVFS struct {
-	RootFS string
+	VfsString string
 }
 
 func (u *Unikraft) CommandString() (string, error) {
@@ -59,7 +59,7 @@ func (u *Unikraft) CommandString() (string, error) {
 		u.Net.Address,
 		u.Net.Gateway,
 		u.Net.Mask,
-		u.VFS.RootFS,
+		u.VFS.VfsString,
 		u.Command), nil
 }
 
@@ -103,35 +103,38 @@ func (u *Unikraft) Init(data UnikernelParams) error {
 		u.Command = strings.Join(data.CmdLine[1:], " ")
 	}
 
-	return u.configureUnikraftArgs(data.RootFSType, data.EthDeviceIP, data.EthDeviceGateway, data.EthDeviceMask)
+	return u.configureUnikraftArgs(data.RootFSType, data.EthDeviceIP, data.EthDeviceGateway, data.EthDeviceMask, data.NinePFSMntPoint)
 }
 
-func (u *Unikraft) configureUnikraftArgs(rootFsType, ethDeviceIP, ethDeviceGateway, ethDeviceMask string) error {
+func (u *Unikraft) configureUnikraftArgs(rootFsType, ethDeviceIP, ethDeviceGateway, ethDeviceMask string, ninePFSMntPoint string) error {
 	setCompatArgs := func() {
 		u.Net.Address = "netdev.ipv4_addr=" + ethDeviceIP
 		u.Net.Gateway = "netdev.ipv4_gw_addr=" + ethDeviceGateway
 		u.Net.Mask = "netdev.ipv4_subnet_mask=" + ethDeviceMask
 		// TODO: We need to add support for actual block devices (e.g. virtio-blk)
 		// and sharedfs or any other Unikraft related ways to pass data to guest.
 		if rootFsType == "initrd" {
-			u.VFS.RootFS = "vfs.rootfs=" + "initrd"
+			u.VFS.VfsString = "vfs.rootfs=" + "initrd"
 		} else {
-			u.VFS.RootFS = ""
+			u.VFS.VfsString = ""
 		}
 	}
 
 	setCurrentArgs := func() {
+		u.VFS.VfsString += "vfs.fstab=[ "
 		u.Net.Address = "netdev.ip=" + ethDeviceIP + "/24:" + ethDeviceGateway + ":8.8.8.8"
 		// TODO: We need to add support for actual block devices (e.g. virtio-blk)
 		// and sharedfs or any other Unikraft related ways to pass data to guest.
 		if rootFsType == "initrd" {
 			// TODO: This needs better handling. We need to revisit this
 			// when we better understand all the available options for
 			// passing info inside unikraft unikernels.
-			u.VFS.RootFS = "vfs.fstab=[ \"initrd0:/:extract:::\" ]"
-		} else {
-			u.VFS.RootFS = ""
+			u.VFS.VfsString += "\"initrd0:/:extract:::\""
+		}
+		if len(ninePFSMntPoint) > 0 {
+			u.VFS.VfsString += "\"fs1:" + ninePFSMntPoint + ":9pfs:::mkmp\""
 		}
+		u.VFS.VfsString += " ]"
 	}
 
 	if u.Version == "" {

pkg/unikontainers/unikontainers.go

@@ -154,6 +154,8 @@ func (u *Unikontainer) Exec() error {
 	unikernelType := u.State.Annotations[annotType]
 	unikernelVersion := u.State.Annotations[annotVersion]
 
+	var mountSpecs []hypervisors.MountSpec
+
 	// TODO: Remove this when we chroot
 	unikernelPath, err := filepath.Rel("/", u.State.Annotations[annotBinary])
 	if err != nil {
@@ -184,6 +186,7 @@ func (u *Unikontainer) Exec() error {
 		BlockDevice:   "",
 		Seccomp:       true, // Enable Seccomp by default
 		MemSizeB:      0,
+		NinePFSVols:   []hypervisors.MountSpec{},
 		Environment:   os.Environ(),
 	}
 
@@ -276,6 +279,24 @@ func (u *Unikontainer) Exec() error {
 		}
 	}
 
+	// Handle custom support for 9pfs
+	// If the POD specifies the magic 'com.urunc.unikernel.9pfsMntPoint, urunc
+	// will search the list of Mounts in u.Spec and will determine the location
+	// on the worker node. That location will then be passed to 9pfs to mount it
+	// inside the unikernel.
+	ninePFSMntPoint := u.Spec.Annotations[annotNinePFSMntPoint]
+	if ninePFSMntPoint != "" {
+		for _, v := range u.Spec.Mounts {
+			if v.Destination == ninePFSMntPoint {
+				mountSpec := hypervisors.MountSpec{Src: v.Source, Dst: v.Destination}
+				// TODO: figure out if we can use a list of mount points or we should limit to just one
+				mountSpecs = append(mountSpecs, mountSpec)
+				unikernelParams.NinePFSMntPoint = ninePFSMntPoint
+			}
+		}
+		vmmArgs.NinePFSVols = mountSpecs
+	}
+
 	if unikernel.SupportsBlock() && vmmArgs.BlockDevice == "" && useDevmapper {
 		rootFsDevice, err := getBlockDevice(rootfsDir)
 		if err != nil {