Add symlink-based LUN device path resolver for Azure NVMe support

handler/common_event_format.go

@@ -52,7 +52,7 @@ func (cef concreteCommonEventFormat) ProduceHTTPRequestEventLog(request *http.Re
 		var buffer bytes.Buffer
 
 		buffer.WriteString(extension)
-		buffer.WriteString(fmt.Sprintf("cs4=%s cs4Label=statusReason", respBody))
+		fmt.Fprintf(&buffer, "cs4=%s cs4Label=statusReason", respBody)
 		extension = buffer.String()
 	}
 
@@ -73,7 +73,7 @@ func (cef concreteCommonEventFormat) ProduceNATSRequestEventLog(addr string, por
 		var buffer bytes.Buffer
 
 		buffer.WriteString(extension)
-		buffer.WriteString(fmt.Sprintf("cs1=%s cs1Label=statusReason", respBody))
+		fmt.Fprintf(&buffer, "cs1=%s cs1Label=statusReason", respBody)
 		extension = buffer.String()
 	}
 

infrastructure/devicepathresolver/fallback_device_path_resolver.go

@@ -0,0 +1,49 @@
+package devicepathresolver
+
+import (
+	bosherr "github.com/cloudfoundry/bosh-utils/errors"
+	boshlog "github.com/cloudfoundry/bosh-utils/logger"
+
+	boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings"
+)
+
+type FallbackDevicePathResolver struct {
+	primary   DevicePathResolver
+	secondary DevicePathResolver
+	logger    boshlog.Logger
+	logTag    string
+}
+
+func NewFallbackDevicePathResolver(
+	primary DevicePathResolver,
+	secondary DevicePathResolver,
+	logger boshlog.Logger,
+) DevicePathResolver {
+	return FallbackDevicePathResolver{
+		primary:   primary,
+		secondary: secondary,
+		logger:    logger,
+		logTag:    "fallbackDevicePathResolver",
+	}
+}
+
+func (r FallbackDevicePathResolver) GetRealDevicePath(diskSettings boshsettings.DiskSettings) (string, bool, error) {
+	realPath, _, err := r.primary.GetRealDevicePath(diskSettings)
+	if err == nil {
+		r.logger.Debug(r.logTag, "Primary resolver resolved disk %+v as '%s'", diskSettings, realPath)
+		return realPath, false, nil
+	}
+
+	r.logger.Debug(r.logTag,
+		"Primary resolver failed for disk %+v: %s. Trying secondary resolver.",
+		diskSettings, err.Error(),
+	)
+
+	realPath, timeout, err := r.secondary.GetRealDevicePath(diskSettings)
+	if err != nil {
+		return "", timeout, bosherr.WrapError(err, "Secondary resolver also failed")
+	}
+
+	r.logger.Debug(r.logTag, "Secondary resolver resolved disk %+v as '%s'", diskSettings, realPath)
+	return realPath, false, nil
+}

infrastructure/devicepathresolver/fallback_device_path_resolver_test.go

@@ -0,0 +1,102 @@
+package devicepathresolver_test
+
+import (
+	"errors"
+
+	boshlog "github.com/cloudfoundry/bosh-utils/logger"
+
+	fakedpresolv "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver/fakes"
+	boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	. "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver"
+)
+
+var _ = Describe("FallbackDevicePathResolver", func() {
+	var (
+		pathResolver      DevicePathResolver
+		primaryResolver   *fakedpresolv.FakeDevicePathResolver
+		secondaryResolver *fakedpresolv.FakeDevicePathResolver
+
+		diskSettings boshsettings.DiskSettings
+	)
+
+	BeforeEach(func() {
+		primaryResolver = fakedpresolv.NewFakeDevicePathResolver()
+		secondaryResolver = fakedpresolv.NewFakeDevicePathResolver()
+		logger := boshlog.NewLogger(boshlog.LevelNone)
+		pathResolver = NewFallbackDevicePathResolver(primaryResolver, secondaryResolver, logger)
+
+		diskSettings = boshsettings.DiskSettings{
+			Lun:          "1",
+			HostDeviceID: "fake-host-device-id",
+		}
+	})
+
+	Describe("GetRealDevicePath", func() {
+		Context("when primary resolver returns a path", func() {
+			BeforeEach(func() {
+				primaryResolver.RealDevicePath = "/dev/nvme0n3"
+			})
+
+			It("returns the primary path", func() {
+				realPath, timeout, err := pathResolver.GetRealDevicePath(diskSettings)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(timeout).To(BeFalse())
+				Expect(realPath).To(Equal("/dev/nvme0n3"))
+			})
+
+			It("does not call the secondary resolver", func() {
+				_, _, err := pathResolver.GetRealDevicePath(diskSettings)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(secondaryResolver.GetRealDevicePathDiskSettings).To(Equal(boshsettings.DiskSettings{}))
+			})
+		})
+
+		Context("when primary resolver errors", func() {
+			BeforeEach(func() {
+				primaryResolver.GetRealDevicePathErr = errors.New("symlink not found")
+			})
+
+			It("falls back to the secondary resolver", func() {
+				secondaryResolver.RealDevicePath = "/dev/sdc"
+
+				realPath, timeout, err := pathResolver.GetRealDevicePath(diskSettings)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(timeout).To(BeFalse())
+				Expect(realPath).To(Equal("/dev/sdc"))
+
+				Expect(secondaryResolver.GetRealDevicePathDiskSettings).To(Equal(diskSettings))
+			})
+
+			Context("when secondary resolver also errors", func() {
+				BeforeEach(func() {
+					secondaryResolver.GetRealDevicePathErr = errors.New("scsi device not found")
+				})
+
+				It("returns the secondary error wrapped", func() {
+					_, timeout, err := pathResolver.GetRealDevicePath(diskSettings)
+					Expect(err).To(HaveOccurred())
+					Expect(err.Error()).To(ContainSubstring("Secondary resolver also failed"))
+					Expect(err.Error()).To(ContainSubstring("scsi device not found"))
+					Expect(timeout).To(BeFalse())
+				})
+			})
+
+			Context("when secondary resolver times out", func() {
+				BeforeEach(func() {
+					secondaryResolver.GetRealDevicePathErr = errors.New("timed out")
+					secondaryResolver.GetRealDevicePathTimedOut = true
+				})
+
+				It("returns timeout from secondary", func() {
+					_, timeout, err := pathResolver.GetRealDevicePath(diskSettings)
+					Expect(err).To(HaveOccurred())
+					Expect(timeout).To(BeTrue())
+				})
+			})
+		})
+	})
+})

infrastructure/devicepathresolver/symlink_lun_device_path_resolver.go

@@ -0,0 +1,69 @@
+package devicepathresolver
+
+import (
+	"path"
+	"time"
+
+	bosherr "github.com/cloudfoundry/bosh-utils/errors"
+	boshlog "github.com/cloudfoundry/bosh-utils/logger"
+	boshsys "github.com/cloudfoundry/bosh-utils/system"
+
+	boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings"
+)
+
+type SymlinkLunDevicePathResolver struct {
+	diskWaitTimeout time.Duration
+	basePath        string
+	fs              boshsys.FileSystem
+
+	logTag string
+	logger boshlog.Logger
+}
+
+func NewSymlinkLunDevicePathResolver(
+	basePath string,
+	diskWaitTimeout time.Duration,
+	fs boshsys.FileSystem,
+	logger boshlog.Logger,
+) SymlinkLunDevicePathResolver {
+	return SymlinkLunDevicePathResolver{
+		basePath:        basePath,
+		diskWaitTimeout: diskWaitTimeout,
+		fs:              fs,
+
+		logTag: "symlinkLunResolver",
+		logger: logger,
+	}
+}
+
+func (r SymlinkLunDevicePathResolver) GetRealDevicePath(diskSettings boshsettings.DiskSettings) (string, bool, error) {
+	if diskSettings.Lun == "" {
+		return "", false, bosherr.Error("Disk lun is not set")
+	}
+
+	lunSymlink := path.Join(r.basePath, diskSettings.Lun)
+	r.logger.Debug(r.logTag, "Looking up LUN symlink '%s'", lunSymlink)
+
+	stopAfter := time.Now().Add(r.diskWaitTimeout)
+
+	for {
+		if time.Now().After(stopAfter) {
+			return "", true, bosherr.Errorf("Timed out waiting for symlink '%s' to resolve", lunSymlink)
+		}
+
+		realPath, err := r.fs.ReadAndFollowLink(lunSymlink)
+		if err != nil {
+			r.logger.Debug(r.logTag, "Symlink '%s' not yet available: %s", lunSymlink, err.Error())
+			time.Sleep(100 * time.Millisecond)
+			continue
+		}
+
+		if r.fs.FileExists(realPath) {
+			r.logger.Debug(r.logTag, "Resolved LUN symlink '%s' to real path '%s'", lunSymlink, realPath)
+			return realPath, false, nil
+		}
+
+		r.logger.Debug(r.logTag, "Real path '%s' does not yet exist", realPath)
+		time.Sleep(100 * time.Millisecond)
+	}
+}

infrastructure/devicepathresolver/symlink_lun_device_path_resolver_test.go

@@ -0,0 +1,143 @@
+package devicepathresolver_test
+
+import (
+	"os"
+	"runtime"
+	"time"
+
+	boshlog "github.com/cloudfoundry/bosh-utils/logger"
+	fakesys "github.com/cloudfoundry/bosh-utils/system/fakes"
+
+	boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	. "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver"
+)
+
+var _ = Describe("SymlinkLunDevicePathResolver", func() {
+	var (
+		fs           *fakesys.FakeFileSystem
+		diskSettings boshsettings.DiskSettings
+		pathResolver DevicePathResolver
+	)
+
+	BeforeEach(func() {
+		if runtime.GOOS == "windows" {
+			Skip("Not applicable on Windows")
+		}
+
+		fs = fakesys.NewFakeFileSystem()
+		pathResolver = NewSymlinkLunDevicePathResolver(
+			"/dev/disk/azure/data/by-lun",
+			500*time.Millisecond,
+			fs,
+			boshlog.NewLogger(boshlog.LevelNone),
+		)
+		diskSettings = boshsettings.DiskSettings{
+			Lun:          "2",
+			HostDeviceID: "fake-host-device-id",
+		}
+	})
+
+	Describe("GetRealDevicePath", func() {
+		Context("when lun is not set", func() {
+			It("returns an error", func() {
+				diskSettings.Lun = ""
+				_, _, err := pathResolver.GetRealDevicePath(diskSettings)
+				Expect(err).To(HaveOccurred())
+				Expect(err.Error()).To(ContainSubstring("Disk lun is not set"))
+			})
+		})
+
+		Context("when symlink exists and points to a real device (SCSI)", func() {
+			BeforeEach(func() {
+				err := fs.MkdirAll("/dev/disk/azure/data/by-lun", os.FileMode(0750))
+				Expect(err).ToNot(HaveOccurred())
+
+				err = fs.Symlink("/dev/sdc", "/dev/disk/azure/data/by-lun/2")
+				Expect(err).ToNot(HaveOccurred())
+
+				err = fs.MkdirAll("/dev/sdc", os.FileMode(0750))
+				Expect(err).ToNot(HaveOccurred())
+			})
+
+			It("returns the real device path", func() {
+				realPath, timeout, err := pathResolver.GetRealDevicePath(diskSettings)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(timeout).To(BeFalse())
+				Expect(realPath).To(Equal("/dev/sdc"))
+			})
+		})
+
+		Context("when symlink exists and points to a real NVMe device", func() {
+			BeforeEach(func() {
+				err := fs.MkdirAll("/dev/disk/azure/data/by-lun", os.FileMode(0750))
+				Expect(err).ToNot(HaveOccurred())
+
+				err = fs.Symlink("/dev/nvme0n4", "/dev/disk/azure/data/by-lun/2")
+				Expect(err).ToNot(HaveOccurred())
+
+				err = fs.MkdirAll("/dev/nvme0n4", os.FileMode(0750))
+				Expect(err).ToNot(HaveOccurred())
+			})
+
+			It("returns the real NVMe device path", func() {
+				realPath, timeout, err := pathResolver.GetRealDevicePath(diskSettings)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(timeout).To(BeFalse())
+				Expect(realPath).To(Equal("/dev/nvme0n4"))
+			})
+		})
+
+		Context("when symlink does not exist", func() {
+			It("times out", func() {
+				_, timeout, err := pathResolver.GetRealDevicePath(diskSettings)
+				Expect(err).To(HaveOccurred())
+				Expect(err.Error()).To(ContainSubstring("Timed out"))
+				Expect(timeout).To(BeTrue())
+			})
+		})
+
+		Context("when symlink exists but real path does not exist yet", func() {
+			BeforeEach(func() {
+				err := fs.MkdirAll("/dev/disk/azure/data/by-lun", os.FileMode(0750))
+				Expect(err).ToNot(HaveOccurred())
+
+				err = fs.Symlink("/dev/sdc", "/dev/disk/azure/data/by-lun/2")
+				Expect(err).ToNot(HaveOccurred())
+				// Note: /dev/sdc is NOT created - real path does not exist yet
+			})
+
+			It("times out waiting for the real device", func() {
+				_, timeout, err := pathResolver.GetRealDevicePath(diskSettings)
+				Expect(err).To(HaveOccurred())
+				Expect(err.Error()).To(ContainSubstring("Timed out"))
+				Expect(timeout).To(BeTrue())
+			})
+		})
+
+		Context("with LUN 0 (ephemeral disk)", func() {
+			BeforeEach(func() {
+				diskSettings.Lun = "0"
+
+				err := fs.MkdirAll("/dev/disk/azure/data/by-lun", os.FileMode(0750))
+				Expect(err).ToNot(HaveOccurred())
+
+				err = fs.Symlink("/dev/nvme0n2", "/dev/disk/azure/data/by-lun/0")
+				Expect(err).ToNot(HaveOccurred())
+
+				err = fs.MkdirAll("/dev/nvme0n2", os.FileMode(0750))
+				Expect(err).ToNot(HaveOccurred())
+			})
+
+			It("returns the correct device for LUN 0", func() {
+				realPath, timeout, err := pathResolver.GetRealDevicePath(diskSettings)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(timeout).To(BeFalse())
+				Expect(realPath).To(Equal("/dev/nvme0n2"))
+			})
+		})
+	})
+})