Skip to content

Commit

Permalink
feat: share /lib/firmware across initramfs and rootfs
Browse files Browse the repository at this point in the history 
See #4816

Depending on the hardware and firmware type, firmware might be either
needed during initial boot (`initramfs`) or in the Talos running phase
(`rootfs`). As we don't want to have two copies of same firmware, share
the firmware by bind-mounting it from the `initramfs` down to `rootfs`
on switchroot.

This also cleans up `Dockerfile` to keep firmware only in `initramfs`.

Eventually we might get rid of some of the firmware and move it to the
system extensions.

Signed-off-by: Andrey Smirnov <[email protected]>
  • Loading branch information
@smira
smira committed Jan 27, 2022
1 parent ebec5d4 commit abfb258
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 29 deletions.
21 changes: 9 additions & 12 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ FROM --platform=arm64 ghcr.io/talos-systems/libressl:${PKGS} AS pkg-libressl-arm
FROM --platform=amd64 ghcr.io/talos-systems/libseccomp:${PKGS} AS pkg-libseccomp-amd64
FROM --platform=arm64 ghcr.io/talos-systems/libseccomp:${PKGS} AS pkg-libseccomp-arm64

FROM --platform=amd64 ghcr.io/talos-systems/linux-firmware:${PKGS} AS pkg-linux-firmware-amd64
FROM --platform=arm64 ghcr.io/talos-systems/linux-firmware:${PKGS} AS pkg-linux-firmware-arm64
# linux-firmware is not arch-specific
FROM --platform=amd64 ghcr.io/talos-systems/linux-firmware:${PKGS} AS pkg-linux-firmware

FROM --platform=amd64 ghcr.io/talos-systems/lvm2:${PKGS} AS pkg-lvm2-amd64
FROM --platform=arm64 ghcr.io/talos-systems/lvm2:${PKGS} AS pkg-lvm2-arm64
Expand Down Expand Up @@ -359,8 +359,6 @@ COPY --from=pkg-libjson-c-amd64 / /rootfs
COPY --from=pkg-libpopt-amd64 / /rootfs
COPY --from=pkg-libressl-amd64 / /rootfs
COPY --from=pkg-libseccomp-amd64 / /rootfs
COPY --from=pkg-linux-firmware-amd64 /lib/firmware/bnx2 /rootfs/lib/firmware/bnx2
COPY --from=pkg-linux-firmware-amd64 /lib/firmware/bnx2x /rootfs/lib/firmware/bnx2x
COPY --from=pkg-lvm2-amd64 / /rootfs
COPY --from=pkg-libaio-amd64 / /rootfs
COPY --from=pkg-musl-amd64 / /rootfs
Expand All @@ -378,7 +376,7 @@ COPY --from=machined-build-amd64 /machined /rootfs/sbin/init
# symlinks to avoid accidentally cleaning them up.
COPY ./hack/cleanup.sh /toolchain/bin/cleanup.sh
RUN cleanup.sh /rootfs
RUN mkdir -pv /rootfs/{boot,etc/cri/conf.d/hosts,usr/local/share,mnt,system,opt}
RUN mkdir -pv /rootfs/{boot,etc/cri/conf.d/hosts,lib/firmware,usr/local/share,mnt,system,opt}
RUN mkdir -pv /rootfs/{etc/kubernetes/manifests,etc/cni/net.d,usr/libexec/kubernetes}
RUN mkdir -pv /rootfs/opt/{containerd/bin,containerd/lib}
COPY --chmod=0644 hack/containerd.toml /rootfs/etc/containerd/config.toml
Expand All @@ -402,10 +400,6 @@ COPY --from=pkg-libjson-c-arm64 / /rootfs
COPY --from=pkg-libpopt-arm64 / /rootfs
COPY --from=pkg-libressl-arm64 / /rootfs
COPY --from=pkg-libseccomp-arm64 / /rootfs
COPY --from=pkg-linux-firmware-arm64 /lib/firmware/bnx2 /rootfs/lib/firmware/bnx2
COPY --from=pkg-linux-firmware-arm64 /lib/firmware/bnx2x /rootfs/lib/firmware/bnx2x
COPY --from=pkg-linux-firmware-arm64 /lib/firmware/rtl_nic /rootfs/lib/firmware/rtl_nic
COPY --from=pkg-linux-firmware-arm64 /lib/firmware/nvidia/tegra210 /rootfs/lib/firmware/nvidia/tegra210
COPY --from=pkg-lvm2-arm64 / /rootfs
COPY --from=pkg-libaio-arm64 / /rootfs
COPY --from=pkg-musl-arm64 / /rootfs
Expand All @@ -423,7 +417,7 @@ COPY --from=machined-build-arm64 /machined /rootfs/sbin/init
# symlinks to avoid accidentally cleaning them up.
COPY ./hack/cleanup.sh /toolchain/bin/cleanup.sh
RUN cleanup.sh /rootfs
RUN mkdir -pv /rootfs/{boot,etc/cri/conf.d/hosts,usr/local/share,mnt,system,opt}
RUN mkdir -pv /rootfs/{boot,etc/cri/conf.d/hosts,lib/firmware,usr/local/share,mnt,system,opt}
RUN mkdir -pv /rootfs/{etc/kubernetes/manifests,etc/cni/net.d,usr/libexec/kubernetes}
RUN mkdir -pv /rootfs/opt/{containerd/bin,containerd/lib}
COPY --chmod=0644 hack/containerd.toml /rootfs/etc/containerd/config.toml
Expand Down Expand Up @@ -463,8 +457,8 @@ WORKDIR /initramfs
COPY --from=squashfs-arm64 /rootfs.sqsh .
COPY --from=init-build-arm64 /init .
# copying over firmware binary blobs to initramfs
COPY --from=pkg-linux-firmware-arm64 /lib/firmware/rtl_nic ./lib/firmware/rtl_nic
COPY --from=pkg-linux-firmware-arm64 /lib/firmware/nvidia/tegra210 ./lib/firmware/nvidia/tegra210
COPY --from=pkg-linux-firmware /lib/firmware/rtl_nic ./lib/firmware/rtl_nic
COPY --from=pkg-linux-firmware /lib/firmware/nvidia/tegra210 ./lib/firmware/nvidia/tegra210
RUN find . -print0 \
| xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}"
RUN set -o pipefail \
Expand All @@ -478,6 +472,9 @@ FROM build AS initramfs-archive-amd64
WORKDIR /initramfs
COPY --from=squashfs-amd64 /rootfs.sqsh .
COPY --from=init-build-amd64 /init .
# copying over firmware binary blobs to initramfs
COPY --from=pkg-linux-firmware /lib/firmware/bnx2 ./lib/firmware/bnx2
COPY --from=pkg-linux-firmware /lib/firmware/bnx2x ./lib/firmware/bnx2x
RUN find . -print0 \
| xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}"
RUN set -o pipefail \
Expand Down
19 changes: 19 additions & 0 deletions internal/app/init/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,11 @@ func run() (err error) {
return err
}

// Bind mount the lib/firmware if needed.
if err = bindMountFirmware(); err != nil {
return err
}

// Switch into the new rootfs.
log.Println("entering the rootfs")

Expand Down Expand Up @@ -172,6 +177,20 @@ func mountRootFS() error {
return nil
}

func bindMountFirmware() error {
if _, err := os.Stat(constants.FirmwarePath); err != nil {
if os.IsNotExist(err) {
return nil
}

return err
}

log.Printf("bind mounting %s", constants.FirmwarePath)

return unix.Mount(constants.FirmwarePath, filepath.Join(constants.NewRoot, constants.FirmwarePath), "", unix.MS_BIND|unix.MS_RDONLY, "")
}

func main() {
defer recovery()

Expand Down
50 changes: 33 additions & 17 deletions internal/pkg/mount/switchroot/switchroot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
// Paths preserved in the initramfs.
var preservedPaths = map[string]struct{}{
constants.ExtensionsConfigFile: {},
constants.FirmwarePath: {},
}

// Switch moves the rootfs to a specified directory. See
Expand Down Expand Up @@ -60,7 +61,7 @@ func Switch(prefix string, mountpoints *mount.Points) (err error) {

log.Println("cleaning up initramfs")

if err = recursiveDelete(int(old.Fd()), "/"); err != nil {
if _, err = recursiveDelete(int(old.Fd()), "/"); err != nil {
return fmt.Errorf("error deleting initramfs: %w", err)
}

Expand All @@ -83,10 +84,10 @@ func Switch(prefix string, mountpoints *mount.Points) (err error) {
return nil
}

func recursiveDelete(fd int, path string) error {
func recursiveDelete(fd int, path string) (preserved bool, err error) {
parentDev, err := getDev(fd)
if err != nil {
return err
return false, err
}

dir := os.NewFile(uintptr(fd), "__ignored__")
Expand All @@ -95,42 +96,57 @@ func recursiveDelete(fd int, path string) error {

names, err := dir.Readdirnames(-1)
if err != nil {
return err
return false, err
}

preserved = false

for _, name := range names {
if err := recusiveDeleteInner(fd, parentDev, name, filepath.Join(path, name)); err != nil {
return err
p, err := recusiveDeleteInner(fd, parentDev, name, filepath.Join(path, name))
if err != nil {
return false, err
}

preserved = preserved || p
}

return nil
return preserved, nil
}

func recusiveDeleteInner(parentFd int, parentDev uint64, childName, path string) error {
if _, preserve := preservedPaths[path]; preserve {
return nil
func recusiveDeleteInner(parentFd int, parentDev uint64, childName, path string) (preserved bool, err error) {
if _, preserved = preservedPaths[path]; preserved {
return preserved, nil
}

childFd, err := unix.Openat(parentFd, childName, unix.O_DIRECTORY|unix.O_NOFOLLOW, unix.O_RDWR)
if err != nil {
return unix.Unlinkat(parentFd, childName, 0)
return false, unix.Unlinkat(parentFd, childName, 0)
}

//nolint:errcheck
defer unix.Close(childFd)

if childFdDev, err := getDev(childFd); err != nil {
return err
var childFdDev uint64

if childFdDev, err = getDev(childFd); err != nil {
return false, err
} else if childFdDev != parentDev {
return nil
return false, nil
}

if err := recursiveDelete(childFd, path); err != nil {
return err
preserved, err = recursiveDelete(childFd, path)
if err != nil {
return false, err
}

return unix.Unlinkat(parentFd, childName, unix.AT_REMOVEDIR)
if preserved {
// some child paths got preserved, skip unlinking the parent
return preserved, nil
}

err = unix.Unlinkat(parentFd, childName, unix.AT_REMOVEDIR)

return false, err
}

func getDev(fd int) (dev uint64, err error) {
Expand Down
3 changes: 3 additions & 0 deletions pkg/machinery/constants/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -592,6 +592,9 @@ const (

// PlatformNetworkConfigFilename is the filename to cache platform network configuration reboots.
PlatformNetworkConfigFilename = "platform-network.yaml"

// FirmwarePath is the path to the standard Linux firmware location.
FirmwarePath = "/lib/firmware"
)

// See https://linux.die.net/man/3/klogctl
Expand Down

0 comments on commit abfb258

Please sign in to comment.