Skip to content

Commit

Permalink
pre-allocate storage for metadata json files, see containers/podman#1…
Browse files Browse the repository at this point in the history
…3967

Keeping a temporary file of at least the same size as the target file
for atomic writes helps reduce the probability of running out of space
when deleting entities from corresponding metadata in a disk full
scenario. This strategy is applied to writing containers.json,
layers.json, images.json and mountpoints.json.

Signed-off-by: Denys Knertser <[email protected]>
  • Loading branch information
chilikk committed Jan 25, 2023
1 parent bf2534b commit 40551df
Show file tree
Hide file tree
Showing 6 changed files with 118 additions and 28 deletions.
10 changes: 5 additions & 5 deletions containers.go
Original file line number Diff line number Diff line change
Expand Up @@ -541,13 +541,13 @@ func (r *containerStore) save(saveLocations containerLocations) error {
if err != nil {
return err
}
var opts *ioutils.AtomicFileWriterOptions
opts := ioutils.AtomicFileWriterOptions{
PreAllocate: true,
}
if location == volatileContainerLocation {
opts = &ioutils.AtomicFileWriterOptions{
NoSync: true,
}
opts.NoSync = true
}
if err := ioutils.AtomicWriteFileWithOpts(rpath, jdata, 0600, opts); err != nil {
if err := ioutils.AtomicWriteFileWithOpts(rpath, jdata, 0600, &opts); err != nil {
return err
}
}
Expand Down
5 changes: 4 additions & 1 deletion images.go
Original file line number Diff line number Diff line change
Expand Up @@ -536,7 +536,10 @@ func (r *imageStore) Save() error {
if err != nil {
return err
}
if err := ioutils.AtomicWriteFile(rpath, jdata, 0600); err != nil {
opts := ioutils.AtomicFileWriterOptions{
PreAllocate: true,
}
if err := ioutils.AtomicWriteFileWithOpts(rpath, jdata, 0600, &opts); err != nil {
return err
}
lw, err := r.lockfile.RecordWrite()
Expand Down
9 changes: 7 additions & 2 deletions layers.go
Original file line number Diff line number Diff line change
Expand Up @@ -942,7 +942,9 @@ func (r *layerStore) saveLayers(saveLocations layerLocations) error {
if err != nil {
return err
}
opts := ioutils.AtomicFileWriterOptions{}
opts := ioutils.AtomicFileWriterOptions{
PreAllocate: true,
}
if location == volatileLayerLocation {
opts.NoSync = true
}
Expand Down Expand Up @@ -984,7 +986,10 @@ func (r *layerStore) saveMounts() error {
if err != nil {
return err
}
if err = ioutils.AtomicWriteFile(mpath, jmdata, 0600); err != nil {
opts := ioutils.AtomicFileWriterOptions{
PreAllocate: true,
}
if err = ioutils.AtomicWriteFileWithOpts(mpath, jmdata, 0600, &opts); err != nil {
return err
}
lw, err := r.mountsLockfile.RecordWrite()
Expand Down
109 changes: 89 additions & 20 deletions pkg/ioutils/fswriters.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ package ioutils

import (
"io"
"math/rand"
"os"
"path/filepath"
"strconv"
"time"
)

Expand All @@ -17,6 +19,9 @@ type AtomicFileWriterOptions struct {
// On successful return from Close() this is set to the mtime of the
// newly written file.
ModTime time.Time
// Whenever an atomic file is successfully written create a temporary
// file of the same size in order to pre-allocate storage for the next write
PreAllocate bool
}

var defaultWriterOptions = AtomicFileWriterOptions{}
Expand All @@ -38,22 +43,33 @@ func NewAtomicFileWriterWithOpts(filename string, perm os.FileMode, opts *Atomic
// temporary file and closing it atomically changes the temporary file to
// destination path. Writing and closing concurrently is not allowed.
func newAtomicFileWriter(filename string, perm os.FileMode, opts *AtomicFileWriterOptions) (*atomicFileWriter, error) {
f, err := os.CreateTemp(filepath.Dir(filename), ".tmp-"+filepath.Base(filename))
if err != nil {
return nil, err
}
dir := filepath.Dir(filename)
base := filepath.Base(filename)
if opts == nil {
opts = &defaultWriterOptions
}
random := ""
// need predictable name when pre-allocated
if !opts.PreAllocate {
random = strconv.FormatUint(rand.Uint64(), 36)
}
tmp := filepath.Join(dir, ".tmp"+random+"-"+base)
// if pre-allocated the temporary file exists and contains some data
// do not truncate it here, instead truncate at Close()
f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE, perm)
if err != nil {
return nil, err
}
abspath, err := filepath.Abs(filename)
if err != nil {
return nil, err
}
return &atomicFileWriter{
f: f,
fn: abspath,
perm: perm,
noSync: opts.NoSync,
f: f,
fn: abspath,
perm: perm,
noSync: opts.NoSync,
preAllocate: opts.PreAllocate,
}, nil
}

Expand Down Expand Up @@ -91,12 +107,13 @@ func AtomicWriteFile(filename string, data []byte, perm os.FileMode) error {
}

type atomicFileWriter struct {
f *os.File
fn string
writeErr error
perm os.FileMode
noSync bool
modTime time.Time
f *os.File
fn string
writeErr error
perm os.FileMode
noSync bool
preAllocate bool
modTime time.Time
}

func (w *atomicFileWriter) Write(dt []byte) (int, error) {
Expand All @@ -108,22 +125,35 @@ func (w *atomicFileWriter) Write(dt []byte) (int, error) {
}

func (w *atomicFileWriter) Close() (retErr error) {
defer func() {
if retErr != nil || w.writeErr != nil {
os.Remove(w.f.Name())
if !w.preAllocate {
defer func() {
if retErr != nil || w.writeErr != nil {
os.Remove(w.f.Name())
}
}()
} else {
truncateAt, err := w.f.Seek(0, io.SeekCurrent)
if err != nil {
return err
}
err = w.f.Truncate(truncateAt)
if err != nil {
return err
}
}()
}
if !w.noSync {
if err := fdatasync(w.f); err != nil {
w.f.Close()
return err
}
}

var size int64 = 0
// fstat before closing the fd
info, statErr := w.f.Stat()
if statErr == nil {
w.modTime = info.ModTime()
size = info.Size()
}
// We delay error reporting until after the real call to close()
// to match the traditional linux close() behaviour that an fd
Expand All @@ -138,11 +168,50 @@ func (w *atomicFileWriter) Close() (retErr error) {
return statErr
}

if err := os.Chmod(w.f.Name(), w.perm); err != nil {
tmpName := w.f.Name()
if err := os.Chmod(tmpName, w.perm); err != nil {
return err
}
if w.writeErr == nil {
return os.Rename(w.f.Name(), w.fn)
if w.preAllocate {
err := swapOrMove(tmpName, w.fn)
if err != nil {
return err
}
// ignore errors, this is a best effort operation
preAllocate(tmpName, size, w.perm)
return nil
}
return os.Rename(tmpName, w.fn)
}
return nil
}

// ensure that the file is of at least indicated size
func preAllocate(filename string, size int64, perm os.FileMode) error {
f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, perm)
if err != nil {
return err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return err
}
extendBytes := size - info.Size()
if extendBytes > 0 {
var blockSize int64 = 65536
block := make([]byte, blockSize)
for extendBytes > 0 {
if blockSize > extendBytes {
blockSize = extendBytes
}
_, err := f.Write(block[:blockSize])
if err != nil {
return err
}
extendBytes -= blockSize
}
}
return nil
}
Expand Down
9 changes: 9 additions & 0 deletions pkg/ioutils/fswriters_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,12 @@ import (
func fdatasync(f *os.File) error {
return unix.Fdatasync(int(f.Fd()))
}

func swapOrMove(oldpath string, newpath string) error {
err := unix.Renameat2(unix.AT_FDCWD, oldpath, unix.AT_FDCWD, newpath, unix.RENAME_EXCHANGE)
if err != nil {
// unlikely that rename will succeed if renameat2 failed, but just in case
err = os.Rename(oldpath, newpath)
}
return err
}
4 changes: 4 additions & 0 deletions pkg/ioutils/fswriters_unsupported.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,7 @@ import (
func fdatasync(f *os.File) error {
return f.Sync()
}

func swapOrMove(oldpath string, newpath string) error {
return os.Rename(oldpath, newpath)
}

0 comments on commit 40551df

Please sign in to comment.