Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

raft: cleanup wal directory if creation fails #10689

Merged
merged 3 commits into from
May 10, 2019
Merged

raft: cleanup wal directory if creation fails #10689

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
51035bf
wal: cleanup wal directory if creation fails
Apr 28, 2019
f7f7e9c
wal: Improve cleanup for robustness and debuggability
May 2, 2019
a0c889d
wal: add a test for wal cleanup, improve comments
joshcc3 May 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 32 additions & 1 deletion wal/wal.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,13 @@ func Create(lg *zap.Logger, dirpath string, metadata []byte) (*WAL, error) {
return nil, err
}

var perr error
defer func() {
if perr != nil {
w.cleanupWAL(lg)
}
}()

// directory was renamed; sync parent dir to persist rename
pdir, perr := fileutil.OpenDir(filepath.Dir(w.dir))
if perr != nil {
Expand All @@ -208,7 +215,7 @@ func Create(lg *zap.Logger, dirpath string, metadata []byte) (*WAL, error) {
}
return nil, perr
}
if perr = pdir.Close(); err != nil {
if perr = pdir.Close(); perr != nil {
if lg != nil {
lg.Warn(
"failed to close the parent data directory file",
Expand All @@ -223,6 +230,30 @@ func Create(lg *zap.Logger, dirpath string, metadata []byte) (*WAL, error) {
return w, nil
}

func (w *WAL) cleanupWAL(lg *zap.Logger) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, wondering, how about adding a test in the https://github.com/etcd-io/etcd/blob/master/wal/wal_test.go

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That would be a good idea, however I'm not sure how to reproduce the case that I encountered (a failing fsync on a writable directory). Are there examples of platform specific tests?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@spzala Any comments on this?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we just need to test if this func works correctly. basically just the rename part.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, something similar to the ones in the wral_test.go that are created for different wral functions.

var err error
if err = w.Close(); err != nil {
if lg != nil {
lg.Panic("failed to close WAL during cleanup", zap.Error(err))
} else {
plog.Panicf("failed to close WAL during cleanup: %v", err)
}
}
brokenDirName := fmt.Sprintf("%s.broken.%v", w.dir, time.Now().Format("20060102.150405.999999"))
if err = os.Rename(w.dir, brokenDirName); err != nil {
if lg != nil {
lg.Panic(
"failed to rename WAL during cleanup",
zap.Error(err),
zap.String("source-path", w.dir),
zap.String("rename-path", brokenDirName),
)
} else {
plog.Panicf("failed to rename WAL during cleanup: %v", err)
}
}
}

func (w *WAL) renameWAL(tmpdirpath string) (*WAL, error) {
if err := os.RemoveAll(w.dir); err != nil {
return nil, err
Expand Down
33 changes: 33 additions & 0 deletions wal/wal_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,15 @@ package wal

import (
"bytes"
"fmt"
"io"
"io/ioutil"
"math"
"os"
"path"
"path/filepath"
"reflect"
"regexp"
"testing"

"go.etcd.io/etcd/v3/pkg/fileutil"
Expand Down Expand Up @@ -101,6 +103,37 @@ func TestCreateFailFromPollutedDir(t *testing.T) {
}
}

func TestWalCleanup(t *testing.T) {
testRoot, err := ioutil.TempDir(os.TempDir(), "waltestroot")
if err != nil {
t.Fatal(err)
}
p, err := ioutil.TempDir(testRoot, "waltest")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(testRoot)

logger := zap.NewExample()
w, err := Create(logger, p, []byte(""))
if err != nil {
t.Fatalf("err = %v, want nil", err)
}
w.cleanupWAL(logger)
fnames, err := fileutil.ReadDir(testRoot)
if err != nil {
t.Fatalf("err = %v, want nil", err)
}
if len(fnames) != 1 {
t.Fatalf("expected 1 file under %v, got %v", testRoot, len(fnames))
}
pattern := fmt.Sprintf(`%s.broken\.[\d]{8}\.[\d]{6}\.[\d]{1,6}?`, filepath.Base(p))
match, _ := regexp.MatchString(pattern, fnames[0])
if !match {
t.Errorf("match = false, expected true for %v with pattern %v", fnames[0], pattern)
}
}

func TestCreateFailFromNoSpaceLeft(t *testing.T) {
p, err := ioutil.TempDir(os.TempDir(), "waltest")
if err != nil {
Expand Down