From e67b5ae9a7f7044005c41ed0a22b9a9b6e9ab3e9 Mon Sep 17 00:00:00 2001 From: Viacheslav Biriukov Date: Fri, 4 Jan 2019 17:45:02 +0000 Subject: [PATCH] etcdserver/*: changes to snapshots and wal logic to fix #10219 --- build | 2 +- etcdserver/raft.go | 8 +++--- etcdserver/server.go | 9 ++++++ etcdserver/storage.go | 14 --------- wal/wal.go | 66 ------------------------------------------- 5 files changed, 14 insertions(+), 85 deletions(-) diff --git a/build b/build index 565d0732a73..2d2ee5cf16c 100755 --- a/build +++ b/build @@ -16,7 +16,7 @@ GO_LDFLAGS="$GO_LDFLAGS -X ${REPO_PATH}/version.GitSHA=${GIT_SHA}" toggle_failpoints() { mode="$1" if command -v gofail >/dev/null 2>&1; then - gofail "$mode" etcdserver/ mvcc/backend/ wal/ + gofail "$mode" etcdserver/ mvcc/backend/ elif [[ "$mode" != "disable" ]]; then echo "FAILPOINTS set but gofail not found" exit 1 diff --git a/etcdserver/raft.go b/etcdserver/raft.go index 396a72b6952..9cfacaa6248 100644 --- a/etcdserver/raft.go +++ b/etcdserver/raft.go @@ -240,7 +240,7 @@ func (r *raftNode) start(rh *raftReadyHandler) { if !raft.IsEmptySnap(rd.Snapshot) { // gofail: var raftBeforeSaveSnap struct{} - if err := r.storage.SaveSnapshot(rd.Snapshot); err != nil { + if err := r.storage.SaveSnap(rd.Snapshot); err != nil { if r.lg != nil { r.lg.Fatal("failed to save Raft snapshot", zap.Error(err)) } else { @@ -250,8 +250,8 @@ func (r *raftNode) start(rh *raftReadyHandler) { // gofail: var raftAfterSaveSnap struct{} } - // gofail: var raftBeforeSaveAll struct{} - if err := r.storage.SaveAll(rd.HardState, rd.Entries, rd.Snapshot); err != nil { + // gofail: var raftBeforeSave struct{} + if err := r.storage.Save(rd.HardState, rd.Entries); err != nil { if r.lg != nil { r.lg.Fatal("failed to save Raft hard state and entries", zap.Error(err)) } else { @@ -261,7 +261,7 @@ func (r *raftNode) start(rh *raftReadyHandler) { if !raft.IsEmptyHardState(rd.HardState) { proposalsCommitted.Set(float64(rd.HardState.Commit)) } - // gofail: var raftAfterSaveAll struct{} + // gofail: var raftAfterSave struct{} if !raft.IsEmptySnap(rd.Snapshot) { // etcdserver now claim the snapshot has been persisted onto the disk diff --git a/etcdserver/server.go b/etcdserver/server.go index 5af566b06f0..d0c9a925630 100644 --- a/etcdserver/server.go +++ b/etcdserver/server.go @@ -2188,6 +2188,15 @@ func (s *EtcdServer) snapshot(snapi uint64, confState raftpb.ConfState) { plog.Fatalf("save snapshot error: %v", err) } } + + if err = s.r.storage.Release(snap); err != nil { + if lg != nil { + lg.Panic("failed to release wal", zap.Error(err)) + } else { + plog.Fatalf("failed to release wal error: %v", err) + } + } + if lg != nil { lg.Info( "saved snapshot", diff --git a/etcdserver/storage.go b/etcdserver/storage.go index 401e162a4c6..29a43691547 100644 --- a/etcdserver/storage.go +++ b/etcdserver/storage.go @@ -36,12 +36,6 @@ type Storage interface { SaveSnap(snap raftpb.Snapshot) error // Close closes the Storage and performs finalization. Close() error - - // SaveSnapshot function saves only snapshot to the underlying stable storage. - SaveSnapshot(snap raftpb.Snapshot) error - // SaveAll function saves ents, snapshot and state to the underlying stable storage. - // SaveAll MUST block until st and ents are on stable storage. - SaveAll(st raftpb.HardState, ents []raftpb.Entry, snap raftpb.Snapshot) error // Release release release the locked wal files since they will not be used. Release(snap raftpb.Snapshot) error } @@ -66,15 +60,7 @@ func (st *storage) SaveSnap(snap raftpb.Snapshot) error { if err != nil { return err } - err = st.Snapshotter.SaveSnap(snap) - if err != nil { - return err - } - return st.WAL.ReleaseLockTo(snap.Metadata.Index) -} -// SaveSnapshot saves the snapshot to disk. -func (st *storage) SaveSnapshot(snap raftpb.Snapshot) error { return st.Snapshotter.SaveSnap(snap) } diff --git a/wal/wal.go b/wal/wal.go index d86a9fea362..7200ad088dd 100644 --- a/wal/wal.go +++ b/wal/wal.go @@ -746,72 +746,6 @@ func (w *WAL) SaveSnapshot(e walpb.Snapshot) error { return w.sync() } -func (w *WAL) SaveAll(st raftpb.HardState, ents []raftpb.Entry, snap raftpb.Snapshot) error { - w.mu.Lock() - defer w.mu.Unlock() - - // short cut, do not call sync - if raft.IsEmptyHardState(st) && len(ents) == 0 && raft.IsEmptySnap(snap) { - return nil - } - - mustSync := raft.MustSync(st, w.state, len(ents)) - - if !raft.IsEmptySnap(snap) { - mustSync = true - } - - // 1. Save entries - // TODO(xiangli): no more reference operator - for i := range ents { - if err := w.saveEntry(&ents[i]); err != nil { - return err - } - // gofail: var raftAfterSaveWALFirstEntry struct{} - } - // gofail: var raftAfterSaveWALEntries struct{} - - // 2. Save snapshot - if !raft.IsEmptySnap(snap) { - e := walpb.Snapshot{ - Index: snap.Metadata.Index, - Term: snap.Metadata.Term, - } - - b := pbutil.MustMarshal(&e) - - rec := &walpb.Record{Type: snapshotType, Data: b} - if err := w.encoder.encode(rec); err != nil { - return err - } - - // update enti only when snapshot is ahead of last index - if w.enti < e.Index { - w.enti = e.Index - } - // gofail: var raftAfterSaveWALSnap struct{} - } - - // 3. Save HardState - if err := w.saveState(&st); err != nil { - return err - } - // gofail: var raftAfterSaveWALState struct{} - - curOff, err := w.tail().Seek(0, io.SeekCurrent) - if err != nil { - return err - } - if curOff < SegmentSizeBytes { - if mustSync { - return w.sync() - } - return nil - } - - return w.cut() -} - func (w *WAL) saveCrc(prevCrc uint32) error { return w.encoder.encode(&walpb.Record{Type: crcType, Crc: prevCrc}) }