Atomize RenameFile in KeyManagedEncryptedEnv (facebook#7)

apache/incubator-pegasus#1575

Cherry-pick from
tikv@93e89a5

fix bug: tikv/tikv#9115

Summary: we need to update encryption metadata via
encryption::DataKeyManager, which cannot combine with the actual file
operation into one atomic operation. In RenameFile, when the src_file
has been removed, power is off, then we may lost the file info of
src_file next restart.

Signed-off-by: Xintao [hunterlxt@live.com](mailto:hunterlxt@live.com)

Signed-off-by: Xintao <hunterlxt@live.com>
Co-authored-by: Xintao <hunterlxt@live.com>

db/db_test_util.h

@@ -85,9 +85,6 @@ class TestKeyManager : public encryption::KeyManager {
   Status LinkFile(const std::string&, const std::string&) override {
     return Status::OK();
   }
-  Status RenameFile(const std::string&, const std::string&) override {
-    return Status::OK();
-  }
 };
 #endif
 

encryption/encryption.cc

@@ -382,9 +382,14 @@ Status KeyManagedEncryptedEnv::ReuseWritableFile(
           "Unsupported encryption method: " +
           std::to_string(static_cast<int>(file_info.method)));
   }
-  if (s.ok()) {
-    s = key_manager_->RenameFile(old_fname, fname);
+  if (!s.ok()) {
+    return s;
+  }
+  s = key_manager_->LinkFile(old_fname, fname);
+  if (!s.ok()) {
+    return s;
   }
+  s = key_manager_->DeleteFile(old_fname);
   return s;
 }
 
@@ -437,22 +442,28 @@ Status KeyManagedEncryptedEnv::LinkFile(const std::string& src_fname,
   }
   s = target()->LinkFile(src_fname, dst_fname);
   if (!s.ok()) {
-    // Ignore error
-    key_manager_->DeleteFile(dst_fname);
+    Status delete_status __attribute__((__unused__)) =
+        key_manager_->DeleteFile(dst_fname);
+    assert(delete_status.ok());
   }
   return s;
 }
 
 Status KeyManagedEncryptedEnv::RenameFile(const std::string& src_fname,
                                           const std::string& dst_fname) {
-  Status s = key_manager_->RenameFile(src_fname, dst_fname);
+  // Link(copy)File instead of RenameFile to avoid losing src_fname info when
+  // failed to rename the src_fname in the file system.
+  Status s = key_manager_->LinkFile(src_fname, dst_fname);
   if (!s.ok()) {
     return s;
   }
   s = target()->RenameFile(src_fname, dst_fname);
-  if (!s.ok()) {
-    // Ignore error
-    key_manager_->RenameFile(dst_fname, src_fname);
+  if (s.ok()) {
+    s = key_manager_->DeleteFile(src_fname);
+  } else {
+    Status delete_status __attribute__((__unused__)) =
+        key_manager_->DeleteFile(dst_fname);
+    assert(delete_status.ok());
   }
   return s;
 }

encryption/in_memory_key_manager.h

@@ -70,17 +70,6 @@ class InMemoryKeyManager final : public KeyManager {
     return Status::OK();
   }
 
-  Status RenameFile(const std::string& src_fname,
-                    const std::string& dst_fname) override {
-    MutexLock l(&mu_);
-    if (files_.count(src_fname) == 0) {
-      return Status::Corruption("File not found: " + src_fname);
-    }
-    files_[dst_fname] = files_[src_fname];
-    files_.erase(src_fname);
-    return Status::OK();
-  }
-
  private:
   mutable port::Mutex mu_;
   Random rnd_;

include/rocksdb/encryption.h

@@ -55,8 +55,6 @@ class KeyManager {
   virtual Status DeleteFile(const std::string& fname) = 0;
   virtual Status LinkFile(const std::string& src_fname,
                           const std::string& dst_fname) = 0;
-  virtual Status RenameFile(const std::string& src_fname,
-                            const std::string& dst_fname) = 0;
 };
 
 // An Env with underlying files being encrypted. It holds a reference to an