Skip to content

Commit

Permalink
fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@hx235
hx235 committed Jun 24, 2024
1 parent 748f74a commit c58edb6
Show file tree
Hide file tree
Showing 6 changed files with 80 additions and 53 deletions.
2 changes: 1 addition & 1 deletion db_stress_tool/cf_consistency_stress.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -359,10 +359,10 @@ class CfConsistencyStressTest : public StressTest {
if (fault_fs_guard) {
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kRead);

fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
}

if (s.ok() || s.IsNotFound()) {
const bool cmp_found = s.ok();

Expand Down
2 changes: 0 additions & 2 deletions db_stress_tool/db_stress_common.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@

ROCKSDB_NAMESPACE::Env* db_stress_listener_env = nullptr;
ROCKSDB_NAMESPACE::Env* db_stress_env = nullptr;
// If non-null, injects read error at a rate specified by the
// read_fault_one_in or write_fault_one_in flag
std::shared_ptr<ROCKSDB_NAMESPACE::FaultInjectionTestFS> fault_fs_guard;
std::shared_ptr<ROCKSDB_NAMESPACE::SecondaryCache> compressed_secondary_cache;
std::shared_ptr<ROCKSDB_NAMESPACE::Cache> block_cache;
Expand Down
59 changes: 37 additions & 22 deletions db_stress_tool/db_stress_test_base.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2476,13 +2476,19 @@ Status StressTest::TestCheckpoint(ThreadState* thread,
std::vector<std::string> files;

// Temporarily disable error injection to print debugging information
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
if (fault_fs_guard) {
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
}

Status my_s = db_stress_env->GetChildren(checkpoint_dir, &files);

// Enable back disable error injection disabled for printing debugging
// information
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
if (fault_fs_guard) {
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
}
assert(my_s.ok());

for (const auto& f : files) {
Expand Down Expand Up @@ -2964,19 +2970,25 @@ void StressTest::TestCompactRange(ThreadState* thread, int64_t rand_key,
uint32_t pre_hash = 0;
if (thread->rand.OneIn(2)) {
// Temporarily disable error injection to for validation
if (fault_fs_guard) {
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kRead);
}

// Declare a snapshot and compare the data before and after the compaction
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kRead);
pre_snapshot = db_->GetSnapshot();
pre_hash =
GetRangeHash(thread, pre_snapshot, column_family, start_key, end_key);

// Enable back error injection disabled for validation
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kRead);
if (fault_fs_guard) {
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kRead);
}
}
std::ostringstream compact_range_opt_oss;
compact_range_opt_oss << "exclusive_manual_compaction: "
Expand Down Expand Up @@ -3012,11 +3024,12 @@ void StressTest::TestCompactRange(ThreadState* thread, int64_t rand_key,

if (pre_snapshot != nullptr) {
// Temporarily disable error injection for validation
// Declaring a snapshot and compare the data before and after the compaction
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kRead);
if (fault_fs_guard) {
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kRead);
}
uint32_t post_hash =
GetRangeHash(thread, pre_snapshot, column_family, start_key, end_key);
if (pre_hash != post_hash) {
Expand All @@ -3032,11 +3045,13 @@ void StressTest::TestCompactRange(ThreadState* thread, int64_t rand_key,
thread->shared->SetVerificationFailure();
}
db_->ReleaseSnapshot(pre_snapshot);
// Enable back error injection disabled for validation
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kRead);
if (fault_fs_guard) {
// Enable back error injection disabled for validation
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kRead);
}
}
}

Expand Down
58 changes: 34 additions & 24 deletions db_stress_tool/no_batched_ops_stress.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1607,29 +1607,34 @@ class NonBatchedOpsStressTest : public StressTest {

if (FLAGS_verify_before_write) {
// Temporarily disable error injection for preparation
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kRead);
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kWrite);
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataWrite);
if (fault_fs_guard) {
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kRead);
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kWrite);
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataWrite);
}

std::string from_db;
Status s = db_->Get(read_opts, cfh, k, &from_db);
bool res = VerifyOrSyncValue(
rand_column_family, rand_key, read_opts, shared,
/* msg_prefix */ "Pre-Put Get verification", from_db, s);

// Enable back error injection disabled for preparation
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kRead);
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kWrite);
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataWrite);
if (fault_fs_guard) {
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kRead);
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kWrite);
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataWrite);
}
if (!res) {
return s;
}
Expand Down Expand Up @@ -1897,19 +1902,24 @@ class NonBatchedOpsStressTest : public StressTest {
std::ostringstream ingest_options_oss;

// Temporarily disable error injection for preparation
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataWrite);
if (fault_fs_guard) {
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->DisableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataWrite);
}

if (db_stress_env->FileExists(sst_filename).ok()) {
// Maybe we terminated abnormally before, so cleanup to give this file
// ingestion a clean slate
s = db_stress_env->DeleteFile(sst_filename);
}
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataWrite);
if (fault_fs_guard) {
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataRead);
fault_fs_guard->EnableThreadLocalErrorInjection(
FaultInjectionIOType::kMetadataWrite);
}

SstFileWriter sst_file_writer(EnvOptions(options_), options_);
if (s.ok()) {
Expand Down
6 changes: 2 additions & 4 deletions utilities/fault_injection_fs.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1228,8 +1228,7 @@ IOStatus FaultInjectionTestFS::MaybeInjectThreadLocalReadError(
ErrorContext* ctx =
static_cast<ErrorContext*>(injected_thread_local_read_error_.Get());
if (ctx == nullptr || !ctx->enable_error_injection || !ctx->one_in ||
io_activties_exempted_from_fault_injection.find(io_options.io_activity) !=
io_activties_exempted_from_fault_injection.end()) {
ShouldIOActivtiesExemptFromFaultInjection(io_options.io_activity)) {
return IOStatus::OK();
}

Expand Down Expand Up @@ -1306,8 +1305,7 @@ IOStatus FaultInjectionTestFS::MaybeInjectThreadLocalError(

ErrorContext* ctx = GetErrorContextFromFaultInjectionIOType(type);
if (ctx == nullptr || !ctx->enable_error_injection || !ctx->one_in ||
io_activties_exempted_from_fault_injection.find(io_options.io_activity) !=
io_activties_exempted_from_fault_injection.end()) {
ShouldIOActivtiesExemptFromFaultInjection(io_options.io_activity)) {
return IOStatus::OK();
}

Expand Down
6 changes: 6 additions & 0 deletions utilities/fault_injection_fs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,12 @@ class FaultInjectionTestFS : public FileSystemWrapper {
io_activties_exempted_from_fault_injection = io_activties;
}

bool ShouldIOActivtiesExemptFromFaultInjection(Env::IOActivity io_activty) {
MutexLock l(&mutex_);
return io_activties_exempted_from_fault_injection.find(io_activty) !=
io_activties_exempted_from_fault_injection.end();
}

void AssertNoOpenFile() { assert(open_managed_files_.empty()); }

IOStatus GetError() { return fs_error_; }
Expand Down

0 comments on commit c58edb6

Please sign in to comment.