diff --git a/db_stress_tool/cf_consistency_stress.cc b/db_stress_tool/cf_consistency_stress.cc index 887b557e4850..9eaaab5f01d3 100644 --- a/db_stress_tool/cf_consistency_stress.cc +++ b/db_stress_tool/cf_consistency_stress.cc @@ -359,10 +359,10 @@ class CfConsistencyStressTest : public StressTest { if (fault_fs_guard) { fault_fs_guard->DisableThreadLocalErrorInjection( FaultInjectionIOType::kRead); - fault_fs_guard->DisableThreadLocalErrorInjection( FaultInjectionIOType::kMetadataRead); } + if (s.ok() || s.IsNotFound()) { const bool cmp_found = s.ok(); diff --git a/db_stress_tool/db_stress_common.cc b/db_stress_tool/db_stress_common.cc index 42a3e460733d..968a6c16c0f8 100644 --- a/db_stress_tool/db_stress_common.cc +++ b/db_stress_tool/db_stress_common.cc @@ -19,8 +19,6 @@ ROCKSDB_NAMESPACE::Env* db_stress_listener_env = nullptr; ROCKSDB_NAMESPACE::Env* db_stress_env = nullptr; -// If non-null, injects read error at a rate specified by the -// read_fault_one_in or write_fault_one_in flag std::shared_ptr fault_fs_guard; std::shared_ptr compressed_secondary_cache; std::shared_ptr block_cache; diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index 19e3a6477985..55b6db7e5943 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -2476,13 +2476,19 @@ Status StressTest::TestCheckpoint(ThreadState* thread, std::vector files; // Temporarily disable error injection to print debugging information - fault_fs_guard->DisableThreadLocalErrorInjection( - FaultInjectionIOType::kMetadataRead); + if (fault_fs_guard) { + fault_fs_guard->DisableThreadLocalErrorInjection( + FaultInjectionIOType::kMetadataRead); + } + Status my_s = db_stress_env->GetChildren(checkpoint_dir, &files); + // Enable back disable error injection disabled for printing debugging // information - fault_fs_guard->EnableThreadLocalErrorInjection( - FaultInjectionIOType::kMetadataRead); + if (fault_fs_guard) { + fault_fs_guard->EnableThreadLocalErrorInjection( + FaultInjectionIOType::kMetadataRead); + } assert(my_s.ok()); for (const auto& f : files) { @@ -2964,19 +2970,25 @@ void StressTest::TestCompactRange(ThreadState* thread, int64_t rand_key, uint32_t pre_hash = 0; if (thread->rand.OneIn(2)) { // Temporarily disable error injection to for validation + if (fault_fs_guard) { + fault_fs_guard->DisableThreadLocalErrorInjection( + FaultInjectionIOType::kMetadataRead); + fault_fs_guard->DisableThreadLocalErrorInjection( + FaultInjectionIOType::kRead); + } + // Declare a snapshot and compare the data before and after the compaction - fault_fs_guard->DisableThreadLocalErrorInjection( - FaultInjectionIOType::kMetadataRead); - fault_fs_guard->DisableThreadLocalErrorInjection( - FaultInjectionIOType::kRead); pre_snapshot = db_->GetSnapshot(); pre_hash = GetRangeHash(thread, pre_snapshot, column_family, start_key, end_key); + // Enable back error injection disabled for validation - fault_fs_guard->EnableThreadLocalErrorInjection( - FaultInjectionIOType::kMetadataRead); - fault_fs_guard->EnableThreadLocalErrorInjection( - FaultInjectionIOType::kRead); + if (fault_fs_guard) { + fault_fs_guard->EnableThreadLocalErrorInjection( + FaultInjectionIOType::kMetadataRead); + fault_fs_guard->EnableThreadLocalErrorInjection( + FaultInjectionIOType::kRead); + } } std::ostringstream compact_range_opt_oss; compact_range_opt_oss << "exclusive_manual_compaction: " @@ -3012,11 +3024,12 @@ void StressTest::TestCompactRange(ThreadState* thread, int64_t rand_key, if (pre_snapshot != nullptr) { // Temporarily disable error injection for validation - // Declaring a snapshot and compare the data before and after the compaction - fault_fs_guard->DisableThreadLocalErrorInjection( - FaultInjectionIOType::kMetadataRead); - fault_fs_guard->DisableThreadLocalErrorInjection( - FaultInjectionIOType::kRead); + if (fault_fs_guard) { + fault_fs_guard->DisableThreadLocalErrorInjection( + FaultInjectionIOType::kMetadataRead); + fault_fs_guard->DisableThreadLocalErrorInjection( + FaultInjectionIOType::kRead); + } uint32_t post_hash = GetRangeHash(thread, pre_snapshot, column_family, start_key, end_key); if (pre_hash != post_hash) { @@ -3032,11 +3045,13 @@ void StressTest::TestCompactRange(ThreadState* thread, int64_t rand_key, thread->shared->SetVerificationFailure(); } db_->ReleaseSnapshot(pre_snapshot); - // Enable back error injection disabled for validation - fault_fs_guard->EnableThreadLocalErrorInjection( - FaultInjectionIOType::kMetadataRead); - fault_fs_guard->EnableThreadLocalErrorInjection( - FaultInjectionIOType::kRead); + if (fault_fs_guard) { + // Enable back error injection disabled for validation + fault_fs_guard->EnableThreadLocalErrorInjection( + FaultInjectionIOType::kMetadataRead); + fault_fs_guard->EnableThreadLocalErrorInjection( + FaultInjectionIOType::kRead); + } } } diff --git a/db_stress_tool/no_batched_ops_stress.cc b/db_stress_tool/no_batched_ops_stress.cc index e95352a5e491..7f1cbb72ff7e 100644 --- a/db_stress_tool/no_batched_ops_stress.cc +++ b/db_stress_tool/no_batched_ops_stress.cc @@ -1607,14 +1607,17 @@ class NonBatchedOpsStressTest : public StressTest { if (FLAGS_verify_before_write) { // Temporarily disable error injection for preparation - fault_fs_guard->DisableThreadLocalErrorInjection( - FaultInjectionIOType::kRead); - fault_fs_guard->DisableThreadLocalErrorInjection( - FaultInjectionIOType::kWrite); - fault_fs_guard->DisableThreadLocalErrorInjection( - FaultInjectionIOType::kMetadataRead); - fault_fs_guard->DisableThreadLocalErrorInjection( - FaultInjectionIOType::kMetadataWrite); + if (fault_fs_guard) { + fault_fs_guard->DisableThreadLocalErrorInjection( + FaultInjectionIOType::kRead); + fault_fs_guard->DisableThreadLocalErrorInjection( + FaultInjectionIOType::kWrite); + fault_fs_guard->DisableThreadLocalErrorInjection( + FaultInjectionIOType::kMetadataRead); + fault_fs_guard->DisableThreadLocalErrorInjection( + FaultInjectionIOType::kMetadataWrite); + } + std::string from_db; Status s = db_->Get(read_opts, cfh, k, &from_db); bool res = VerifyOrSyncValue( @@ -1622,14 +1625,16 @@ class NonBatchedOpsStressTest : public StressTest { /* msg_prefix */ "Pre-Put Get verification", from_db, s); // Enable back error injection disabled for preparation - fault_fs_guard->EnableThreadLocalErrorInjection( - FaultInjectionIOType::kRead); - fault_fs_guard->EnableThreadLocalErrorInjection( - FaultInjectionIOType::kWrite); - fault_fs_guard->EnableThreadLocalErrorInjection( - FaultInjectionIOType::kMetadataRead); - fault_fs_guard->EnableThreadLocalErrorInjection( - FaultInjectionIOType::kMetadataWrite); + if (fault_fs_guard) { + fault_fs_guard->EnableThreadLocalErrorInjection( + FaultInjectionIOType::kRead); + fault_fs_guard->EnableThreadLocalErrorInjection( + FaultInjectionIOType::kWrite); + fault_fs_guard->EnableThreadLocalErrorInjection( + FaultInjectionIOType::kMetadataRead); + fault_fs_guard->EnableThreadLocalErrorInjection( + FaultInjectionIOType::kMetadataWrite); + } if (!res) { return s; } @@ -1897,19 +1902,24 @@ class NonBatchedOpsStressTest : public StressTest { std::ostringstream ingest_options_oss; // Temporarily disable error injection for preparation - fault_fs_guard->DisableThreadLocalErrorInjection( - FaultInjectionIOType::kMetadataRead); - fault_fs_guard->DisableThreadLocalErrorInjection( - FaultInjectionIOType::kMetadataWrite); + if (fault_fs_guard) { + fault_fs_guard->DisableThreadLocalErrorInjection( + FaultInjectionIOType::kMetadataRead); + fault_fs_guard->DisableThreadLocalErrorInjection( + FaultInjectionIOType::kMetadataWrite); + } + if (db_stress_env->FileExists(sst_filename).ok()) { // Maybe we terminated abnormally before, so cleanup to give this file // ingestion a clean slate s = db_stress_env->DeleteFile(sst_filename); } - fault_fs_guard->EnableThreadLocalErrorInjection( - FaultInjectionIOType::kMetadataRead); - fault_fs_guard->EnableThreadLocalErrorInjection( - FaultInjectionIOType::kMetadataWrite); + if (fault_fs_guard) { + fault_fs_guard->EnableThreadLocalErrorInjection( + FaultInjectionIOType::kMetadataRead); + fault_fs_guard->EnableThreadLocalErrorInjection( + FaultInjectionIOType::kMetadataWrite); + } SstFileWriter sst_file_writer(EnvOptions(options_), options_); if (s.ok()) { diff --git a/utilities/fault_injection_fs.cc b/utilities/fault_injection_fs.cc index ba451adc0245..cbd2edc4a82e 100644 --- a/utilities/fault_injection_fs.cc +++ b/utilities/fault_injection_fs.cc @@ -1228,8 +1228,7 @@ IOStatus FaultInjectionTestFS::MaybeInjectThreadLocalReadError( ErrorContext* ctx = static_cast(injected_thread_local_read_error_.Get()); if (ctx == nullptr || !ctx->enable_error_injection || !ctx->one_in || - io_activties_exempted_from_fault_injection.find(io_options.io_activity) != - io_activties_exempted_from_fault_injection.end()) { + ShouldIOActivtiesExemptFromFaultInjection(io_options.io_activity)) { return IOStatus::OK(); } @@ -1306,8 +1305,7 @@ IOStatus FaultInjectionTestFS::MaybeInjectThreadLocalError( ErrorContext* ctx = GetErrorContextFromFaultInjectionIOType(type); if (ctx == nullptr || !ctx->enable_error_injection || !ctx->one_in || - io_activties_exempted_from_fault_injection.find(io_options.io_activity) != - io_activties_exempted_from_fault_injection.end()) { + ShouldIOActivtiesExemptFromFaultInjection(io_options.io_activity)) { return IOStatus::OK(); } diff --git a/utilities/fault_injection_fs.h b/utilities/fault_injection_fs.h index 5b4916066584..3456eda77600 100644 --- a/utilities/fault_injection_fs.h +++ b/utilities/fault_injection_fs.h @@ -425,6 +425,12 @@ class FaultInjectionTestFS : public FileSystemWrapper { io_activties_exempted_from_fault_injection = io_activties; } + bool ShouldIOActivtiesExemptFromFaultInjection(Env::IOActivity io_activty) { + MutexLock l(&mutex_); + return io_activties_exempted_from_fault_injection.find(io_activty) != + io_activties_exempted_from_fault_injection.end(); + } + void AssertNoOpenFile() { assert(open_managed_files_.empty()); } IOStatus GetError() { return fs_error_; }