diff --git a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp index 5cd0cddc39b4..7dc2fd48dd71 100644 --- a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp @@ -384,10 +384,10 @@ void Fuzzer::SetMaxMutationLen(size_t MaxMutationLen) { void Fuzzer::CheckExitOnSrcPosOrItem() { if (!Options.ExitOnSrcPos.empty()) { static auto *PCsSet = new Set; - auto HandlePC = [&](uintptr_t PC) { - if (!PCsSet->insert(PC).second) + auto HandlePC = [&](const TracePC::PCTableEntry *TE) { + if (!PCsSet->insert(TE->PC).second) return; - std::string Descr = DescribePC("%F %L", PC + 1); + std::string Descr = DescribePC("%F %L", TE->PC + 1); if (Descr.find(Options.ExitOnSrcPos) != std::string::npos) { Printf("INFO: found line matching '%s', exiting.\n", Options.ExitOnSrcPos.c_str()); diff --git a/compiler-rt/lib/fuzzer/FuzzerMerge.cpp b/compiler-rt/lib/fuzzer/FuzzerMerge.cpp index 556a231f11b0..55e5c9ad3545 100644 --- a/compiler-rt/lib/fuzzer/FuzzerMerge.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerMerge.cpp @@ -42,10 +42,12 @@ void Merger::ParseOrExit(std::istream &IS, bool ParseCoverage) { // file1 // file2 # One file name per line. // STARTED 0 123 # FileID, file size -// DONE 0 1 4 6 8 # FileID COV1 COV2 ... -// STARTED 1 456 # If DONE is missing, the input crashed while processing. +// FT 0 1 4 6 8 # FileID COV1 COV2 ... +// COV 0 7 8 9 # FileID COV1 COV1 +// STARTED 1 456 # If FT is missing, the input crashed while processing. // STARTED 2 567 -// DONE 2 8 9 +// FT 2 8 9 +// COV 2 11 12 bool Merger::Parse(std::istream &IS, bool ParseCoverage) { LastFailure.clear(); std::string Line; @@ -70,11 +72,12 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { if (!std::getline(IS, Files[i].Name, '\n')) return false; - // Parse STARTED and DONE lines. + // Parse STARTED, FT, and COV lines. size_t ExpectedStartMarker = 0; const size_t kInvalidStartMarker = -1; size_t LastSeenStartMarker = kInvalidStartMarker; Vector TmpFeatures; + Set PCs; while (std::getline(IS, Line, '\n')) { std::istringstream ISS1(Line); std::string Marker; @@ -89,8 +92,8 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { LastSeenStartMarker = ExpectedStartMarker; assert(ExpectedStartMarker < Files.size()); ExpectedStartMarker++; - } else if (Marker == "DONE") { - // DONE FILE_ID COV1 COV2 COV3 ... + } else if (Marker == "FT") { + // FT FILE_ID COV1 COV2 COV3 ... size_t CurrentFileIdx = N; if (CurrentFileIdx != LastSeenStartMarker) return false; @@ -102,6 +105,11 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { std::sort(TmpFeatures.begin(), TmpFeatures.end()); Files[CurrentFileIdx].Features = TmpFeatures; } + } else if (Marker == "COV") { + if (ParseCoverage) + while (ISS1 >> std::hex >> N) + if (PCs.insert(N).second) + NumCoveredPCs++; } else { return false; } @@ -199,6 +207,7 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app); Set AllFeatures; + Set AllPCs; for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) { Fuzzer::MaybeExitGracefully(); auto U = FileToVector(M.Files[i].Name); @@ -223,16 +232,24 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { if (AllFeatures.insert(Feature).second) UniqFeatures.insert(Feature); }); + TPC.UpdateObservedPCs(); // Show stats. if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1))) PrintStats("pulse "); // Write the post-run marker and the coverage. - OF << "DONE " << i; + OF << "FT " << i; for (size_t F : UniqFeatures) OF << " " << std::hex << F; OF << "\n"; + OF << "COV " << i; + TPC.ForEachObservedPC([&](const TracePC::PCTableEntry *TE) { + if (AllPCs.insert(TE).second) + OF << " " << TPC.PCTableEntryIdx(TE); + }); + OF << "\n"; OF.flush(); } + PrintStats("DONE "); } static void WriteNewControlFile(const std::string &CFPath, diff --git a/compiler-rt/lib/fuzzer/FuzzerMerge.h b/compiler-rt/lib/fuzzer/FuzzerMerge.h index 065dbd44a072..157611cb9395 100644 --- a/compiler-rt/lib/fuzzer/FuzzerMerge.h +++ b/compiler-rt/lib/fuzzer/FuzzerMerge.h @@ -56,6 +56,7 @@ struct MergeFileInfo { struct Merger { Vector Files; + size_t NumCoveredPCs = 0; size_t NumFilesInFirstCorpus = 0; size_t FirstNotProcessedFile = 0; std::string LastFailure; diff --git a/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp b/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp index 240d76fb61e2..f5fdbf5bd0d6 100644 --- a/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp @@ -187,18 +187,19 @@ inline ALWAYS_INLINE uintptr_t GetNextInstructionPc(uintptr_t PC) { void TracePC::UpdateObservedPCs() { Vector CoveredFuncs; - auto ObservePC = [&](uintptr_t PC) { - if (ObservedPCs.insert(PC).second && DoPrintNewPCs) { - PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", GetNextInstructionPc(PC)); + auto ObservePC = [&](const PCTableEntry *TE) { + if (ObservedPCs.insert(TE).second && DoPrintNewPCs) { + PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", + GetNextInstructionPc(TE->PC)); Printf("\n"); } }; - auto Observe = [&](const PCTableEntry &TE) { - if (TE.PCFlags & 1) - if (++ObservedFuncs[TE.PC] == 1 && NumPrintNewFuncs) - CoveredFuncs.push_back(TE.PC); - ObservePC(TE.PC); + auto Observe = [&](const PCTableEntry *TE) { + if (TE->PCFlags & 1) + if (++ObservedFuncs[TE->PC] == 1 && NumPrintNewFuncs) + CoveredFuncs.push_back(TE->PC); + ObservePC(TE); }; if (NumPCsInPCTables) { @@ -212,7 +213,7 @@ void TracePC::UpdateObservedPCs() { if (!R.Enabled) continue; for (uint8_t *P = R.Start; P < R.Stop; P++) if (*P) - Observe(ModulePCTable[i].Start[M.Idx(P)]); + Observe(&ModulePCTable[i].Start[M.Idx(P)]); } } } @@ -226,6 +227,17 @@ void TracePC::UpdateObservedPCs() { } } +uintptr_t TracePC::PCTableEntryIdx(const PCTableEntry *TE) { + size_t TotalTEs = 0; + for (size_t i = 0; i < NumPCTables; i++) { + auto &M = ModulePCTable[i]; + if (TE >= M.Start && TE < M.Stop) + return TotalTEs + TE - M.Start; + TotalTEs += M.Stop - M.Start; + } + assert(0); + return 0; +} static std::string GetModuleName(uintptr_t PC) { char ModulePathRaw[4096] = ""; // What's PATH_MAX in portable C++? @@ -303,7 +315,7 @@ void TracePC::PrintCoverage() { size_t NumEdges = Last - First; Vector UncoveredPCs; for (auto TE = First; TE < Last; TE++) - if (!ObservedPCs.count(TE->PC)) + if (!ObservedPCs.count(TE)) UncoveredPCs.push_back(TE->PC); Printf("%sCOVERED_FUNC: hits: %zd", Counter ? "" : "UN", Counter); Printf(" edges: %zd/%zd", NumEdges - UncoveredPCs.size(), NumEdges); diff --git a/compiler-rt/lib/fuzzer/FuzzerTracePC.h b/compiler-rt/lib/fuzzer/FuzzerTracePC.h index 49514b92e765..f1d523e33483 100644 --- a/compiler-rt/lib/fuzzer/FuzzerTracePC.h +++ b/compiler-rt/lib/fuzzer/FuzzerTracePC.h @@ -122,6 +122,12 @@ class TracePC { void ProtectLazyCounters(); bool UnprotectLazyCounters(void *CounterPtr); + struct PCTableEntry { + uintptr_t PC, PCFlags; + }; + + uintptr_t PCTableEntryIdx(const PCTableEntry *TE); + private: bool UseCounters = false; uint32_t UseValueProfileMask = false; @@ -159,16 +165,11 @@ private: CB(Modules[m].Regions[r]); } - - struct PCTableEntry { - uintptr_t PC, PCFlags; - }; - struct { const PCTableEntry *Start, *Stop; } ModulePCTable[4096]; size_t NumPCTables; size_t NumPCsInPCTables; - Set ObservedPCs; + Set ObservedPCs; std::unordered_map ObservedFuncs; // PC => Counter. uint8_t *FocusFunctionCounterPtr = nullptr; diff --git a/compiler-rt/lib/fuzzer/tests/FuzzerUnittest.cpp b/compiler-rt/lib/fuzzer/tests/FuzzerUnittest.cpp index 46abc156fa33..f3fee9579db2 100644 --- a/compiler-rt/lib/fuzzer/tests/FuzzerUnittest.cpp +++ b/compiler-rt/lib/fuzzer/tests/FuzzerUnittest.cpp @@ -619,7 +619,7 @@ TEST(Merge, Bad) { "2\n2\nA\n", "2\n2\nA\nB\nC\n", "0\n0\n", - "1\n1\nA\nDONE 0", + "1\n1\nA\nFT 0", "1\n1\nA\nSTARTED 1", }; Merger M; @@ -670,9 +670,9 @@ TEST(Merge, Good) { EXPECT_TRUE(M.Parse("3\n1\nAA\nBB\nC\n" "STARTED 0 1000\n" - "DONE 0 1 2 3\n" + "FT 0 1 2 3\n" "STARTED 1 1001\n" - "DONE 1 4 5 6 \n" + "FT 1 4 5 6 \n" "STARTED 2 1002\n" "", true)); EXPECT_EQ(M.Files.size(), 3U); @@ -693,9 +693,9 @@ TEST(Merge, Good) { Set NewFeatures; EXPECT_TRUE(M.Parse("3\n2\nAA\nBB\nC\n" - "STARTED 0 1000\nDONE 0 1 2 3\n" - "STARTED 1 1001\nDONE 1 4 5 6 \n" - "STARTED 2 1002\nDONE 2 6 1 3 \n" + "STARTED 0 1000\nFT 0 1 2 3\n" + "STARTED 1 1001\nFT 1 4 5 6 \n" + "STARTED 2 1002\nFT 2 6 1 3 \n" "", true)); EXPECT_EQ(M.Files.size(), 3U); EXPECT_EQ(M.NumFilesInFirstCorpus, 2U); @@ -708,9 +708,9 @@ TEST(Merge, Good) { EQ(NewFiles, {}); EXPECT_TRUE(M.Parse("3\n1\nA\nB\nC\n" - "STARTED 0 1000\nDONE 0 1 2 3\n" - "STARTED 1 1001\nDONE 1 4 5 6 \n" - "STARTED 2 1002\nDONE 2 6 1 3\n" + "STARTED 0 1000\nFT 0 1 2 3\n" + "STARTED 1 1001\nFT 1 4 5 6 \n" + "STARTED 2 1002\nFT 2 6 1 3\n" "", true)); EQ(M.Files[0].Features, {1, 2, 3}); EQ(M.Files[1].Features, {4, 5, 6}); @@ -720,8 +720,8 @@ TEST(Merge, Good) { // Same as the above, but with InitialFeatures. EXPECT_TRUE(M.Parse("2\n0\nB\nC\n" - "STARTED 0 1001\nDONE 0 4 5 6 \n" - "STARTED 1 1002\nDONE 1 6 1 3\n" + "STARTED 0 1001\nFT 0 4 5 6 \n" + "STARTED 1 1002\nFT 1 6 1 3\n" "", true)); EQ(M.Files[0].Features, {4, 5, 6}); EQ(M.Files[1].Features, {1, 3, 6}); @@ -736,29 +736,29 @@ TEST(Merge, Good) { TEST(Merge, Merge) { Merge("3\n1\nA\nB\nC\n" - "STARTED 0 1000\nDONE 0 1 2 3\n" - "STARTED 1 1001\nDONE 1 4 5 6 \n" - "STARTED 2 1002\nDONE 2 6 1 3 \n", + "STARTED 0 1000\nFT 0 1 2 3\n" + "STARTED 1 1001\nFT 1 4 5 6 \n" + "STARTED 2 1002\nFT 2 6 1 3 \n", {"B"}, 3); Merge("3\n0\nA\nB\nC\n" - "STARTED 0 2000\nDONE 0 1 2 3\n" - "STARTED 1 1001\nDONE 1 4 5 6 \n" - "STARTED 2 1002\nDONE 2 6 1 3 \n", + "STARTED 0 2000\nFT 0 1 2 3\n" + "STARTED 1 1001\nFT 1 4 5 6 \n" + "STARTED 2 1002\nFT 2 6 1 3 \n", {"A", "B", "C"}, 6); Merge("4\n0\nA\nB\nC\nD\n" - "STARTED 0 2000\nDONE 0 1 2 3\n" - "STARTED 1 1101\nDONE 1 4 5 6 \n" - "STARTED 2 1102\nDONE 2 6 1 3 100 \n" - "STARTED 3 1000\nDONE 3 1 \n", + "STARTED 0 2000\nFT 0 1 2 3\n" + "STARTED 1 1101\nFT 1 4 5 6 \n" + "STARTED 2 1102\nFT 2 6 1 3 100 \n" + "STARTED 3 1000\nFT 3 1 \n", {"A", "B", "C", "D"}, 7); Merge("4\n1\nA\nB\nC\nD\n" - "STARTED 0 2000\nDONE 0 4 5 6 7 8\n" - "STARTED 1 1100\nDONE 1 1 2 3 \n" - "STARTED 2 1100\nDONE 2 2 3 \n" - "STARTED 3 1000\nDONE 3 1 \n", + "STARTED 0 2000\nFT 0 4 5 6 7 8\n" + "STARTED 1 1100\nFT 1 1 2 3 \n" + "STARTED 2 1100\nFT 2 2 3 \n" + "STARTED 3 1000\nFT 3 1 \n", {"B", "D"}, 3); } diff --git a/compiler-rt/test/fuzzer/merge-control-file.test b/compiler-rt/test/fuzzer/merge-control-file.test index 0fed37565333..f269bed0ef52 100644 --- a/compiler-rt/test/fuzzer/merge-control-file.test +++ b/compiler-rt/test/fuzzer/merge-control-file.test @@ -32,9 +32,9 @@ OK_0: MERGE-OUTER: 3 new files with {{.*}} new features added RUN: rm -f %t/T1/*; cp %t/T0/* %t/T1 RUN: echo 3 > %t/MCF; echo 0 >> %t/MCF; echo %t/T1/1 >> %t/MCF; echo %t/T1/2 >> %t/MCF; echo %t/T1/3 >> %t/MCF RUN: echo STARTED 0 1 >> %t/MCF -RUN: echo DONE 0 11 >> %t/MCF +RUN: echo FT 0 11 >> %t/MCF RUN: echo STARTED 1 2 >> %t/MCF -RUN: echo DONE 1 12 >> %t/MCF +RUN: echo FT 1 12 >> %t/MCF RUN: %run %t/T.exe -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=OK_2 OK_2: MERGE-OUTER: control file ok, 3 files total, first not processed file 2 OK_2: MERGE-OUTER: 3 new files with {{.*}} new features added @@ -42,10 +42,10 @@ OK_2: MERGE-OUTER: 3 new files with {{.*}} new features added RUN: rm -f %t/T1/*; cp %t/T0/* %t/T1 RUN: echo 3 > %t/MCF; echo 0 >> %t/MCF; echo %t/T1/1 >> %t/MCF; echo %t/T1/2 >> %t/MCF; echo %t/T1/3 >> %t/MCF RUN: echo STARTED 0 1 >> %t/MCF -RUN: echo DONE 0 11 >> %t/MCF +RUN: echo FT 0 11 >> %t/MCF RUN: echo STARTED 1 2 >> %t/MCF -RUN: echo DONE 1 12 >> %t/MCF +RUN: echo FT 1 12 >> %t/MCF RUN: echo STARTED 2 2 >> %t/MCF -RUN: echo DONE 2 13 >> %t/MCF +RUN: echo FT 2 13 >> %t/MCF RUN: %run %t/T.exe -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=OK_3 OK_3: MERGE-OUTER: nothing to do, merge has been completed before diff --git a/compiler-rt/test/fuzzer/merge-sigusr.test b/compiler-rt/test/fuzzer/merge-sigusr.test index 44448ca29e63..1b16d3ccf835 100644 --- a/compiler-rt/test/fuzzer/merge-sigusr.test +++ b/compiler-rt/test/fuzzer/merge-sigusr.test @@ -22,7 +22,8 @@ RUN: sleep 3 RUN: cat %t/log | FileCheck %s RUN: grep C2/g %t/MCF RUN: grep STARTED %t/MCF -RUN: tail -n 1 %t/MCF | grep DONE +RUN: tail -n 2 %t/MCF | grep FT +RUN: tail -n 1 %t/MCF | grep COV CHECK: INFO: signal received, trying to exit gracefully CHECK: INFO: libFuzzer: exiting as requested diff --git a/compiler-rt/test/fuzzer/merge.test b/compiler-rt/test/fuzzer/merge.test index 5e97d006d4bf..c003df282354 100644 --- a/compiler-rt/test/fuzzer/merge.test +++ b/compiler-rt/test/fuzzer/merge.test @@ -44,7 +44,7 @@ RUN: cp %t/T0/* %t/T1/ RUN: rm -f %t/MCF RUN: %run %t-FullCoverageSetTest -merge=1 -merge_control_file=%t/MCF %t/T1 %t/T2 2>&1 | FileCheck %s --check-prefix=MCF RUN: grep STARTED %t/MCF -RUN: grep DONE %t/MCF +RUN: grep FT %t/MCF MCF: MERGE-INNER: using the control file {{.*}}MCF MCF: MERGE-OUTER: 3 new files