[verify] Add support for location markers in directives.

A marker (matching /#[A-Za-z0-9_-]/) is specified by attaching a comment
containing the marker to the line at which the diagnostic is expected,
and then can be referenced from an expected-* directive after an @:

  foo // #1
  // expected-error@#1 {{undeclared identifier 'foo'}}

The intent is for markers to be used in situations where relative line
numbers are currently used, to avoid the need to renumber when the test
case is rearranged.

llvm-svn: 358326

clang/include/clang/Basic/DiagnosticFrontendKinds.td

@@ -121,6 +121,12 @@ def err_verify_missing_file : Error<
     "file '%0' could not be located in expected %1">;
 def err_verify_invalid_range : Error<
     "invalid range following '-' in expected %0">;
+def err_verify_ambiguous_marker : Error<
+    "reference to marker '%0' is ambiguous">;
+def note_verify_ambiguous_marker : Note<
+    "ambiguous marker '%0' is defined here">;
+def err_verify_no_such_marker : Error<
+    "use of undefined marker '%0'">;
 def err_verify_missing_start : Error<
     "cannot find start ('{{') of expected %0">;
 def err_verify_missing_end : Error<

clang/include/clang/Frontend/VerifyDiagnosticConsumer.h

@@ -81,6 +81,19 @@ class TextDiagnosticBuffer;
 /// the included file is, for example, a system header where the actual line
 /// number may change and is not critical).
 ///
+/// As an alternative to specifying a fixed line number, the location of a
+/// diagnostic can instead be indicated by a marker of the form "#<marker>".
+/// Markers are specified by including them in a comment, and then referenced
+/// by appending the marker to the diagnostic with "@#<marker>":
+///
+/// \code
+///   #warning some text  // #1
+///   // expected-warning@#1 {{some text}}
+/// \endcode
+///
+/// The name of a marker used in a directive must be unique within the
+/// compilation.
+///
 /// The simple syntax above allows each specification to match exactly one
 /// error.  You can use the extended syntax to customize this. The extended
 /// syntax is "expected-<type> <n> {{diag text}}", where \<type> is one of
@@ -212,11 +225,14 @@ public:
     HasOtherExpectedDirectives
   };
 
+  class MarkerTracker;
+
 private:
   DiagnosticsEngine &Diags;
   DiagnosticConsumer *PrimaryClient;
   std::unique_ptr<DiagnosticConsumer> PrimaryClientOwner;
   std::unique_ptr<TextDiagnosticBuffer> Buffer;
+  std::unique_ptr<MarkerTracker> Markers;
   const Preprocessor *CurrentPreprocessor = nullptr;
   const LangOptions *LangOpts = nullptr;
   SourceManager *SrcManager = nullptr;

clang/lib/Frontend/VerifyDiagnosticConsumer.cpp

@@ -50,23 +50,6 @@ using Directive = VerifyDiagnosticConsumer::Directive;
 using DirectiveList = VerifyDiagnosticConsumer::DirectiveList;
 using ExpectedData = VerifyDiagnosticConsumer::ExpectedData;
 
-VerifyDiagnosticConsumer::VerifyDiagnosticConsumer(DiagnosticsEngine &Diags_)
-    : Diags(Diags_), PrimaryClient(Diags.getClient()),
-      PrimaryClientOwner(Diags.takeClient()),
-      Buffer(new TextDiagnosticBuffer()), Status(HasNoDirectives) {
-  if (Diags.hasSourceManager())
-    setSourceManager(Diags.getSourceManager());
-}
-
-VerifyDiagnosticConsumer::~VerifyDiagnosticConsumer() {
-  assert(!ActiveSourceFiles && "Incomplete parsing of source files!");
-  assert(!CurrentPreprocessor && "CurrentPreprocessor should be invalid!");
-  SrcManager = nullptr;
-  CheckDiagnostics();
-  assert(!Diags.ownsClient() &&
-         "The VerifyDiagnosticConsumer takes over ownership of the client!");
-}
-
 #ifndef NDEBUG
 
 namespace {
@@ -93,86 +76,6 @@ public:
 
 #endif
 
-// DiagnosticConsumer interface.
-
-void VerifyDiagnosticConsumer::BeginSourceFile(const LangOptions &LangOpts,
-                                               const Preprocessor *PP) {
-  // Attach comment handler on first invocation.
-  if (++ActiveSourceFiles == 1) {
-    if (PP) {
-      CurrentPreprocessor = PP;
-      this->LangOpts = &LangOpts;
-      setSourceManager(PP->getSourceManager());
-      const_cast<Preprocessor *>(PP)->addCommentHandler(this);
-#ifndef NDEBUG
-      // Debug build tracks parsed files.
-      const_cast<Preprocessor *>(PP)->addPPCallbacks(
-                      llvm::make_unique<VerifyFileTracker>(*this, *SrcManager));
-#endif
-    }
-  }
-
-  assert((!PP || CurrentPreprocessor == PP) && "Preprocessor changed!");
-  PrimaryClient->BeginSourceFile(LangOpts, PP);
-}
-
-void VerifyDiagnosticConsumer::EndSourceFile() {
-  assert(ActiveSourceFiles && "No active source files!");
-  PrimaryClient->EndSourceFile();
-
-  // Detach comment handler once last active source file completed.
-  if (--ActiveSourceFiles == 0) {
-    if (CurrentPreprocessor)
-      const_cast<Preprocessor *>(CurrentPreprocessor)->
-          removeCommentHandler(this);
-
-    // Check diagnostics once last file completed.
-    CheckDiagnostics();
-    CurrentPreprocessor = nullptr;
-    LangOpts = nullptr;
-  }
-}
-
-void VerifyDiagnosticConsumer::HandleDiagnostic(
-      DiagnosticsEngine::Level DiagLevel, const Diagnostic &Info) {
-  if (Info.hasSourceManager()) {
-    // If this diagnostic is for a different source manager, ignore it.
-    if (SrcManager && &Info.getSourceManager() != SrcManager)
-      return;
-
-    setSourceManager(Info.getSourceManager());
-  }
-
-#ifndef NDEBUG
-  // Debug build tracks unparsed files for possible
-  // unparsed expected-* directives.
-  if (SrcManager) {
-    SourceLocation Loc = Info.getLocation();
-    if (Loc.isValid()) {
-      ParsedStatus PS = IsUnparsed;
-
-      Loc = SrcManager->getExpansionLoc(Loc);
-      FileID FID = SrcManager->getFileID(Loc);
-
-      const FileEntry *FE = SrcManager->getFileEntryForID(FID);
-      if (FE && CurrentPreprocessor && SrcManager->isLoadedFileID(FID)) {
-        // If the file is a modules header file it shall not be parsed
-        // for expected-* directives.
-        HeaderSearch &HS = CurrentPreprocessor->getHeaderSearchInfo();
-        if (HS.findModuleForHeader(FE))
-          PS = IsUnparsedNoDirectives;
-      }
-
-      UpdateParsedFileStatus(*SrcManager, FID, PS);
-    }
-  }
-#endif
-
-  // Send the diagnostic to the buffer, we will check it once we reach the end
-  // of the source file (or are destructed).
-  Buffer->HandleDiagnostic(DiagLevel, Info);
-}
-
 //===----------------------------------------------------------------------===//
 // Checking diagnostics implementation.
 //===----------------------------------------------------------------------===//
@@ -241,17 +144,31 @@ public:
   bool Next(unsigned &N) {
     unsigned TMP = 0;
     P = C;
-    for (; P < End && P[0] >= '0' && P[0] <= '9'; ++P) {
-      TMP *= 10;
-      TMP += P[0] - '0';
-    }
-    if (P == C)
-      return false;
     PEnd = P;
+    for (; PEnd < End && *PEnd >= '0' && *PEnd <= '9'; ++PEnd) {
+      TMP *= 10;
+      TMP += *PEnd - '0';
+    }
+    if (PEnd == C)
+      return false;
     N = TMP;
     return true;
   }
 
+  // Return true if a marker is next.
+  // A marker is the longest match for /#[A-Za-z0-9_-]+/.
+  bool NextMarker() {
+    P = C;
+    if (P == End || *P != '#')
+      return false;
+    PEnd = P;
+    ++PEnd;
+    while ((isAlphanumeric(*PEnd) || *PEnd == '-' || *PEnd == '_') &&
+           PEnd < End)
+      ++PEnd;
+    return PEnd > P + 1;
+  }
+
   // Return true if string literal S is matched in content.
   // When true, P marks begin-position of the match, and calling Advance sets C
   // to end-position of the match.
@@ -333,6 +250,10 @@ public:
     return C < End;
   }
 
+  // Return the text matched by the previous next/search.
+  // Behavior is undefined if previous next/search failed.
+  StringRef Match() { return StringRef(P, PEnd - P); }
+
   // Skip zero or more whitespace.
   void SkipWhitespace() {
     for (; C < End && isWhitespace(*C); ++C)
@@ -353,6 +274,7 @@ public:
   // Position of next char in content.
   const char *C;
 
+  // Previous next/search subject start.
   const char *P;
 
 private:
@@ -360,17 +282,142 @@ private:
   const char *PEnd = nullptr;
 };
 
+// The information necessary to create a directive.
+struct UnattachedDirective {
+  DirectiveList *DL = nullptr;
+  bool RegexKind = false;
+  SourceLocation DirectivePos, ContentBegin;
+  std::string Text;
+  unsigned Min = 1, Max = 1;
+};
+
+// Attach the specified directive to the line of code indicated by
+// \p ExpectedLoc.
+void attachDirective(DiagnosticsEngine &Diags, const UnattachedDirective &UD,
+                     SourceLocation ExpectedLoc, bool MatchAnyLine = false) {
+  // Construct new directive.
+  std::unique_ptr<Directive> D =
+      Directive::create(UD.RegexKind, UD.DirectivePos, ExpectedLoc,
+                        MatchAnyLine, UD.Text, UD.Min, UD.Max);
+
+  std::string Error;
+  if (!D->isValid(Error)) {
+    Diags.Report(UD.ContentBegin, diag::err_verify_invalid_content)
+      << (UD.RegexKind ? "regex" : "string") << Error;
+  }
+
+  UD.DL->push_back(std::move(D));
+}
+
 } // anonymous
 
+// Tracker for markers in the input files. A marker is a comment of the form
+//
+//   n = 123; // #123
+//
+// ... that can be referred to by a later expected-* directive:
+//
+//   // expected-error@#123 {{undeclared identifier 'n'}}
+//
+// Marker declarations must be at the start of a comment or preceded by
+// whitespace to distinguish them from uses of markers in directives.
+class VerifyDiagnosticConsumer::MarkerTracker {
+  DiagnosticsEngine &Diags;
+
+  struct Marker {
+    SourceLocation DefLoc;
+    SourceLocation RedefLoc;
+    SourceLocation UseLoc;
+  };
+  llvm::StringMap<Marker> Markers;
+
+  // Directives that couldn't be created yet because they name an unknown
+  // marker.
+  llvm::StringMap<llvm::SmallVector<UnattachedDirective, 2>> DeferredDirectives;
+
+public:
+  MarkerTracker(DiagnosticsEngine &Diags) : Diags(Diags) {}
+
+  // Register a marker.
+  void addMarker(StringRef MarkerName, SourceLocation Pos) {
+    auto InsertResult = Markers.insert(
+        {MarkerName, Marker{Pos, SourceLocation(), SourceLocation()}});
+
+    Marker &M = InsertResult.first->second;
+    if (!InsertResult.second) {
+      // Marker was redefined.
+      M.RedefLoc = Pos;
+    } else {
+      // First definition: build any deferred directives.
+      auto Deferred = DeferredDirectives.find(MarkerName);
+      if (Deferred != DeferredDirectives.end()) {
+        for (auto &UD : Deferred->second) {
+          if (M.UseLoc.isInvalid())
+            M.UseLoc = UD.DirectivePos;
+          attachDirective(Diags, UD, Pos);
+        }
+        DeferredDirectives.erase(Deferred);
+      }
+    }
+  }
+
+  // Register a directive at the specified marker.
+  void addDirective(StringRef MarkerName, const UnattachedDirective &UD) {
+    auto MarkerIt = Markers.find(MarkerName);
+    if (MarkerIt != Markers.end()) {
+      Marker &M = MarkerIt->second;
+      if (M.UseLoc.isInvalid())
+        M.UseLoc = UD.DirectivePos;
+      return attachDirective(Diags, UD, M.DefLoc);
+    }
+    DeferredDirectives[MarkerName].push_back(UD);
+  }
+
+  // Ensure we have no remaining deferred directives, and no
+  // multiply-defined-and-used markers.
+  void finalize() {
+    for (auto &MarkerInfo : Markers) {
+      StringRef Name = MarkerInfo.first();
+      Marker &M = MarkerInfo.second;
+      if (M.RedefLoc.isValid() && M.UseLoc.isValid()) {
+        Diags.Report(M.UseLoc, diag::err_verify_ambiguous_marker) << Name;
+        Diags.Report(M.DefLoc, diag::note_verify_ambiguous_marker) << Name;
+        Diags.Report(M.RedefLoc, diag::note_verify_ambiguous_marker) << Name;
+      }
+    }
+
+    for (auto &DeferredPair : DeferredDirectives) {
+      Diags.Report(DeferredPair.second.front().DirectivePos,
+                   diag::err_verify_no_such_marker)
+          << DeferredPair.first();
+    }
+  }
+};
+
 /// ParseDirective - Go through the comment and see if it indicates expected
 /// diagnostics. If so, then put them in the appropriate directive list.
 ///
 /// Returns true if any valid directives were found.
 static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
                            Preprocessor *PP, SourceLocation Pos,
-                           VerifyDiagnosticConsumer::DirectiveStatus &Status) {
+                           VerifyDiagnosticConsumer::DirectiveStatus &Status,
+                           VerifyDiagnosticConsumer::MarkerTracker &Markers) {
   DiagnosticsEngine &Diags = PP ? PP->getDiagnostics() : SM.getDiagnostics();
 
+  // First, scan the comment looking for markers.
+  for (ParseHelper PH(S); !PH.Done();) {
+    if (!PH.Search("#", true))
+      break;
+    PH.C = PH.P;
+    if (!PH.NextMarker()) {
+      PH.Next("#");
+      PH.Advance();
+      continue;
+    }
+    PH.Advance();
+    Markers.addMarker(PH.Match(), Pos);
+  }
+
   // A single comment may contain multiple directives.
   bool FoundDirective = false;
   for (ParseHelper PH(S); !PH.Done();) {
@@ -381,41 +428,41 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
     if (!(Prefixes.size() == 1 ? PH.Search(*Prefixes.begin(), true, true)
                                : PH.Search("", true, true)))
       break;
+
+    StringRef DToken = PH.Match();
     PH.Advance();
 
     // Default directive kind.
-    bool RegexKind = false;
+    UnattachedDirective D;
-    const char* KindStr = "string";
+    const char *KindStr = "string";
 
     // Parse the initial directive token in reverse so we can easily determine
     // its exact actual prefix.  If we were to parse it from the front instead,
     // it would be harder to determine where the prefix ends because there
     // might be multiple matching -verify prefixes because some might prefix
     // others.
-    StringRef DToken(PH.P, PH.C - PH.P);
 
     // Regex in initial directive token: -re
     if (DToken.endswith("-re")) {
-      RegexKind = true;
+      D.RegexKind = true;
       KindStr = "regex";
       DToken = DToken.substr(0, DToken.size()-3);
     }
 
     // Type in initial directive token: -{error|warning|note|no-diagnostics}
-    DirectiveList *DL = nullptr;
     bool NoDiag = false;
     StringRef DType;
     if (DToken.endswith(DType="-error"))
-      DL = ED ? &ED->Errors : nullptr;
+      D.DL = ED ? &ED->Errors : nullptr;
     else if (DToken.endswith(DType="-warning"))
-      DL = ED ? &ED->Warnings : nullptr;
+      D.DL = ED ? &ED->Warnings : nullptr;
     else if (DToken.endswith(DType="-remark"))
-      DL = ED ? &ED->Remarks : nullptr;
+      D.DL = ED ? &ED->Remarks : nullptr;
     else if (DToken.endswith(DType="-note"))
-      DL = ED ? &ED->Notes : nullptr;
+      D.DL = ED ? &ED->Notes : nullptr;
     else if (DToken.endswith(DType="-no-diagnostics")) {
       NoDiag = true;
-      if (RegexKind)
+      if (D.RegexKind)
         continue;
     }
     else
@@ -445,11 +492,12 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
 
     // If a directive has been found but we're not interested
     // in storing the directive information, return now.
-    if (!DL)
+    if (!D.DL)
       return true;
 
     // Next optional token: @
     SourceLocation ExpectedLoc;
+    StringRef Marker;
     bool MatchAnyLine = false;
     if (!PH.Next("@")) {
       ExpectedLoc = Pos;
@@ -471,6 +519,8 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
         // Absolute line number.
         if (Line > 0)
           ExpectedLoc = SM.translateLineCol(SM.getFileID(Pos), Line, 1);
+      } else if (PH.NextMarker()) {
+        Marker = PH.Match();
       } else if (PP && PH.Search(":")) {
         // Specific source file.
         StringRef Filename(PH.C, PH.P-PH.C);
@@ -501,7 +551,7 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
         ExpectedLoc = SourceLocation();
       }
 
-      if (ExpectedLoc.isInvalid() && !MatchAnyLine) {
+      if (ExpectedLoc.isInvalid() && !MatchAnyLine && Marker.empty()) {
         Diags.Report(Pos.getLocWithOffset(PH.C-PH.Begin),
                      diag::err_verify_missing_line) << KindStr;
         continue;
@@ -513,29 +563,27 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
     PH.SkipWhitespace();
 
     // Next optional token: positive integer or a '+'.
-    unsigned Min = 1;
+    if (PH.Next(D.Min)) {
-    unsigned Max = 1;
-    if (PH.Next(Min)) {
       PH.Advance();
       // A positive integer can be followed by a '+' meaning min
       // or more, or by a '-' meaning a range from min to max.
       if (PH.Next("+")) {
-        Max = Directive::MaxCount;
+        D.Max = Directive::MaxCount;
         PH.Advance();
       } else if (PH.Next("-")) {
         PH.Advance();
-        if (!PH.Next(Max) || Max < Min) {
+        if (!PH.Next(D.Max) || D.Max < D.Min) {
           Diags.Report(Pos.getLocWithOffset(PH.C-PH.Begin),
                        diag::err_verify_invalid_range) << KindStr;
           continue;
         }
         PH.Advance();
       } else {
-        Max = Min;
+        D.Max = D.Min;
       }
     } else if (PH.Next("+")) {
       // '+' on its own means "1 or more".
-      Max = Directive::MaxCount;
+      D.Max = Directive::MaxCount;
       PH.Advance();
     }
 
@@ -550,7 +598,6 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
     }
     PH.Advance();
     const char* const ContentBegin = PH.C; // mark content begin
-
     // Search for token: }}
     if (!PH.SearchClosingBrace("{{", "}}")) {
       Diags.Report(Pos.getLocWithOffset(PH.C-PH.Begin),
@@ -560,45 +607,139 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
     const char* const ContentEnd = PH.P; // mark content end
     PH.Advance();
 
+    D.DirectivePos = Pos;
+    D.ContentBegin = Pos.getLocWithOffset(ContentBegin - PH.Begin);
+
     // Build directive text; convert \n to newlines.
-    std::string Text;
     StringRef NewlineStr = "\\n";
     StringRef Content(ContentBegin, ContentEnd-ContentBegin);
     size_t CPos = 0;
     size_t FPos;
     while ((FPos = Content.find(NewlineStr, CPos)) != StringRef::npos) {
-      Text += Content.substr(CPos, FPos-CPos);
+      D.Text += Content.substr(CPos, FPos-CPos);
-      Text += '\n';
+      D.Text += '\n';
       CPos = FPos + NewlineStr.size();
     }
-    if (Text.empty())
+    if (D.Text.empty())
-      Text.assign(ContentBegin, ContentEnd);
+      D.Text.assign(ContentBegin, ContentEnd);
 
     // Check that regex directives contain at least one regex.
-    if (RegexKind && Text.find("{{") == StringRef::npos) {
+    if (D.RegexKind && D.Text.find("{{") == StringRef::npos) {
-      Diags.Report(Pos.getLocWithOffset(ContentBegin-PH.Begin),
+      Diags.Report(D.ContentBegin, diag::err_verify_missing_regex) << D.Text;
-                   diag::err_verify_missing_regex) << Text;
       return false;
     }
 
-    // Construct new directive.
+    if (Marker.empty())
-    std::unique_ptr<Directive> D = Directive::create(
+      attachDirective(Diags, D, ExpectedLoc, MatchAnyLine);
-        RegexKind, Pos, ExpectedLoc, MatchAnyLine, Text, Min, Max);
+    else
-
+      Markers.addDirective(Marker, D);
-    std::string Error;
+    FoundDirective = true;
-    if (D->isValid(Error)) {
-      DL->push_back(std::move(D));
-      FoundDirective = true;
-    } else {
-      Diags.Report(Pos.getLocWithOffset(ContentBegin-PH.Begin),
-                   diag::err_verify_invalid_content)
-        << KindStr << Error;
-    }
   }
 
   return FoundDirective;
 }
 
+VerifyDiagnosticConsumer::VerifyDiagnosticConsumer(DiagnosticsEngine &Diags_)
+    : Diags(Diags_), PrimaryClient(Diags.getClient()),
+      PrimaryClientOwner(Diags.takeClient()),
+      Buffer(new TextDiagnosticBuffer()), Markers(new MarkerTracker(Diags)),
+      Status(HasNoDirectives) {
+  if (Diags.hasSourceManager())
+    setSourceManager(Diags.getSourceManager());
+}
+
+VerifyDiagnosticConsumer::~VerifyDiagnosticConsumer() {
+  assert(!ActiveSourceFiles && "Incomplete parsing of source files!");
+  assert(!CurrentPreprocessor && "CurrentPreprocessor should be invalid!");
+  SrcManager = nullptr;
+  CheckDiagnostics();
+  assert(!Diags.ownsClient() &&
+         "The VerifyDiagnosticConsumer takes over ownership of the client!");
+}
+
+// DiagnosticConsumer interface.
+
+void VerifyDiagnosticConsumer::BeginSourceFile(const LangOptions &LangOpts,
+                                               const Preprocessor *PP) {
+  // Attach comment handler on first invocation.
+  if (++ActiveSourceFiles == 1) {
+    if (PP) {
+      CurrentPreprocessor = PP;
+      this->LangOpts = &LangOpts;
+      setSourceManager(PP->getSourceManager());
+      const_cast<Preprocessor *>(PP)->addCommentHandler(this);
+#ifndef NDEBUG
+      // Debug build tracks parsed files.
+      const_cast<Preprocessor *>(PP)->addPPCallbacks(
+                      llvm::make_unique<VerifyFileTracker>(*this, *SrcManager));
+#endif
+    }
+  }
+
+  assert((!PP || CurrentPreprocessor == PP) && "Preprocessor changed!");
+  PrimaryClient->BeginSourceFile(LangOpts, PP);
+}
+
+void VerifyDiagnosticConsumer::EndSourceFile() {
+  assert(ActiveSourceFiles && "No active source files!");
+  PrimaryClient->EndSourceFile();
+
+  // Detach comment handler once last active source file completed.
+  if (--ActiveSourceFiles == 0) {
+    if (CurrentPreprocessor)
+      const_cast<Preprocessor *>(CurrentPreprocessor)->
+          removeCommentHandler(this);
+
+    // Diagnose any used-but-not-defined markers.
+    Markers->finalize();
+
+    // Check diagnostics once last file completed.
+    CheckDiagnostics();
+    CurrentPreprocessor = nullptr;
+    LangOpts = nullptr;
+  }
+}
+
+void VerifyDiagnosticConsumer::HandleDiagnostic(
+      DiagnosticsEngine::Level DiagLevel, const Diagnostic &Info) {
+  if (Info.hasSourceManager()) {
+    // If this diagnostic is for a different source manager, ignore it.
+    if (SrcManager && &Info.getSourceManager() != SrcManager)
+      return;
+
+    setSourceManager(Info.getSourceManager());
+  }
+
+#ifndef NDEBUG
+  // Debug build tracks unparsed files for possible
+  // unparsed expected-* directives.
+  if (SrcManager) {
+    SourceLocation Loc = Info.getLocation();
+    if (Loc.isValid()) {
+      ParsedStatus PS = IsUnparsed;
+
+      Loc = SrcManager->getExpansionLoc(Loc);
+      FileID FID = SrcManager->getFileID(Loc);
+
+      const FileEntry *FE = SrcManager->getFileEntryForID(FID);
+      if (FE && CurrentPreprocessor && SrcManager->isLoadedFileID(FID)) {
+        // If the file is a modules header file it shall not be parsed
+        // for expected-* directives.
+        HeaderSearch &HS = CurrentPreprocessor->getHeaderSearchInfo();
+        if (HS.findModuleForHeader(FE))
+          PS = IsUnparsedNoDirectives;
+      }
+
+      UpdateParsedFileStatus(*SrcManager, FID, PS);
+    }
+  }
+#endif
+
+  // Send the diagnostic to the buffer, we will check it once we reach the end
+  // of the source file (or are destructed).
+  Buffer->HandleDiagnostic(DiagLevel, Info);
+}
+
 /// HandleComment - Hook into the preprocessor and extract comments containing
 ///  expected errors and warnings.
 bool VerifyDiagnosticConsumer::HandleComment(Preprocessor &PP,
@@ -620,7 +761,7 @@ bool VerifyDiagnosticConsumer::HandleComment(Preprocessor &PP,
   // Fold any "\<EOL>" sequences
   size_t loc = C.find('\\');
   if (loc == StringRef::npos) {
-    ParseDirective(C, &ED, SM, &PP, CommentBegin, Status);
+    ParseDirective(C, &ED, SM, &PP, CommentBegin, Status, *Markers);
     return false;
   }
 
@@ -650,7 +791,7 @@ bool VerifyDiagnosticConsumer::HandleComment(Preprocessor &PP,
   }
 
   if (!C2.empty())
-    ParseDirective(C2, &ED, SM, &PP, CommentBegin, Status);
+    ParseDirective(C2, &ED, SM, &PP, CommentBegin, Status, *Markers);
   return false;
 }
 
@@ -684,9 +825,12 @@ static bool findDirectives(SourceManager &SM, FileID FID,
     std::string Comment = RawLex.getSpelling(Tok, SM, LangOpts);
     if (Comment.empty()) continue;
 
+    // We don't care about tracking markers for this phase.
+    VerifyDiagnosticConsumer::MarkerTracker Markers(SM.getDiagnostics());
+
     // Find first directive.
     if (ParseDirective(Comment, nullptr, SM, nullptr, Tok.getLocation(),
-                       Status))
+                       Status, Markers))
       return true;
   }
   return false;

clang/test/Frontend/verify-marker.c

@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 -verify %s
+
+#include "verify-marker.h" // expected-error@#1 {{unknown type name 'unknown_type'}}
+
+int x = 1; // #a
+int x = 2; // #b
+// expected-error@#b {{redefinition of 'x'}}
+// expected-note@#a {{previous}}
+
+// expected-error@#unknown {{}}  expected-error {{use of undefined marker '#unknown'}}
+
+// This is OK: there's no problem with a source file containing what looks like
+// a duplicate definition of a marker if that marker is never used.
+// #foo
+// #foo
+
+// #bar  expected-note {{ambiguous marker '#bar' is defined here}}
+// #bar  expected-note {{ambiguous marker '#bar' is defined here}}
+// expected-error@#bar 0-1{{oops}}  expected-error{{reference to marker '#bar' is ambiguous}}
+
+// expected-error@#forward_ref {{undeclared identifier 'future'}}
+int y = future; // #forward_ref

clang/test/Frontend/verify-marker.h

@@ -0,0 +1 @@
+unknown_type x; // #1