[libclang] Introduce a new function to apply the indexing callbacks on an existing

CXTranslationUnit, mainly to be used for indexing a PCH.

llvm-svn: 144623

clang/include/clang-c/Index.h

@@ -4059,6 +4059,8 @@ typedef struct {
 typedef struct {
   const CXIdxObjCContainerDeclInfo *containerInfo;
   const CXIdxEntityInfo *objcClass;
+  CXCursor classCursor;
+  CXIdxLoc classLoc;
 } CXIdxObjCCategoryDeclInfo;
 
 typedef struct {
@@ -4193,6 +4195,19 @@ clang_index_getObjCCategoryDeclInfo(const CXIdxDeclInfo *);
 CINDEX_LINKAGE const CXIdxObjCProtocolRefListInfo *
 clang_index_getObjCProtocolRefListInfo(const CXIdxDeclInfo *);
 
+typedef enum {
+  /**
+   * \brief Used to indicate that no special indexing options are needed.
+   */
+  CXIndexOpt_None = 0x0,
+  
+  /**
+   * \brief Used to indicate that \see indexEntityReference should be invoked
+   * for only one reference of an entity per source file.
+   */
+  CXIndexOpt_OneRefPerFile = 0x1
+} CXIndexOptFlags;
+
 /**
  * \brief Index the given source file and the translation unit corresponding
  * to that file via callbacks implemented through \see IndexerCallbacks.
@@ -4206,17 +4221,18 @@ clang_index_getObjCProtocolRefListInfo(const CXIdxDeclInfo *);
  * \param index_callbacks_size Size of \see IndexerCallbacks structure that gets
  * passed in index_callbacks.
  *
- * \param index_options Options affecting indexing; reserved.
+ * \param index_options A bitmask of options that affects how indexing is
+ * performed. This should be a bitwise OR of the CXIndexOpt_XXX flags.
  *
  * \param out_TU [out] pointer to store a CXTranslationUnit that can be reused
  * after indexing is finished. Set to NULL if you do not require it.
  *
- * \returns If there is a failure from which the compiler cannot recover returns
+ * \returns If there is a failure from which the there is no recovery, returns
  * non-zero, otherwise returns 0.
- * 
+ *
  * The rest of the parameters are the same as \see clang_parseTranslationUnit.
  */
-CINDEX_LINKAGE int clang_indexTranslationUnit(CXIndex CIdx,
+CINDEX_LINKAGE int clang_indexSourceFile(CXIndex CIdx,
                                          CXClientData client_data,
                                          IndexerCallbacks *index_callbacks,
                                          unsigned index_callbacks_size,
@@ -4229,6 +4245,28 @@ CINDEX_LINKAGE int clang_indexTranslationUnit(CXIndex CIdx,
                                          CXTranslationUnit *out_TU,
                                          unsigned TU_options);
 
+/**
+ * \brief Index the given translation unit via callbacks implemented through
+ * \see IndexerCallbacks.
+ * 
+ * The order of callback invocations is not guaranteed to be the same as
+ * when indexing a source file. The high level order will be:
+ * 
+ *   -Preprocessor callbacks invocations
+ *   -Declaration/reference callbacks invocations
+ *   -Diagnostic callback invocations
+ *
+ * The parameters are the same as \see clang_indexSourceFile.
+ * 
+ * \returns If there is a failure from which the there is no recovery, returns
+ * non-zero, otherwise returns 0.
+ */
+CINDEX_LINKAGE int clang_indexTranslationUnit(CXTranslationUnit,
+                                              CXClientData client_data,
+                                              IndexerCallbacks *index_callbacks,
+                                              unsigned index_callbacks_size,
+                                              unsigned index_options);
+
 /**
  * \brief Retrieve the CXIdxFile, file, line, column, and offset represented by
  * the given CXIdxLoc.

clang/tools/c-index-test/c-index-test.c

@@ -1765,6 +1765,10 @@ static void index_indexDeclaration(CXClientData client_data,
   if ((CatInfo = clang_index_getObjCCategoryDeclInfo(info))) {
     printEntityInfo("     <ObjCCategoryInfo>: class", client_data,
                     CatInfo->objcClass);
+    printf(" | cursor: ");
+    PrintCursor(CatInfo->classCursor);
+    printf(" | loc: ");
+    printCXIndexLoc(CatInfo->classLoc);
     printf("\n");
   }
 
@@ -1842,9 +1846,51 @@ static int index_file(int argc, const char **argv) {
   index_data.first_check_printed = 0;
   index_data.fail_for_error = 0;
 
-  result = clang_indexTranslationUnit(CIdx, &index_data,
-                                      &IndexCB,sizeof(IndexCB),
-                                      0, 0, argv, argc, 0, 0, 0, 0);
+  result = clang_indexSourceFile(CIdx, &index_data,
+                                 &IndexCB,sizeof(IndexCB),
+                                 0, 0, argv, argc, 0, 0, 0, 0);
+  if (index_data.fail_for_error)
+    return -1;
+
+  return result;
+}
+
+static int index_tu(int argc, const char **argv) {
+  CXIndex Idx;
+  CXTranslationUnit TU;
+  const char *check_prefix;
+  IndexData index_data;
+  int result;
+
+  check_prefix = 0;
+  if (argc > 0) {
+    if (strstr(argv[0], "-check-prefix=") == argv[0]) {
+      check_prefix = argv[0] + strlen("-check-prefix=");
+      ++argv;
+      --argc;
+    }
+  }
+
+  if (argc == 0) {
+    fprintf(stderr, "no ast file\n");
+    return -1;
+  }
+
+  if (!(Idx = clang_createIndex(/* excludeDeclsFromPCH */ 1,
+                                /* displayDiagnosics=*/1))) {
+    fprintf(stderr, "Could not create Index\n");
+    return 1;
+  }
+
+  if (!CreateTranslationUnit(Idx, argv[0], &TU))
+    return 1;
+
+  index_data.check_prefix = check_prefix;
+  index_data.first_check_printed = 0;
+  index_data.fail_for_error = 0;
+
+  result = clang_indexTranslationUnit(TU, &index_data,
+                                      &IndexCB,sizeof(IndexCB), 0);
   if (index_data.fail_for_error)
     return -1;
 
@@ -2394,6 +2440,7 @@ static void print_usage(void) {
     "       c-index-test -cursor-at=<site> <compiler arguments>\n"
     "       c-index-test -file-refs-at=<site> <compiler arguments>\n"
     "       c-index-test -index-file [-check-prefix=<FileCheck prefix>] <compiler arguments>\n"
+    "       c-index-test -index-tu [-check-prefix=<FileCheck prefix>] <AST file>\n"
     "       c-index-test -test-file-scan <AST file> <source file> "
           "[FileCheck prefix]\n");
   fprintf(stderr,
@@ -2449,6 +2496,8 @@ int cindextest_main(int argc, const char **argv) {
     return find_file_refs_at(argc, argv);
   if (argc > 2 && strcmp(argv[1], "-index-file") == 0)
     return index_file(argc - 2, argv + 2);
+  if (argc > 2 && strcmp(argv[1], "-index-tu") == 0)
+    return index_tu(argc - 2, argv + 2);
   else if (argc >= 4 && strncmp(argv[1], "-test-load-tu", 13) == 0) {
     CXCursorVisitor I = GetVisitor(argv[1] + 13);
     if (I)

clang/tools/libclang/IndexDecl.cpp

@@ -172,17 +172,19 @@ void IndexingContext::indexDeclContext(const DeclContext *DC) {
   }
 }
 
+void IndexingContext::indexTopLevelDecl(Decl *D) {
+  if (isNotFromSourceFile(D->getLocation()))
+    return;
+
+  if (isa<ObjCMethodDecl>(D))
+    return; // Wait for the objc container.
+
+  indexDecl(D);
+}
+
 void IndexingContext::indexDeclGroupRef(DeclGroupRef DG) {
-  for (DeclGroupRef::iterator I = DG.begin(), E = DG.end(); I != E; ++I) {
-    Decl *D = *I;
-    if (isNotFromSourceFile(D->getLocation()))
-      return;
-
-    if (isa<ObjCMethodDecl>(D))
-      continue; // Wait for the objc container.
-
-    indexDecl(D);
-  }
+  for (DeclGroupRef::iterator I = DG.begin(), E = DG.end(); I != E; ++I)
+    indexTopLevelDecl(*I);
 }
 
 void IndexingContext::indexTUDeclsInObjCContainer() {

clang/tools/libclang/Indexing.cpp

@@ -200,10 +200,10 @@ public:
 };
 
 //===----------------------------------------------------------------------===//
-// clang_indexTranslationUnit Implementation
+// clang_indexSourceFileUnit Implementation
 //===----------------------------------------------------------------------===//
 
-struct IndexTranslationUnitInfo {
+struct IndexSourceFileInfo {
   CXIndex CIdx;
   CXClientData client_data;
   IndexerCallbacks *index_callbacks;
@@ -231,9 +231,9 @@ struct MemBufferOwner {
 
 } // anonymous namespace
 
-static void clang_indexTranslationUnit_Impl(void *UserData) {
-  IndexTranslationUnitInfo *ITUI =
-    static_cast<IndexTranslationUnitInfo*>(UserData);
+static void clang_indexSourceFile_Impl(void *UserData) {
+  IndexSourceFileInfo *ITUI =
+    static_cast<IndexSourceFileInfo*>(UserData);
   CXIndex CIdx = ITUI->CIdx;
   CXClientData client_data = ITUI->client_data;
   IndexerCallbacks *client_index_callbacks = ITUI->index_callbacks;
@@ -366,6 +366,141 @@ static void clang_indexTranslationUnit_Impl(void *UserData) {
   ITUI->result = 0; // success.
 }
 
+//===----------------------------------------------------------------------===//
+// clang_indexTranslationUnit Implementation
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+struct IndexTranslationUnitInfo {
+  CXTranslationUnit TU;
+  CXClientData client_data;
+  IndexerCallbacks *index_callbacks;
+  unsigned index_callbacks_size;
+  unsigned index_options;
+  int result;
+};
+
+} // anonymous namespace
+
+static void indexPreprocessingRecord(ASTUnit &Unit, IndexingContext &IdxCtx) {
+  Preprocessor &PP = Unit.getPreprocessor();
+  if (!PP.getPreprocessingRecord())
+    return;
+
+  PreprocessingRecord &PPRec = *PP.getPreprocessingRecord();
+
+  // FIXME: Only deserialize inclusion directives.
+  // FIXME: Only deserialize stuff from the last chained PCH, not the PCH/Module
+  // that it depends on.
+
+  bool OnlyLocal = !Unit.isMainFileAST() && Unit.getOnlyLocalDecls();
+  PreprocessingRecord::iterator I, E;
+  if (OnlyLocal) {
+    I = PPRec.local_begin();
+    E = PPRec.local_end();
+  } else {
+    I = PPRec.begin();
+    E = PPRec.end();
+  }
+
+  for (; I != E; ++I) {
+    PreprocessedEntity *PPE = *I;
+
+    if (InclusionDirective *ID = dyn_cast<InclusionDirective>(PPE)) {
+      IdxCtx.ppIncludedFile(ID->getSourceRange().getBegin(), ID->getFileName(),
+                     ID->getFile(), ID->getKind() == InclusionDirective::Import,
+                     !ID->wasInQuotes());
+    }
+  }
+}
+
+static void indexTranslationUnit(ASTUnit &Unit, IndexingContext &IdxCtx) {
+  // FIXME: Only deserialize stuff from the last chained PCH, not the PCH/Module
+  // that it depends on.
+
+  bool OnlyLocal = !Unit.isMainFileAST() && Unit.getOnlyLocalDecls();
+
+  if (OnlyLocal) {
+    for (ASTUnit::top_level_iterator TL = Unit.top_level_begin(),
+                                  TLEnd = Unit.top_level_end();
+           TL != TLEnd; ++TL) {
+      IdxCtx.indexTopLevelDecl(*TL);
+    }
+
+  } else {
+    TranslationUnitDecl *TUDecl = Unit.getASTContext().getTranslationUnitDecl();
+    for (TranslationUnitDecl::decl_iterator
+           I = TUDecl->decls_begin(), E = TUDecl->decls_end(); I != E; ++I) {
+      IdxCtx.indexTopLevelDecl(*I);
+    }
+  }
+}
+
+static void indexDiagnostics(CXTranslationUnit TU, IndexingContext &IdxCtx) {
+  unsigned Num = clang_getNumDiagnostics(TU);
+  for (unsigned i = 0; i != Num; ++i) {
+    CXDiagnostic Diag = clang_getDiagnostic(TU, i);
+    IdxCtx.handleDiagnostic(Diag);
+    clang_disposeDiagnostic(Diag);
+  }
+}
+
+static void clang_indexTranslationUnit_Impl(void *UserData) {
+  IndexTranslationUnitInfo *ITUI =
+    static_cast<IndexTranslationUnitInfo*>(UserData);
+  CXTranslationUnit TU = ITUI->TU;
+  CXClientData client_data = ITUI->client_data;
+  IndexerCallbacks *client_index_callbacks = ITUI->index_callbacks;
+  unsigned index_callbacks_size = ITUI->index_callbacks_size;
+  unsigned index_options = ITUI->index_options;
+  ITUI->result = 1; // init as error.
+
+  if (!TU)
+    return;
+  if (!client_index_callbacks || index_callbacks_size == 0)
+    return;
+
+  IndexerCallbacks CB;
+  memset(&CB, 0, sizeof(CB));
+  unsigned ClientCBSize = index_callbacks_size < sizeof(CB)
+                                  ? index_callbacks_size : sizeof(CB);
+  memcpy(&CB, client_index_callbacks, ClientCBSize);
+
+  llvm::OwningPtr<IndexingContext> IndexCtx;
+  IndexCtx.reset(new IndexingContext(client_data, CB, index_options, TU));
+
+  // Recover resources if we crash before exiting this method.
+  llvm::CrashRecoveryContextCleanupRegistrar<IndexingContext>
+    IndexCtxCleanup(IndexCtx.get());
+
+  llvm::OwningPtr<IndexingConsumer> IndexConsumer;
+  IndexConsumer.reset(new IndexingConsumer(*IndexCtx));
+
+  // Recover resources if we crash before exiting this method.
+  llvm::CrashRecoveryContextCleanupRegistrar<IndexingConsumer>
+    IndexConsumerCleanup(IndexConsumer.get());
+
+  ASTUnit *Unit = static_cast<ASTUnit *>(TU->TUData);
+  if (!Unit)
+    return;
+
+  FileManager &FileMgr = Unit->getFileManager();
+
+  if (Unit->getOriginalSourceFileName().empty())
+    IndexCtx->enteredMainFile(0);
+  else
+    IndexCtx->enteredMainFile(FileMgr.getFile(Unit->getOriginalSourceFileName()));
+
+  IndexConsumer->Initialize(Unit->getASTContext());
+
+  indexPreprocessingRecord(*Unit, *IndexCtx);
+  indexTranslationUnit(*Unit, *IndexCtx);
+  indexDiagnostics(TU, *IndexCtx);
+
+  ITUI->result = 0;
+}
+
 //===----------------------------------------------------------------------===//
 // libclang public APIs.
 //===----------------------------------------------------------------------===//
@@ -433,7 +568,7 @@ clang_index_getObjCProtocolRefListInfo(const CXIdxDeclInfo *DInfo) {
   return 0;
 }
 
-int clang_indexTranslationUnit(CXIndex CIdx,
+int clang_indexSourceFile(CXIndex CIdx,
                                 CXClientData client_data,
                                 IndexerCallbacks *index_callbacks,
                                 unsigned index_callbacks_size,
@@ -445,21 +580,22 @@ int clang_indexTranslationUnit(CXIndex CIdx,
                                 unsigned num_unsaved_files,
                                 CXTranslationUnit *out_TU,
                                 unsigned TU_options) {
-  IndexTranslationUnitInfo ITUI = { CIdx, client_data, index_callbacks,
+
+  IndexSourceFileInfo ITUI = { CIdx, client_data, index_callbacks,
                                     index_callbacks_size, index_options,
                                     source_filename, command_line_args,
                                     num_command_line_args, unsaved_files,
                                     num_unsaved_files, out_TU, TU_options, 0 };
 
   if (getenv("LIBCLANG_NOTHREADS")) {
-    clang_indexTranslationUnit_Impl(&ITUI);
+    clang_indexSourceFile_Impl(&ITUI);
     return ITUI.result;
   }
 
   llvm::CrashRecoveryContext CRC;
 
-  if (!RunSafely(CRC, clang_indexTranslationUnit_Impl, &ITUI)) {
-    fprintf(stderr, "libclang: crash detected during parsing: {\n");
+  if (!RunSafely(CRC, clang_indexSourceFile_Impl, &ITUI)) {
+    fprintf(stderr, "libclang: crash detected during indexing source file: {\n");
     fprintf(stderr, "  'source_filename' : '%s'\n", source_filename);
     fprintf(stderr, "  'command_line_args' : [");
     for (int i = 0; i != num_command_line_args; ++i) {
@@ -488,6 +624,31 @@ int clang_indexTranslationUnit(CXIndex CIdx,
   return ITUI.result;
 }
 
+int clang_indexTranslationUnit(CXTranslationUnit TU,
+                               CXClientData client_data,
+                               IndexerCallbacks *index_callbacks,
+                               unsigned index_callbacks_size,
+                               unsigned index_options) {
+
+  IndexTranslationUnitInfo ITUI = { TU, client_data, index_callbacks,
+                                    index_callbacks_size, index_options, 0 };
+
+  if (getenv("LIBCLANG_NOTHREADS")) {
+    clang_indexTranslationUnit_Impl(&ITUI);
+    return ITUI.result;
+  }
+
+  llvm::CrashRecoveryContext CRC;
+
+  if (!RunSafely(CRC, clang_indexTranslationUnit_Impl, &ITUI)) {
+    fprintf(stderr, "libclang: crash detected during indexing TU\n");
+    
+    return 1;
+  }
+
+  return ITUI.result;
+}
+
 void clang_indexLoc_getFileLocation(CXIdxLoc location,
                                     CXIdxClientFile *indexFile,
                                     CXFile *file,

clang/tools/libclang/IndexingContext.cpp

@@ -95,6 +95,13 @@ void IndexingContext::handleDiagnostic(const StoredDiagnostic &StoredDiag) {
   CB.diagnostic(ClientData, &CXDiag, 0);
 }
 
+void IndexingContext::handleDiagnostic(CXDiagnostic CXDiag) {
+  if (!CB.diagnostic)
+    return;
+
+  CB.diagnostic(ClientData, CXDiag, 0);
+}
+
 void IndexingContext::handleDecl(const NamedDecl *D,
                                  SourceLocation Loc, CXCursor Cursor,
                                  DeclInfo &DInfo) {
@@ -230,11 +237,17 @@ void IndexingContext::handleObjCCategory(const ObjCCategoryDecl *D) {
   ObjCCategoryDeclInfo CatDInfo(/*isImplementation=*/false);
   CXIdxEntityInfo ClassEntity;
   StrAdapter SA(*this);
+  const ObjCInterfaceDecl *IFaceD = D->getClassInterface();
+  SourceLocation ClassLoc = D->getLocation();
+  SourceLocation CategoryLoc = D->getCategoryNameLoc();
   getEntityInfo(D->getClassInterface(), ClassEntity, SA);
 
   CatDInfo.ObjCCatDeclInfo.containerInfo = &CatDInfo.ObjCContDeclInfo;
   CatDInfo.ObjCCatDeclInfo.objcClass = &ClassEntity;
-  handleObjCContainer(D, D->getLocation(), getCursor(D), CatDInfo);
+  CatDInfo.ObjCCatDeclInfo.classCursor =
+      MakeCursorObjCClassRef(IFaceD, ClassLoc, CXTU);
+  CatDInfo.ObjCCatDeclInfo.classLoc = getIndexLoc(ClassLoc);
+  handleObjCContainer(D, CategoryLoc, getCursor(D), CatDInfo);
 }
 
 void IndexingContext::handleObjCCategoryImpl(const ObjCCategoryImplDecl *D) {

clang/tools/libclang/IndexingContext.h

@@ -227,6 +227,7 @@ public:
   void indexBody(const Stmt *S, const DeclContext *DC);
 
   void handleDiagnostic(const StoredDiagnostic &StoredDiag);
+  void handleDiagnostic(CXDiagnostic CXDiag);
 
   void handleFunction(const FunctionDecl *FD);
 
@@ -265,6 +266,7 @@ public:
 
   bool isNotFromSourceFile(SourceLocation Loc) const;
 
+  void indexTopLevelDecl(Decl *D);
   void indexTUDeclsInObjCContainer();
   void indexDeclGroupRef(DeclGroupRef DG);
 

clang/tools/libclang/libclang.exports

@@ -142,6 +142,7 @@ clang_index_getObjCProtocolRefListInfo
 clang_index_isEntityObjCContainerKind
 clang_indexLoc_getCXSourceLocation
 clang_indexLoc_getFileLocation
+clang_indexSourceFile
 clang_indexTranslationUnit
 clang_isAttribute
 clang_isConstQualifiedType