[clang-repl] Recommit "Land initial infrastructure for incremental parsing"

Original commit message:

  In http://lists.llvm.org/pipermail/llvm-dev/2020-July/143257.html we have
  mentioned our plans to make some of the incremental compilation facilities
  available in llvm mainline.

  This patch proposes a minimal version of a repl, clang-repl, which enables
  interpreter-like interaction for C++. For instance:

  ./bin/clang-repl
  clang-repl> int i = 42;
  clang-repl> extern "C" int printf(const char*,...);
  clang-repl> auto r1 = printf("i=%d\n", i);
  i=42
  clang-repl> quit

  The patch allows very limited functionality, for example, it crashes on invalid
  C++. The design of the proposed patch follows closely the design of cling. The
  idea is to gather feedback and gradually evolve both clang-repl and cling to
  what the community agrees upon.

  The IncrementalParser class is responsible for driving the clang parser and
  codegen and allows the compiler infrastructure to process more than one input.
  Every input adds to the “ever-growing” translation unit. That model is enabled
  by an IncrementalAction which prevents teardown when HandleTranslationUnit.

  The IncrementalExecutor class hides some of the underlying implementation
  details of the concrete JIT infrastructure. It exposes the minimal set of
  functionality required by our incremental compiler/interpreter.

  The Transaction class keeps track of the AST and the LLVM IR for each
  incremental input. That tracking information will be later used to implement
  error recovery.

  The Interpreter class orchestrates the IncrementalParser and the
  IncrementalExecutor to model interpreter-like behavior. It provides the public
  API which can be used (in future) when using the interpreter library.

  Differential revision: https://reviews.llvm.org/D96033

clang/include/clang/CodeGen/CodeGenAction.h

@@ -19,6 +19,7 @@ namespace llvm {
 
 namespace clang {
 class BackendConsumer;
+class CodeGenerator;
 
 class CodeGenAction : public ASTFrontendAction {
 private:
@@ -77,6 +78,8 @@ public:
   /// Take the LLVM context used by this action.
   llvm::LLVMContext *takeLLVMContext();
 
+  CodeGenerator *getCodeGenerator() const;
+
   BackendConsumer *BEConsumer;
 };
 

clang/include/clang/Frontend/FrontendAction.h

@@ -234,7 +234,7 @@ public:
 
   /// Perform any per-file post processing, deallocate per-file
   /// objects, and run statistics and output file cleanup code.
-  void EndSourceFile();
+  virtual void EndSourceFile();
 
   /// @}
 };
@@ -302,15 +302,16 @@ public:
 /// some existing action's behavior. It implements every virtual method in
 /// the FrontendAction interface by forwarding to the wrapped action.
 class WrapperFrontendAction : public FrontendAction {
+protected:
   std::unique_ptr<FrontendAction> WrappedAction;
 
-protected:
   bool PrepareToExecuteAction(CompilerInstance &CI) override;
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
                                                  StringRef InFile) override;
   bool BeginInvocation(CompilerInstance &CI) override;
   bool BeginSourceFileAction(CompilerInstance &CI) override;
   void ExecuteAction() override;
+  void EndSourceFile() override;
   void EndSourceFileAction() override;
   bool shouldEraseOutputFiles() override;
 

clang/include/clang/Interpreter/Interpreter.h

@@ -0,0 +1,71 @@
+//===--- Interpreter.h - Incremental Compilation and Execution---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the component which performs incremental code
+// compilation and execution.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_INTERPRETER_INTERPRETER_H
+#define LLVM_CLANG_INTERPRETER_INTERPRETER_H
+
+#include "clang/Interpreter/Transaction.h"
+
+#include "llvm/Support/Error.h"
+
+#include <memory>
+#include <vector>
+
+namespace llvm {
+namespace orc {
+class ThreadSafeContext;
+}
+class Module;
+} // namespace llvm
+
+namespace clang {
+
+class CompilerInstance;
+class DeclGroupRef;
+class IncrementalExecutor;
+class IncrementalParser;
+
+/// Create a pre-configured \c CompilerInstance for incremental processing.
+class IncrementalCompilerBuilder {
+public:
+  static llvm::Expected<std::unique_ptr<CompilerInstance>>
+  create(std::vector<const char *> &ClangArgv);
+};
+
+/// Provides top-level interfaces for incremental compilation and execution.
+class Interpreter {
+  std::unique_ptr<llvm::orc::ThreadSafeContext> TSCtx;
+  std::unique_ptr<IncrementalParser> IncrParser;
+  std::unique_ptr<IncrementalExecutor> IncrExecutor;
+
+  Interpreter(std::unique_ptr<CompilerInstance> CI, llvm::Error &Err);
+
+public:
+  ~Interpreter();
+  static llvm::Expected<std::unique_ptr<Interpreter>>
+  create(std::unique_ptr<CompilerInstance> CI);
+  const CompilerInstance *getCompilerInstance() const;
+  llvm::Expected<Transaction &> Parse(llvm::StringRef Code);
+  llvm::Error Execute(Transaction &T);
+  llvm::Error ParseAndExecute(llvm::StringRef Code) {
+    auto ErrOrTransaction = Parse(Code);
+    if (auto Err = ErrOrTransaction.takeError())
+      return Err;
+    if (ErrOrTransaction->TheModule)
+      return Execute(*ErrOrTransaction);
+    return llvm::Error::success();
+  }
+};
+} // namespace clang
+
+#endif // LLVM_CLANG_INTERPRETER_INTERPRETER_H

clang/include/clang/Interpreter/Transaction.h

@@ -0,0 +1,39 @@
+//===--- Transaction.h - Incremental Compilation and Execution---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines utilities tracking the incrementally processed pieces of
+// code.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_INTERPRETER_TRANSACTION_H
+#define LLVM_CLANG_INTERPRETER_TRANSACTION_H
+
+#include <memory>
+#include <vector>
+
+namespace llvm {
+class Module;
+}
+
+namespace clang {
+
+class DeclGroupRef;
+
+/// The class keeps track of various objects created as part of processing
+/// incremental inputs.
+struct Transaction {
+  /// The decls created for the input.
+  std::vector<clang::DeclGroupRef> Decls;
+
+  /// The llvm IR produced for the input.
+  std::unique_ptr<llvm::Module> TheModule;
+};
+} // namespace clang
+
+#endif // LLVM_CLANG_INTERPRETER_TRANSACTION_H

clang/lib/CMakeLists.txt

@@ -25,3 +25,4 @@ add_subdirectory(IndexSerialization)
 add_subdirectory(StaticAnalyzer)
 add_subdirectory(Format)
 add_subdirectory(Testing)
+add_subdirectory(Interpreter)

clang/lib/CodeGen/CodeGenAction.cpp

@@ -885,6 +885,10 @@ llvm::LLVMContext *CodeGenAction::takeLLVMContext() {
   return VMContext;
 }
 
+CodeGenerator *CodeGenAction::getCodeGenerator() const {
+  return BEConsumer->getCodeGenerator();
+}
+
 static std::unique_ptr<raw_pwrite_stream>
 GetOutputStream(CompilerInstance &CI, StringRef InFile, BackendAction Action) {
   switch (Action) {

clang/lib/Frontend/FrontendAction.cpp

@@ -1087,6 +1087,7 @@ bool WrapperFrontendAction::BeginSourceFileAction(CompilerInstance &CI) {
 void WrapperFrontendAction::ExecuteAction() {
   WrappedAction->ExecuteAction();
 }
+void WrapperFrontendAction::EndSourceFile() { WrappedAction->EndSourceFile(); }
 void WrapperFrontendAction::EndSourceFileAction() {
   WrappedAction->EndSourceFileAction();
 }

clang/lib/Interpreter/CMakeLists.txt

@@ -0,0 +1,23 @@
+set(LLVM_LINK_COMPONENTS
+   core
+   native
+   OrcJit
+   Support
+   Target
+  )
+
+add_clang_library(clangInterpreter
+  IncrementalExecutor.cpp
+  IncrementalParser.cpp
+  Interpreter.cpp
+
+  LINK_LIBS
+  clangAST
+  clangAnalysis
+  clangBasic
+  clangEdit
+  clangLex
+  clangSema
+  clangCodeGen
+  clangFrontendTool
+  )

clang/lib/Interpreter/IncrementalExecutor.cpp

@@ -0,0 +1,61 @@
+//===--- IncrementalExecutor.cpp - Incremental Execution --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the class which performs incremental code execution.
+//
+//===----------------------------------------------------------------------===//
+
+#include "IncrementalExecutor.h"
+
+#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
+#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
+#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
+#include "llvm/ExecutionEngine/Orc/LLJIT.h"
+#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/SectionMemoryManager.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/TargetSelect.h"
+
+namespace clang {
+
+IncrementalExecutor::IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC,
+                                         llvm::Error &Err)
+    : TSCtx(TSC) {
+  using namespace llvm::orc;
+  llvm::ErrorAsOutParameter EAO(&Err);
+
+  if (auto JitOrErr = LLJITBuilder().create())
+    Jit = std::move(*JitOrErr);
+  else {
+    Err = JitOrErr.takeError();
+    return;
+  }
+
+  const char Pref = Jit->getDataLayout().getGlobalPrefix();
+  // Discover symbols from the process as a fallback.
+  if (auto PSGOrErr = DynamicLibrarySearchGenerator::GetForCurrentProcess(Pref))
+    Jit->getMainJITDylib().addGenerator(std::move(*PSGOrErr));
+  else {
+    Err = PSGOrErr.takeError();
+    return;
+  }
+}
+
+IncrementalExecutor::~IncrementalExecutor() {}
+
+llvm::Error IncrementalExecutor::addModule(std::unique_ptr<llvm::Module> M) {
+  return Jit->addIRModule(llvm::orc::ThreadSafeModule(std::move(M), TSCtx));
+}
+
+llvm::Error IncrementalExecutor::runCtors() const {
+  return Jit->initialize(Jit->getMainJITDylib());
+}
+
+} // end namespace clang

clang/lib/Interpreter/IncrementalExecutor.h

@@ -0,0 +1,46 @@
+//===--- IncrementalExecutor.h - Incremental Execution ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the class which performs incremental code execution.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_INTERPRETER_INCREMENTALEXECUTOR_H
+#define LLVM_CLANG_LIB_INTERPRETER_INCREMENTALEXECUTOR_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
+
+#include <memory>
+
+namespace llvm {
+class Error;
+class Module;
+namespace orc {
+class LLJIT;
+class ThreadSafeContext;
+} // namespace orc
+} // namespace llvm
+
+namespace clang {
+class IncrementalExecutor {
+  using CtorDtorIterator = llvm::orc::CtorDtorIterator;
+  std::unique_ptr<llvm::orc::LLJIT> Jit;
+  llvm::orc::ThreadSafeContext &TSCtx;
+
+public:
+  IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, llvm::Error &Err);
+  ~IncrementalExecutor();
+
+  llvm::Error addModule(std::unique_ptr<llvm::Module> M);
+  llvm::Error runCtors() const;
+};
+
+} // end namespace clang
+
+#endif // LLVM_CLANG_LIB_INTERPRETER_INCREMENTALEXECUTOR_H

clang/lib/Interpreter/IncrementalParser.cpp

@@ -0,0 +1,254 @@
+//===--------- IncrementalParser.cpp - Incremental Compilation  -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the class which performs incremental code compilation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "IncrementalParser.h"
+
+#include "clang/CodeGen/BackendUtil.h"
+#include "clang/CodeGen/CodeGenAction.h"
+#include "clang/CodeGen/ModuleBuilder.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendAction.h"
+#include "clang/FrontendTool/Utils.h"
+#include "clang/Parse/Parser.h"
+#include "clang/Sema/Sema.h"
+
+#include "llvm/Option/ArgList.h"
+#include "llvm/Support/CrashRecoveryContext.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/Timer.h"
+
+#include <sstream>
+
+namespace clang {
+
+/// A custom action enabling the incremental processing functionality.
+///
+/// The usual \p FrontendAction expects one call to ExecuteAction and once it
+/// sees a call to \p EndSourceFile it deletes some of the important objects
+/// such as \p Preprocessor and \p Sema assuming no further input will come.
+///
+/// \p IncrementalAction ensures it keep its underlying action's objects alive
+/// as long as the \p IncrementalParser needs them.
+///
+class IncrementalAction : public WrapperFrontendAction {
+private:
+  bool IsTerminating = false;
+
+public:
+  IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
+                    llvm::Error &Err)
+      : WrapperFrontendAction([&]() {
+          llvm::ErrorAsOutParameter EAO(&Err);
+          std::unique_ptr<FrontendAction> Act;
+          switch (CI.getFrontendOpts().ProgramAction) {
+          default:
+            Err = llvm::createStringError(
+                std::errc::state_not_recoverable,
+                "Driver initialization failed. "
+                "Incremental mode for action is not supported");
+            return Act;
+          case frontend::ASTDump:
+            LLVM_FALLTHROUGH;
+          case frontend::ASTPrint:
+            LLVM_FALLTHROUGH;
+          case frontend::ParseSyntaxOnly:
+            Act = CreateFrontendAction(CI);
+            break;
+          case frontend::EmitObj:
+            LLVM_FALLTHROUGH;
+          case frontend::EmitLLVMOnly:
+            Act.reset(new EmitLLVMOnlyAction(&LLVMCtx));
+            break;
+          }
+          return Act;
+        }()) {}
+  FrontendAction *getWrapped() const { return WrappedAction.get(); }
+  void ExecuteAction() override {
+    CompilerInstance &CI = getCompilerInstance();
+    assert(CI.hasPreprocessor() && "No PP!");
+
+    // FIXME: Move the truncation aspect of this into Sema, we delayed this till
+    // here so the source manager would be initialized.
+    if (hasCodeCompletionSupport() &&
+        !CI.getFrontendOpts().CodeCompletionAt.FileName.empty())
+      CI.createCodeCompletionConsumer();
+
+    // Use a code completion consumer?
+    CodeCompleteConsumer *CompletionConsumer = nullptr;
+    if (CI.hasCodeCompletionConsumer())
+      CompletionConsumer = &CI.getCodeCompletionConsumer();
+
+    Preprocessor &PP = CI.getPreprocessor();
+    PP.enableIncrementalProcessing();
+    PP.EnterMainSourceFile();
+
+    if (!CI.hasSema())
+      CI.createSema(getTranslationUnitKind(), CompletionConsumer);
+  }
+
+  // Do not terminate after processing the input. This allows us to keep various
+  // clang objects alive and to incrementally grow the current TU.
+  void EndSourceFile() override {
+    // The WrappedAction can be nullptr if we issued an error in the ctor.
+    if (IsTerminating && getWrapped())
+      WrapperFrontendAction::EndSourceFile();
+  }
+
+  void FinalizeAction() {
+    assert(!IsTerminating && "Already finalized!");
+    IsTerminating = true;
+    EndSourceFile();
+  }
+};
+
+IncrementalParser::IncrementalParser(std::unique_ptr<CompilerInstance> Instance,
+                                     llvm::LLVMContext &LLVMCtx,
+                                     llvm::Error &Err)
+    : CI(std::move(Instance)) {
+  llvm::ErrorAsOutParameter EAO(&Err);
+  Act = std::make_unique<IncrementalAction>(*CI, LLVMCtx, Err);
+  if (Err)
+    return;
+  CI->ExecuteAction(*Act);
+  Consumer = &CI->getASTConsumer();
+  P.reset(
+      new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false));
+  P->Initialize();
+}
+
+IncrementalParser::~IncrementalParser() { Act->FinalizeAction(); }
+
+llvm::Expected<Transaction &> IncrementalParser::ParseOrWrapTopLevelDecl() {
+  DiagnosticsEngine &Diags = getCI()->getDiagnostics();
+
+  if (Diags.hasErrorOccurred())
+    llvm::report_fatal_error("Previous input had errors, "
+                             "recovery not yet supported",
+                             /*GenCrashDiag=*/false);
+
+  // Recover resources if we crash before exiting this method.
+  Sema &S = CI->getSema();
+  llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
+  Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true);
+  Sema::LocalEagerInstantiationScope LocalInstantiations(S);
+
+  // Skip previous eof due to last incremental input.
+  if (P->getCurToken().is(tok::eof))
+    P->ConsumeToken();
+
+  Transactions.emplace_back(Transaction());
+  Transaction &LastTransaction = Transactions.back();
+
+  Parser::DeclGroupPtrTy ADecl;
+  for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl); !AtEOF;
+       AtEOF = P->ParseTopLevelDecl(ADecl)) {
+    // If we got a null return and something *was* parsed, ignore it.  This
+    // is due to a top-level semicolon, an action override, or a parse error
+    // skipping something.
+    if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get()))
+      return llvm::make_error<llvm::StringError>("Parsing failed. "
+                                                 "The consumer rejected a decl",
+                                                 std::error_code());
+    LastTransaction.Decls.push_back(ADecl.get());
+  }
+
+  // Process any TopLevelDecls generated by #pragma weak.
+  for (Decl *D : S.WeakTopLevelDecls()) {
+    DeclGroupRef DGR(D);
+    LastTransaction.Decls.push_back(DGR);
+    Consumer->HandleTopLevelDecl(DGR);
+  }
+
+  LocalInstantiations.perform();
+  GlobalInstantiations.perform();
+
+  Consumer->HandleTranslationUnit(S.getASTContext());
+
+  if (Diags.hasErrorOccurred())
+    return llvm::make_error<llvm::StringError>("Parsing failed.",
+                                               std::error_code());
+
+  return LastTransaction;
+}
+
+static CodeGenerator *getCodeGen(FrontendAction *Act) {
+  IncrementalAction *IncrAct = static_cast<IncrementalAction *>(Act);
+  FrontendAction *WrappedAct = IncrAct->getWrapped();
+  if (!WrappedAct->hasIRSupport())
+    return nullptr;
+  return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
+}
+
+llvm::Expected<Transaction &> IncrementalParser::Parse(llvm::StringRef input) {
+  Preprocessor &PP = CI->getPreprocessor();
+  assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
+
+  std::ostringstream SourceName;
+  SourceName << "input_line_" << InputCount++;
+
+  // Create an uninitialized memory buffer, copy code in and append "\n"
+  size_t InputSize = input.size(); // don't include trailing 0
+  // MemBuffer size should *not* include terminating zero
+  std::unique_ptr<llvm::MemoryBuffer> MB(
+      llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
+                                                        SourceName.str()));
+  char *MBStart = const_cast<char *>(MB->getBufferStart());
+  memcpy(MBStart, input.data(), InputSize);
+  MBStart[InputSize] = '\n';
+
+  SourceManager &SM = CI->getSourceManager();
+
+  // FIXME: Create SourceLocation, which will allow clang to order the overload
+  // candidates for example
+  SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
+
+  // Create FileID for the current buffer.
+  FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
+                               /*LoadedOffset=*/0, NewLoc);
+
+  // NewLoc only used for diags.
+  if (PP.EnterSourceFile(FID, /*DirLookup=*/0, NewLoc))
+    return llvm::make_error<llvm::StringError>("Parsing failed. "
+                                               "Cannot enter source file.",
+                                               std::error_code());
+
+  auto ErrOrTransaction = ParseOrWrapTopLevelDecl();
+  if (auto Err = ErrOrTransaction.takeError())
+    return std::move(Err);
+
+  if (PP.getLangOpts().DelayedTemplateParsing) {
+    // Microsoft-specific:
+    // Late parsed templates can leave unswallowed "macro"-like tokens.
+    // They will seriously confuse the Parser when entering the next
+    // source file. So lex until we are EOF.
+    Token Tok;
+    do {
+      PP.Lex(Tok);
+    } while (Tok.isNot(tok::eof));
+  }
+
+  Token AssertTok;
+  PP.Lex(AssertTok);
+  assert(AssertTok.is(tok::eof) &&
+         "Lexer must be EOF when starting incremental parse!");
+
+  if (CodeGenerator *CG = getCodeGen(Act.get())) {
+    std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
+    CG->StartModule("incr_module_" + std::to_string(Transactions.size()),
+                    M->getContext());
+
+    ErrOrTransaction->TheModule = std::move(M);
+  }
+
+  return ErrOrTransaction;
+}
+} // end namespace clang

clang/lib/Interpreter/IncrementalParser.h

@@ -0,0 +1,77 @@
+//===--- IncrementalParser.h - Incremental Compilation ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the class which performs incremental code compilation.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_INTERPRETER_INCREMENTALPARSER_H
+#define LLVM_CLANG_LIB_INTERPRETER_INCREMENTALPARSER_H
+
+#include "clang/Interpreter/Transaction.h"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+
+#include <list>
+#include <memory>
+namespace llvm {
+class LLVMContext;
+}
+
+namespace clang {
+class ASTConsumer;
+class CompilerInstance;
+class CodeGenerator;
+class DeclGroupRef;
+class FrontendAction;
+class IncrementalAction;
+class Parser;
+
+/// Provides support for incremental compilation. Keeps track of the state
+/// changes between the subsequent incremental input.
+///
+class IncrementalParser {
+  /// Long-lived, incremental parsing action.
+  std::unique_ptr<IncrementalAction> Act;
+
+  /// Compiler instance performing the incremental compilation.
+  std::unique_ptr<CompilerInstance> CI;
+
+  /// Parser.
+  std::unique_ptr<Parser> P;
+
+  /// Consumer to process the produced top level decls. Owned by Act.
+  ASTConsumer *Consumer = nullptr;
+
+  /// Counts the number of direct user input lines that have been parsed.
+  unsigned InputCount = 0;
+
+  /// List containing every information about every incrementally parsed piece
+  /// of code.
+  std::list<Transaction> Transactions;
+
+public:
+  IncrementalParser(std::unique_ptr<CompilerInstance> Instance,
+                    llvm::LLVMContext &LLVMCtx, llvm::Error &Err);
+  ~IncrementalParser();
+
+  const CompilerInstance *getCI() const { return CI.get(); }
+
+  /// Parses incremental input by creating an in-memory file.
+  ///\returns a \c Transaction which holds information about the \c Decls and
+  /// \c llvm::Module corresponding to the input.
+  llvm::Expected<Transaction &> Parse(llvm::StringRef Input);
+
+private:
+  llvm::Expected<Transaction &> ParseOrWrapTopLevelDecl();
+};
+} // end namespace clang
+
+#endif // LLVM_CLANG_LIB_INTERPRETER_INCREMENTALPARSER_H

clang/lib/Interpreter/Interpreter.cpp

@@ -0,0 +1,220 @@
+//===------ Interpreter.cpp - Incremental Compilation and Execution -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the component which performs incremental code
+// compilation and execution.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Interpreter/Interpreter.h"
+
+#include "IncrementalExecutor.h"
+#include "IncrementalParser.h"
+
+#include "clang/Basic/TargetInfo.h"
+#include "clang/CodeGen/ModuleBuilder.h"
+#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
+#include "clang/Driver/Compilation.h"
+#include "clang/Driver/Driver.h"
+#include "clang/Driver/Job.h"
+#include "clang/Driver/Options.h"
+#include "clang/Driver/Tool.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/TextDiagnosticBuffer.h"
+#include "clang/Lex/PreprocessorOptions.h"
+
+#include "llvm/IR/Module.h"
+#include "llvm/Support/Host.h"
+
+using namespace clang;
+
+// FIXME: Figure out how to unify with namespace init_convenience from
+//        tools/clang-import-test/clang-import-test.cpp and
+//        examples/clang-interpreter/main.cpp
+namespace {
+/// Retrieves the clang CC1 specific flags out of the compilation's jobs.
+/// \returns NULL on error.
+static llvm::Expected<const llvm::opt::ArgStringList *>
+GetCC1Arguments(DiagnosticsEngine *Diagnostics,
+                driver::Compilation *Compilation) {
+  // We expect to get back exactly one Command job, if we didn't something
+  // failed. Extract that job from the Compilation.
+  const driver::JobList &Jobs = Compilation->getJobs();
+  if (!Jobs.size() || !isa<driver::Command>(*Jobs.begin()))
+    return llvm::createStringError(std::errc::state_not_recoverable,
+                                   "Driver initialization failed. "
+                                   "Unable to create a driver job");
+
+  // The one job we find should be to invoke clang again.
+  const driver::Command *Cmd = cast<driver::Command>(&(*Jobs.begin()));
+  if (llvm::StringRef(Cmd->getCreator().getName()) != "clang")
+    return llvm::createStringError(std::errc::state_not_recoverable,
+                                   "Driver initialization failed");
+
+  return &Cmd->getArguments();
+}
+
+static llvm::Expected<std::unique_ptr<CompilerInstance>>
+CreateCI(const llvm::opt::ArgStringList &Argv) {
+  std::unique_ptr<CompilerInstance> Clang(new CompilerInstance());
+  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
+
+  // Register the support for object-file-wrapped Clang modules.
+  // FIXME: Clang should register these container operations automatically.
+  auto PCHOps = Clang->getPCHContainerOperations();
+  PCHOps->registerWriter(std::make_unique<ObjectFilePCHContainerWriter>());
+  PCHOps->registerReader(std::make_unique<ObjectFilePCHContainerReader>());
+
+  // Buffer diagnostics from argument parsing so that we can output them using
+  // a well formed diagnostic object.
+  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
+  TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer;
+  DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer);
+  bool Success = CompilerInvocation::CreateFromArgs(
+      Clang->getInvocation(), llvm::makeArrayRef(Argv.begin(), Argv.size()),
+      Diags);
+
+  // Infer the builtin include path if unspecified.
+  if (Clang->getHeaderSearchOpts().UseBuiltinIncludes &&
+      Clang->getHeaderSearchOpts().ResourceDir.empty())
+    Clang->getHeaderSearchOpts().ResourceDir =
+        CompilerInvocation::GetResourcesPath(Argv[0], nullptr);
+
+  // Create the actual diagnostics engine.
+  Clang->createDiagnostics();
+  if (!Clang->hasDiagnostics())
+    return llvm::createStringError(std::errc::state_not_recoverable,
+                                   "Initialization failed. "
+                                   "Unable to create diagnostics engine");
+
+  DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics());
+  if (!Success)
+    return llvm::createStringError(std::errc::state_not_recoverable,
+                                   "Initialization failed. "
+                                   "Unable to flush diagnostics");
+
+  // FIXME: Merge with CompilerInstance::ExecuteAction.
+  llvm::MemoryBuffer *MB = llvm::MemoryBuffer::getMemBuffer("").release();
+  Clang->getPreprocessorOpts().addRemappedFile("<<< inputs >>>", MB);
+
+  Clang->setTarget(TargetInfo::CreateTargetInfo(
+      Clang->getDiagnostics(), Clang->getInvocation().TargetOpts));
+  if (!Clang->hasTarget())
+    return llvm::createStringError(std::errc::state_not_recoverable,
+                                   "Initialization failed. "
+                                   "Target is missing");
+
+  Clang->getTarget().adjust(Clang->getLangOpts());
+
+  return std::move(Clang);
+}
+
+} // anonymous namespace
+
+llvm::Expected<std::unique_ptr<CompilerInstance>>
+IncrementalCompilerBuilder::create(std::vector<const char *> &ClangArgv) {
+
+  // If we don't know ClangArgv0 or the address of main() at this point, try
+  // to guess it anyway (it's possible on some platforms).
+  std::string MainExecutableName =
+      llvm::sys::fs::getMainExecutable(nullptr, nullptr);
+
+  ClangArgv.insert(ClangArgv.begin(), MainExecutableName.c_str());
+
+  // Prepending -c to force the driver to do something if no action was
+  // specified. By prepending we allow users to override the default
+  // action and use other actions in incremental mode.
+  // FIXME: Print proper driver diagnostics if the driver flags are wrong.
+  ClangArgv.insert(ClangArgv.begin() + 1, "-c");
+
+  if (!llvm::is_contained(ClangArgv, " -x")) {
+    // We do C++ by default; append right after argv[0] if no "-x" given
+    ClangArgv.push_back("-x");
+    ClangArgv.push_back("c++");
+  }
+
+  // Put a dummy C++ file on to ensure there's at least one compile job for the
+  // driver to construct.
+  ClangArgv.push_back("<<< inputs >>>");
+
+  CompilerInvocation Invocation;
+  // Buffer diagnostics from argument parsing so that we can output them using a
+  // well formed diagnostic object.
+  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
+  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
+  TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer;
+  DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer);
+  unsigned MissingArgIndex, MissingArgCount;
+  const llvm::opt::OptTable &Opts = driver::getDriverOptTable();
+  llvm::opt::InputArgList ParsedArgs =
+      Opts.ParseArgs(ArrayRef<const char *>(ClangArgv).slice(1),
+                     MissingArgIndex, MissingArgCount);
+  ParseDiagnosticArgs(*DiagOpts, ParsedArgs, &Diags);
+
+  driver::Driver Driver(/*MainBinaryName=*/ClangArgv[0],
+                        llvm::sys::getDefaultTargetTriple(), Diags);
+  Driver.setCheckInputsExist(false); // the input comes from mem buffers
+  llvm::ArrayRef<const char *> RF = llvm::makeArrayRef(ClangArgv);
+  std::unique_ptr<driver::Compilation> Compilation(Driver.BuildCompilation(RF));
+
+  if (Compilation->getArgs().hasArg(driver::options::OPT_v))
+    Compilation->getJobs().Print(llvm::errs(), "\n", /*Quote=*/false);
+
+  auto ErrOrCC1Args = GetCC1Arguments(&Diags, Compilation.get());
+  if (auto Err = ErrOrCC1Args.takeError())
+    return std::move(Err);
+
+  return CreateCI(**ErrOrCC1Args);
+}
+
+Interpreter::Interpreter(std::unique_ptr<CompilerInstance> CI,
+                         llvm::Error &Err) {
+  llvm::ErrorAsOutParameter EAO(&Err);
+  auto LLVMCtx = std::make_unique<llvm::LLVMContext>();
+  TSCtx = std::make_unique<llvm::orc::ThreadSafeContext>(std::move(LLVMCtx));
+  IncrParser = std::make_unique<IncrementalParser>(std::move(CI),
+                                                   *TSCtx->getContext(), Err);
+}
+
+Interpreter::~Interpreter() {}
+
+llvm::Expected<std::unique_ptr<Interpreter>>
+Interpreter::create(std::unique_ptr<CompilerInstance> CI) {
+  llvm::Error Err = llvm::Error::success();
+  auto Interp =
+      std::unique_ptr<Interpreter>(new Interpreter(std::move(CI), Err));
+  if (Err)
+    return std::move(Err);
+  return std::move(Interp);
+}
+
+const CompilerInstance *Interpreter::getCompilerInstance() const {
+  return IncrParser->getCI();
+}
+
+llvm::Expected<Transaction &> Interpreter::Parse(llvm::StringRef Code) {
+  return IncrParser->Parse(Code);
+}
+
+llvm::Error Interpreter::Execute(Transaction &T) {
+  assert(T.TheModule);
+  if (!IncrExecutor) {
+    llvm::Error Err = llvm::Error::success();
+    IncrExecutor = std::make_unique<IncrementalExecutor>(*TSCtx, Err);
+    if (Err)
+      return Err;
+  }
+  // FIXME: Add a callback to retain the llvm::Module once the JIT is done.
+  if (auto Err = IncrExecutor->addModule(std::move(T.TheModule)))
+    return Err;
+
+  if (auto Err = IncrExecutor->runCtors())
+    return Err;
+
+  return llvm::Error::success();
+}

clang/test/CMakeLists.txt

@@ -68,6 +68,7 @@ list(APPEND CLANG_TEST_DEPS
   clang-import-test
   clang-rename
   clang-refactor
+  clang-repl
   clang-diff
   clang-scan-deps
   diagtool

clang/test/Interpreter/execute.cpp

@@ -0,0 +1,14 @@
+// RUN: cat %s | clang-repl | FileCheck %s
+// REQUIRES: host-supports-jit
+
+extern "C" int printf(const char *, ...);
+int i = 42;
+auto r1 = printf("i = %d\n", i);
+// CHECK: i = 42
+
+struct S { float f = 1.0; S *m = nullptr;} s;
+
+auto r2 = printf("S[f=%f, m=0x%llx]\n", s.f, reinterpret_cast<unsigned long long>(s.m));
+// CHECK-NEXT: S[f=1.000000, m=0x0]
+
+quit

clang/test/Interpreter/sanity.c

@@ -0,0 +1,18 @@
+// RUN: cat %s | \
+// RUN:   clang-repl -Xcc -fno-color-diagnostics -Xcc -Xclang -Xcc -ast-dump \
+// RUN:            -Xcc -Xclang -Xcc -ast-dump-filter -Xcc -Xclang -Xcc Test 2>&1| \
+// RUN:         FileCheck %s
+
+int TestVar = 12;
+// CHECK: Dumping TestVar:
+// CHECK-NEXT: VarDecl [[var_ptr:0x[0-9a-f]+]] <{{.*}} TestVar 'int' cinit
+// CHECK-NEXT:   IntegerLiteral {{.*}} 'int' 12
+
+void TestFunc() { ++TestVar; }
+// CHECK: Dumping TestFunc:
+// CHECK-NEXT: FunctionDecl {{.*}} TestFunc 'void ()'
+// CHECK-NEXT:   CompoundStmt{{.*}}
+// CHECK-NEXT:     UnaryOperator{{.*}} 'int' lvalue prefix '++'
+// CHECK-NEXT:       DeclRefExpr{{.*}} 'int' lvalue Var [[var_ptr]] 'TestVar' 'int'
+
+quit

clang/test/lit.cfg.py

@@ -63,7 +63,7 @@ config.substitutions.append(('%PATH%', config.environment['PATH']))
 tool_dirs = [config.clang_tools_dir, config.llvm_tools_dir]
 
 tools = [
-    'apinotes-test', 'c-index-test', 'clang-diff', 'clang-format',
+    'apinotes-test', 'c-index-test', 'clang-diff', 'clang-format', 'clang-repl',
     'clang-tblgen', 'opt', 'llvm-ifs', 'yaml2obj',
     ToolSubst('%clang_extdef_map', command=FindTool(
         'clang-extdef-mapping'), unresolved='ignore'),
@@ -73,6 +73,28 @@ if config.clang_examples:
     config.available_features.add('examples')
     tools.append('clang-interpreter')
 
+def have_host_jit_support():
+    clang_repl_exe = lit.util.which('clang-repl', config.clang_tools_dir)
+
+    if not clang_repl_exe:
+        print('clang-repl not found')
+        return False
+
+    try:
+        clang_repl_cmd = subprocess.Popen(
+            [clang_repl_exe, '--host-supports-jit'], stdout=subprocess.PIPE)
+    except OSError:
+        print('could not exec clang-repl')
+        return False
+
+    clang_repl_out = clang_repl_cmd.stdout.read().decode('ascii')
+    clang_repl_cmd.wait()
+
+    return 'true' in clang_repl_out
+
+if have_host_jit_support():
+    config.available_features.add('host-supports-jit')
+
 if config.clang_staticanalyzer:
     config.available_features.add('staticanalyzer')
     tools.append('clang-check')

clang/tools/CMakeLists.txt

@@ -11,6 +11,7 @@ add_clang_subdirectory(clang-import-test)
 add_clang_subdirectory(clang-offload-bundler)
 add_clang_subdirectory(clang-offload-wrapper)
 add_clang_subdirectory(clang-scan-deps)
+add_clang_subdirectory(clang-repl)
 
 add_clang_subdirectory(c-index-test)
 

clang/tools/clang-repl/CMakeLists.txt

@@ -0,0 +1,16 @@
+set( LLVM_LINK_COMPONENTS
+  ${LLVM_TARGETS_TO_BUILD}
+  Option
+  Support
+  )
+
+add_clang_executable(clang-repl
+  EXCLUDE_FROM_ALL
+  ClangRepl.cpp
+  )
+
+target_link_libraries(clang-repl PUBLIC
+  clangInterpreter
+  clangTooling
+  LLVMLineEditor
+  )

clang/tools/clang-repl/ClangRepl.cpp

@@ -0,0 +1,98 @@
+//===--- tools/clang-repl/ClangRepl.cpp - clang-repl - the Clang REPL -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements a REPL tool on top of clang.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendDiagnostic.h"
+#include "clang/Interpreter/Interpreter.h"
+
+#include "llvm/ExecutionEngine/Orc/LLJIT.h"
+#include "llvm/LineEditor/LineEditor.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ManagedStatic.h" // llvm_shutdown
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/TargetSelect.h" // llvm::Initialize*
+
+static llvm::cl::list<std::string>
+    ClangArgs("Xcc", llvm::cl::ZeroOrMore,
+              llvm::cl::desc("Argument to pass to the CompilerInvocation"),
+              llvm::cl::CommaSeparated);
+static llvm::cl::opt<bool> OptHostSupportsJit("host-supports-jit",
+                                              llvm::cl::Hidden);
+
+static void LLVMErrorHandler(void *UserData, const std::string &Message,
+                             bool GenCrashDiag) {
+  auto &Diags = *static_cast<clang::DiagnosticsEngine *>(UserData);
+
+  Diags.Report(clang::diag::err_fe_error_backend) << Message;
+
+  // Run the interrupt handlers to make sure any special cleanups get done, in
+  // particular that we remove files registered with RemoveFileOnSignal.
+  llvm::sys::RunInterruptHandlers();
+
+  // We cannot recover from llvm errors.  When reporting a fatal error, exit
+  // with status 70 to generate crash diagnostics.  For BSD systems this is
+  // defined as an internal software error. Otherwise, exit with status 1.
+
+  exit(GenCrashDiag ? 70 : 1);
+}
+
+llvm::ExitOnError ExitOnErr;
+int main(int argc, const char **argv) {
+  ExitOnErr.setBanner("clang-repl: ");
+  llvm::cl::ParseCommandLineOptions(argc, argv);
+
+  std::vector<const char *> ClangArgv(ClangArgs.size());
+  std::transform(ClangArgs.begin(), ClangArgs.end(), ClangArgv.begin(),
+                 [](const std::string &s) -> const char * { return s.data(); });
+  llvm::InitializeNativeTarget();
+  llvm::InitializeNativeTargetAsmPrinter();
+
+  if (OptHostSupportsJit) {
+    auto J = llvm::orc::LLJITBuilder().create();
+    if (J)
+      llvm::outs() << "true\n";
+    else {
+      llvm::consumeError(J.takeError());
+      llvm::outs() << "false\n";
+    }
+    return 0;
+  }
+
+  // FIXME: Investigate if we could use runToolOnCodeWithArgs from tooling. It
+  // can replace the boilerplate code for creation of the compiler instance.
+  auto CI = ExitOnErr(clang::IncrementalCompilerBuilder::create(ClangArgv));
+
+  // Set an error handler, so that any LLVM backend diagnostics go through our
+  // error handler.
+  llvm::install_fatal_error_handler(LLVMErrorHandler,
+                                    static_cast<void *>(&CI->getDiagnostics()));
+
+  auto Interp = ExitOnErr(clang::Interpreter::create(std::move(CI)));
+  llvm::LineEditor LE("clang-repl");
+  // FIXME: Add LE.setListCompleter
+  while (llvm::Optional<std::string> Line = LE.readLine()) {
+    if (*Line == "quit")
+      break;
+    if (auto Err = Interp->ParseAndExecute(*Line))
+      llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: ");
+  }
+
+  // Our error handler depends on the Diagnostics object, which we're
+  // potentially about to delete. Uninstall the handler now so that any
+  // later errors use the default handling behavior instead.
+  llvm::remove_fatal_error_handler();
+
+  llvm::llvm_shutdown();
+
+  return 0;
+}

clang/unittests/CMakeLists.txt

@@ -35,6 +35,7 @@ add_subdirectory(Frontend)
 add_subdirectory(Rewrite)
 add_subdirectory(Sema)
 add_subdirectory(CodeGen)
+add_subdirectory(Interpreter)
 # FIXME: libclang unit tests are disabled on Windows due
 # to failures, mostly in libclang.VirtualFileOverlay_*.
 if(NOT WIN32 AND CLANG_TOOL_LIBCLANG_BUILD) 

clang/unittests/CodeGen/CMakeLists.txt

@@ -6,7 +6,6 @@ set(LLVM_LINK_COMPONENTS
 add_clang_unittest(ClangCodeGenTests
   BufferSourceTest.cpp
   CodeGenExternalTest.cpp
-  IncrementalProcessingTest.cpp
   TBAAMetadataTest.cpp
   CheckTargetFeaturesTest.cpp
   )
@@ -17,6 +16,7 @@ clang_target_link_libraries(ClangCodeGenTests
   clangBasic
   clangCodeGen
   clangFrontend
+  clangInterpreter
   clangLex
   clangParse
   clangSerialization

clang/unittests/CodeGen/IncrementalProcessingTest.cpp

@@ -1,155 +0,0 @@
-//=== unittests/CodeGen/IncrementalProcessingTest.cpp - IncrementalCodeGen ===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "TestCompiler.h"
-
-#include "clang/AST/ASTConsumer.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/AST/RecursiveASTVisitor.h"
-#include "clang/Basic/TargetInfo.h"
-#include "clang/CodeGen/ModuleBuilder.h"
-#include "clang/Frontend/CompilerInstance.h"
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Parse/Parser.h"
-#include "clang/Sema/Sema.h"
-#include "llvm/ADT/Triple.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Support/Host.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "gtest/gtest.h"
-
-#include <memory>
-
-using namespace llvm;
-using namespace clang;
-
-namespace {
-
-// Incremental processing produces several modules, all using the same "main
-// file". Make sure CodeGen can cope with that, e.g. for static initializers.
-const char TestProgram1[] =
-    "extern \"C\" int funcForProg1() { return 17; }\n"
-    "struct EmitCXXGlobalInitFunc1 {\n"
-    "   EmitCXXGlobalInitFunc1() {}\n"
-    "} test1;";
-
-const char TestProgram2[] =
-    "extern \"C\" int funcForProg2() { return 42; }\n"
-    "struct EmitCXXGlobalInitFunc2 {\n"
-    "   EmitCXXGlobalInitFunc2() {}\n"
-    "} test2;";
-
-
-/// An incremental version of ParseAST().
-static std::unique_ptr<llvm::Module>
-IncrementalParseAST(CompilerInstance& CI, Parser& P,
-                    CodeGenerator& CG, const char* code) {
-  static int counter = 0;
-  struct IncreaseCounterOnRet {
-    ~IncreaseCounterOnRet() {
-      ++counter;
-    }
-  } ICOR;
-
-  Sema& S = CI.getSema();
-  clang::SourceManager &SM = S.getSourceManager();
-  if (!code) {
-    // Main file
-    SM.setMainFileID(SM.createFileID(
-        llvm::MemoryBuffer::getMemBuffer("    "), clang::SrcMgr::C_User));
-
-    S.getPreprocessor().EnterMainSourceFile();
-    P.Initialize();
-  } else {
-    FileID FID = SM.createFileID(
-        llvm::MemoryBuffer::getMemBuffer(code), clang::SrcMgr::C_User);
-    SourceLocation MainStartLoc = SM.getLocForStartOfFile(SM.getMainFileID());
-    SourceLocation InclLoc = MainStartLoc.getLocWithOffset(counter);
-    S.getPreprocessor().EnterSourceFile(FID, 0, InclLoc);
-  }
-
-  ExternalASTSource *External = S.getASTContext().getExternalSource();
-  if (External)
-    External->StartTranslationUnit(&CG);
-
-  Parser::DeclGroupPtrTy ADecl;
-  for (bool AtEOF = P.ParseFirstTopLevelDecl(ADecl); !AtEOF;
-       AtEOF = P.ParseTopLevelDecl(ADecl)) {
-    // If we got a null return and something *was* parsed, ignore it.  This
-    // is due to a top-level semicolon, an action override, or a parse error
-    // skipping something.
-    if (ADecl && !CG.HandleTopLevelDecl(ADecl.get()))
-      return nullptr;
-  }
-
-  // Process any TopLevelDecls generated by #pragma weak.
-  for (Decl *D : S.WeakTopLevelDecls())
-    CG.HandleTopLevelDecl(DeclGroupRef(D));
-
-  CG.HandleTranslationUnit(S.getASTContext());
-
-  std::unique_ptr<llvm::Module> M(CG.ReleaseModule());
-  // Switch to next module.
-  CG.StartModule("incremental-module-" + std::to_string(counter),
-                 M->getContext());
-  return M;
-}
-
-const Function* getGlobalInit(llvm::Module& M) {
-  for (const auto& Func: M)
-    if (Func.hasName() && Func.getName().startswith("_GLOBAL__sub_I_"))
-      return &Func;
-
-  return nullptr;
-}
-
-TEST(IncrementalProcessing, EmitCXXGlobalInitFunc) {
-    clang::LangOptions LO;
-    LO.CPlusPlus = 1;
-    LO.CPlusPlus11 = 1;
-    TestCompiler Compiler(LO);
-    clang::CompilerInstance &CI = Compiler.compiler;
-    CI.getPreprocessor().enableIncrementalProcessing();
-    CI.setASTConsumer(std::move(Compiler.CG));
-    clang::CodeGenerator& CG =
-      static_cast<clang::CodeGenerator&>(CI.getASTConsumer());
-    CI.createSema(clang::TU_Prefix, nullptr);
-
-    Sema& S = CI.getSema();
-
-    std::unique_ptr<Parser> ParseOP(new Parser(S.getPreprocessor(), S,
-                                               /*SkipFunctionBodies*/ false));
-    Parser &P = *ParseOP.get();
-
-    std::array<std::unique_ptr<llvm::Module>, 3> M;
-    M[0] = IncrementalParseAST(CI, P, CG, nullptr);
-    ASSERT_TRUE(M[0]);
-
-    M[1] = IncrementalParseAST(CI, P, CG, TestProgram1);
-    ASSERT_TRUE(M[1]);
-    ASSERT_TRUE(M[1]->getFunction("funcForProg1"));
-
-    M[2] = IncrementalParseAST(CI, P, CG, TestProgram2);
-    ASSERT_TRUE(M[2]);
-    ASSERT_TRUE(M[2]->getFunction("funcForProg2"));
-    // First code should not end up in second module:
-    ASSERT_FALSE(M[2]->getFunction("funcForProg1"));
-
-    // Make sure global inits exist and are unique:
-    const Function* GlobalInit1 = getGlobalInit(*M[1]);
-    ASSERT_TRUE(GlobalInit1);
-
-    const Function* GlobalInit2 = getGlobalInit(*M[2]);
-    ASSERT_TRUE(GlobalInit2);
-
-    ASSERT_FALSE(GlobalInit1->getName() == GlobalInit2->getName());
-
-}
-
-} // end anonymous namespace

clang/unittests/Interpreter/CMakeLists.txt

@@ -0,0 +1,11 @@
+set(LLVM_LINK_COMPONENTS
+  )
+
+add_clang_unittest(ClangReplInterpreterTests
+  IncrementalProcessingTest.cpp
+  InterpreterTest.cpp
+  )
+target_link_libraries(ClangReplInterpreterTests PUBLIC
+  clangInterpreter
+  clangFrontend
+  )

clang/unittests/Interpreter/IncrementalProcessingTest.cpp

@@ -0,0 +1,80 @@
+//=== unittests/CodeGen/IncrementalProcessingTest.cpp - IncrementalCodeGen ===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/ASTConsumer.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/CodeGen/ModuleBuilder.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Interpreter/Interpreter.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Parse/Parser.h"
+#include "clang/Sema/Sema.h"
+#include "llvm/ADT/Triple.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/Host.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "gtest/gtest.h"
+
+#include <memory>
+
+using namespace llvm;
+using namespace clang;
+
+namespace {
+
+// Incremental processing produces several modules, all using the same "main
+// file". Make sure CodeGen can cope with that, e.g. for static initializers.
+const char TestProgram1[] = "extern \"C\" int funcForProg1() { return 17; }\n"
+                            "struct EmitCXXGlobalInitFunc1 {\n"
+                            "   EmitCXXGlobalInitFunc1() {}\n"
+                            "} test1;";
+
+const char TestProgram2[] = "extern \"C\" int funcForProg2() { return 42; }\n"
+                            "struct EmitCXXGlobalInitFunc2 {\n"
+                            "   EmitCXXGlobalInitFunc2() {}\n"
+                            "} test2;";
+
+const Function *getGlobalInit(llvm::Module *M) {
+  for (const auto &Func : *M)
+    if (Func.hasName() && Func.getName().startswith("_GLOBAL__sub_I_"))
+      return &Func;
+
+  return nullptr;
+}
+
+TEST(IncrementalProcessing, EmitCXXGlobalInitFunc) {
+  std::vector<const char *> ClangArgv = {"-Xclang", "-emit-llvm-only"};
+  auto CI = llvm::cantFail(IncrementalCompilerBuilder::create(ClangArgv));
+  auto Interp = llvm::cantFail(Interpreter::create(std::move(CI)));
+
+  std::array<clang::Transaction *, 2> Transactions;
+
+  Transactions[0] = &llvm::cantFail(Interp->Parse(TestProgram1));
+  ASSERT_TRUE(Transactions[0]->TheModule);
+  ASSERT_TRUE(Transactions[0]->TheModule->getFunction("funcForProg1"));
+
+  Transactions[1] = &llvm::cantFail(Interp->Parse(TestProgram2));
+  ASSERT_TRUE(Transactions[1]->TheModule);
+  ASSERT_TRUE(Transactions[1]->TheModule->getFunction("funcForProg2"));
+  // First code should not end up in second module:
+  ASSERT_FALSE(Transactions[1]->TheModule->getFunction("funcForProg1"));
+
+  // Make sure global inits exist and are unique:
+  const Function *GlobalInit1 = getGlobalInit(Transactions[0]->TheModule.get());
+  ASSERT_TRUE(GlobalInit1);
+
+  const Function *GlobalInit2 = getGlobalInit(Transactions[1]->TheModule.get());
+  ASSERT_TRUE(GlobalInit2);
+
+  ASSERT_FALSE(GlobalInit1->getName() == GlobalInit2->getName());
+}
+
+} // end anonymous namespace

clang/unittests/Interpreter/InterpreterTest.cpp

@@ -0,0 +1,122 @@
+//===- unittests/Interpreter/InterpreterTest.cpp --- Interpreter tests ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Unit tests for Clang's Interpreter library.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Interpreter/Interpreter.h"
+
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclGroup.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/TextDiagnosticPrinter.h"
+
+#include "llvm/ADT/ArrayRef.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using namespace clang;
+
+namespace {
+using Args = std::vector<const char *>;
+static std::unique_ptr<Interpreter>
+createInterpreter(const Args &ExtraArgs = {},
+                  DiagnosticConsumer *Client = nullptr) {
+  Args ClangArgs = {"-Xclang", "-emit-llvm-only"};
+  ClangArgs.insert(ClangArgs.end(), ExtraArgs.begin(), ExtraArgs.end());
+  auto CI = cantFail(clang::IncrementalCompilerBuilder::create(ClangArgs));
+  if (Client)
+    CI->getDiagnostics().setClient(Client, /*ShouldOwnClient=*/false);
+  return cantFail(clang::Interpreter::create(std::move(CI)));
+}
+
+TEST(InterpreterTest, Sanity) {
+  std::unique_ptr<Interpreter> Interp = createInterpreter();
+  Transaction &R1(cantFail(Interp->Parse("void g(); void g() {}")));
+  EXPECT_EQ(2U, R1.Decls.size());
+
+  Transaction &R2(cantFail(Interp->Parse("int i;")));
+  EXPECT_EQ(1U, R2.Decls.size());
+}
+
+static std::string DeclToString(DeclGroupRef DGR) {
+  return llvm::cast<NamedDecl>(DGR.getSingleDecl())->getQualifiedNameAsString();
+}
+
+TEST(InterpreterTest, IncrementalInputTopLevelDecls) {
+  std::unique_ptr<Interpreter> Interp = createInterpreter();
+  auto R1OrErr = Interp->Parse("int var1 = 42; int f() { return var1; }");
+  // gtest doesn't expand into explicit bool conversions.
+  EXPECT_TRUE(!!R1OrErr);
+  auto R1 = R1OrErr->Decls;
+  EXPECT_EQ(2U, R1.size());
+  EXPECT_EQ("var1", DeclToString(R1[0]));
+  EXPECT_EQ("f", DeclToString(R1[1]));
+
+  auto R2OrErr = Interp->Parse("int var2 = f();");
+  EXPECT_TRUE(!!R2OrErr);
+  auto R2 = R2OrErr->Decls;
+  EXPECT_EQ(1U, R2.size());
+  EXPECT_EQ("var2", DeclToString(R2[0]));
+}
+
+TEST(InterpreterTest, Errors) {
+  Args ExtraArgs = {"-Xclang", "-diagnostic-log-file", "-Xclang", "-"};
+
+  // Create the diagnostic engine with unowned consumer.
+  std::string DiagnosticOutput;
+  llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
+  auto DiagPrinter = std::make_unique<TextDiagnosticPrinter>(
+      DiagnosticsOS, new DiagnosticOptions());
+
+  auto Interp = createInterpreter(ExtraArgs, DiagPrinter.get());
+  auto Err = Interp->Parse("intentional_error v1 = 42; ").takeError();
+  using ::testing::HasSubstr;
+  EXPECT_THAT(DiagnosticsOS.str(),
+              HasSubstr("error: unknown type name 'intentional_error'"));
+  EXPECT_EQ("Parsing failed.", llvm::toString(std::move(Err)));
+
+#ifdef GTEST_HAS_DEATH_TEST
+  EXPECT_DEATH((void)Interp->Parse("int var1 = 42;"), "");
+#endif
+}
+
+// Here we test whether the user can mix declarations and statements. The
+// interpreter should be smart enough to recognize the declarations from the
+// statements and wrap the latter into a declaration, producing valid code.
+TEST(InterpreterTest, DeclsAndStatements) {
+  Args ExtraArgs = {"-Xclang", "-diagnostic-log-file", "-Xclang", "-"};
+
+  // Create the diagnostic engine with unowned consumer.
+  std::string DiagnosticOutput;
+  llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
+  auto DiagPrinter = std::make_unique<TextDiagnosticPrinter>(
+      DiagnosticsOS, new DiagnosticOptions());
+
+  auto Interp = createInterpreter(ExtraArgs, DiagPrinter.get());
+  auto R1OrErr = Interp->Parse(
+      "int var1 = 42; extern \"C\" int printf(const char*, ...);");
+  // gtest doesn't expand into explicit bool conversions.
+  EXPECT_TRUE(!!R1OrErr);
+
+  auto R1 = R1OrErr->Decls;
+  EXPECT_EQ(2U, R1.size());
+
+  // FIXME: Add support for wrapping and running statements.
+  auto R2OrErr = Interp->Parse("var1++; printf(\"var1 value %d\\n\", var1);");
+  EXPECT_FALSE(!!R2OrErr);
+  using ::testing::HasSubstr;
+  EXPECT_THAT(DiagnosticsOS.str(),
+              HasSubstr("error: unknown type name 'var1'"));
+  auto Err = R2OrErr.takeError();
+  EXPECT_EQ("Parsing failed.", llvm::toString(std::move(Err)));
+}
+
+} // end anonymous namespace