[Flang][Driver] Add PrintPreprocessedInput FrontendAction (flang-new -E)

This patch implements the first frontend action for the Flang parser (i.e.
Fortran::parser). This action runs the preprocessor and is invoked with the
`-E` flag. (i.e. `flang-new -E <input-file>). The generated output is printed
to either stdout or the output file (specified with `-` or `-o <output-file>`).

Note that currently there is no mechanism to map options for the
frontend driver (i.e. Fortran::frontend::FrontendOptions) to options for
the parser (i.e. Fortran::parser::Options). Instead,
Frotran::parser::options are hard-coded to:

```
std::vector<std::string> searchDirectories{"."s};
searchDirectories = searchDirectories;
isFixedForm = false;
_encoding(Fortran::parser::Encoding::UTF_8);
```

These default settings are compatible with the current Flang driver. Further
work is required in order for CompilerInvocation to read and map
clang::driver::options to Fortran::parser::options.

Co-authored-by: Andrzej Warzynski <andrzej.warzynski@arm.com>

Differential Revision: https://reviews.llvm.org/D88381

clang/include/clang/Driver/Options.td

@@ -391,7 +391,7 @@ def C : Flag<["-"], "C">, Flags<[CC1Option]>, Group<Preprocessor_Group>,
 def D : JoinedOrSeparate<["-"], "D">, Group<Preprocessor_Group>,
     Flags<[CC1Option]>, MetaVarName<"<macro>=<value>">,
     HelpText<"Define <macro> to <value> (or 1 if <value> omitted)">;
-def E : Flag<["-"], "E">, Flags<[NoXarchOption,CC1Option]>, Group<Action_Group>,
+def E : Flag<["-"], "E">, Flags<[NoXarchOption,CC1Option, FlangOption, FC1Option]>, Group<Action_Group>,
     HelpText<"Only run the preprocessor">;
 def F : JoinedOrSeparate<["-"], "F">, Flags<[RenderJoined,CC1Option]>,
     HelpText<"Add directory to framework include search path">;

flang/include/flang/Frontend/CompilerInstance.h

@@ -10,12 +10,10 @@
 
 #include "flang/Frontend/CompilerInvocation.h"
 #include "flang/Frontend/FrontendAction.h"
+#include "flang/Parser/parsing.h"
 #include "flang/Parser/provenance.h"
 #include "llvm/Support/raw_ostream.h"
 
-#include <cassert>
-#include <memory>
-
 namespace Fortran::frontend {
 
 class CompilerInstance {
@@ -26,6 +24,10 @@ class CompilerInstance {
   /// Flang file  manager.
   std::shared_ptr<Fortran::parser::AllSources> allSources_;
 
+  std::shared_ptr<Fortran::parser::AllCookedSources> allCookedSources_;
+
+  std::shared_ptr<Fortran::parser::Parsing> parsing_;
+
   /// The diagnostics engine instance.
   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_;
 
@@ -75,6 +77,13 @@ public:
 
   bool HasAllSources() const { return allSources_ != nullptr; }
 
+  /// }
+  /// @name Parser Operations
+  /// {
+
+  /// Return parsing to be used by Actions.
+  Fortran::parser::Parsing &parsing() const { return *parsing_; }
+
   /// }
   /// @name High-Level Operations
   /// {

flang/include/flang/Frontend/CompilerInvocation.h

@@ -9,6 +9,7 @@
 #define LLVM_FLANG_FRONTEND_COMPILERINVOCATION_H
 
 #include "flang/Frontend/FrontendOptions.h"
+#include "flang/Parser/parsing.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticOptions.h"
 #include "llvm/Option/ArgList.h"
@@ -40,15 +41,25 @@ public:
 };
 
 class CompilerInvocation : public CompilerInvocationBase {
-  /// Options controlling the frontend itself.
+  /// Options for the frontend driver
+  // TODO: Merge with or translate to parserOpts_. We shouldn't need two sets of
+  // options.
   FrontendOptions frontendOpts_;
 
+  /// Options for Flang parser
+  // TODO: Merge with or translate to frontendOpts_. We shouldn't need two sets
+  // of options.
+  Fortran::parser::Options parserOpts_;
+
 public:
   CompilerInvocation() = default;
 
   FrontendOptions &frontendOpts() { return frontendOpts_; }
   const FrontendOptions &frontendOpts() const { return frontendOpts_; }
 
+  Fortran::parser::Options &fortranOpts() { return parserOpts_; }
+  const Fortran::parser::Options &fortranOpts() const { return parserOpts_; }
+
   /// Create a compiler invocation from a list of input options.
   /// \returns true on success.
   /// \returns false if an error was encountered while parsing the arguments
@@ -56,6 +67,13 @@ public:
   static bool CreateFromArgs(CompilerInvocation &res,
       llvm::ArrayRef<const char *> commandLineArgs,
       clang::DiagnosticsEngine &diags);
+
+  /// Set the Fortran options to predifined defaults. These defaults are
+  /// consistend with f18/f18.cpp.
+  // TODO: We should map frontendOpts_ to parserOpts_ instead. For that, we
+  // need to extend frontendOpts_ first. Next, we need to add the corresponding
+  // compiler driver options in libclangDriver.
+  void SetDefaultFortranOpts();
 };
 
 } // end namespace Fortran::frontend

flang/include/flang/Frontend/FrontendActions.h

@@ -21,6 +21,10 @@ class InputOutputTestAction : public FrontendAction {
   void ExecuteAction() override;
 };
 
+class PrintPreprocessedAction : public FrontendAction {
+  void ExecuteAction() override;
+};
+
 } // namespace Fortran::frontend
 
 #endif // LLVM_FLANG_FRONTEND_FRONTENDACTIONS_H

flang/include/flang/Frontend/FrontendOptions.h

@@ -22,15 +22,18 @@ enum ActionKind {
   /// -test-io mode
   InputOutputTest,
 
-  // TODO: ADD flags as the Actions are implemented, e.g.
-  // RunPreprocessor, ParserSyntaxOnly, EmitLLVM, EmitLLVMOnly,
-  // EmitCodeGenOnly, EmitAssembly, (...)
+  /// -E mode.
+  PrintPreprocessedInput,
+  /// TODO: RunPreprocessor, ParserSyntaxOnly, EmitLLVM, EmitLLVMOnly,
+  /// EmitCodeGenOnly, EmitAssembly, (...)
 };
 
 inline const char *GetActionKindName(const ActionKind ak) {
   switch (ak) {
   case InputOutputTest:
     return "InputOutputTest";
+  case PrintPreprocessedInput:
+    return "PrintPreprocessedInput";
   default:
     return "<unknown ActionKind>";
     // TODO:

flang/lib/Frontend/CompilerInstance.cpp

@@ -9,6 +9,7 @@
 #include "flang/Frontend/CompilerInstance.h"
 #include "flang/Frontend/CompilerInvocation.h"
 #include "flang/Frontend/TextDiagnosticPrinter.h"
+#include "flang/Parser/parsing.h"
 #include "flang/Parser/provenance.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
@@ -20,7 +21,14 @@ using namespace Fortran::frontend;
 
 CompilerInstance::CompilerInstance()
     : invocation_(new CompilerInvocation()),
-      allSources_(new Fortran::parser::AllSources()) {}
+      allSources_(new Fortran::parser::AllSources()),
+      allCookedSources_(new Fortran::parser::AllCookedSources(*allSources_)),
+      parsing_(new Fortran::parser::Parsing(*allCookedSources_)) {
+
+  // TODO: This is a good default during development, but ultimately we should
+  // give the user the opportunity to specify this.
+  allSources_->set_encoding(Fortran::parser::Encoding::UTF_8);
+}
 
 CompilerInstance::~CompilerInstance() {
   assert(outputFiles_.empty() && "Still output files in flight?");
@@ -118,6 +126,10 @@ void CompilerInstance::ClearOutputFiles(bool eraseFiles) {
 }
 
 bool CompilerInstance::ExecuteAction(FrontendAction &act) {
+  // Set some sane defaults for the frontend.
+  // TODO: Instead of defaults we should be setting these options based on the
+  // user input.
+  this->invocation().SetDefaultFortranOpts();
 
   // Connect Input to a CompileInstance
   for (const FrontendInputFile &fif : frontendOpts().inputs_) {

flang/lib/Frontend/CompilerInvocation.cpp

@@ -90,8 +90,11 @@ static InputKind ParseFrontendArgs(FrontendOptions &opts,
     case clang::driver::options::OPT_test_io:
       opts.programAction_ = InputOutputTest;
       break;
+    case clang::driver::options::OPT_E:
+      opts.programAction_ = PrintPreprocessedInput;
+      break;
+
       // TODO:
-      // case clang::driver::options::OPT_E:
       // case clang::driver::options::OPT_emit_obj:
       // case calng::driver::options::OPT_emit_llvm:
       // case clang::driver::options::OPT_emit_llvm_only:
@@ -180,3 +183,12 @@ bool CompilerInvocation::CreateFromArgs(CompilerInvocation &res,
 
   return success;
 }
+
+void CompilerInvocation::SetDefaultFortranOpts() {
+  auto fortranOptions = fortranOpts();
+
+  // These defaults are based on the defaults in f18/f18.cpp.
+  std::vector<std::string> searchDirectories{"."s};
+  fortranOptions.searchDirectories = searchDirectories;
+  fortranOptions.isFixedForm = false;
+}

flang/lib/Frontend/FrontendAction.cpp

@@ -45,7 +45,15 @@ bool FrontendAction::ShouldEraseOutputFiles() {
 }
 
 llvm::Error FrontendAction::Execute() {
+  std::string currentInputPath{GetCurrentFileOrBufferName()};
+
+  Fortran::parser::Options parserOptions =
+      this->instance().invocation().fortranOpts();
+
+  this->instance().parsing().Prescan(currentInputPath, parserOptions);
+
   ExecuteAction();
+
   return llvm::Error::success();
 }
 

flang/lib/Frontend/FrontendActions.cpp

@@ -5,12 +5,12 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+
 #include "flang/Frontend/FrontendActions.h"
-#include "flang/Common/Fortran-features.h"
-#include "flang/Common/default-kinds.h"
 #include "flang/Frontend/CompilerInstance.h"
+#include "flang/Parser/parsing.h"
+#include "flang/Parser/provenance.h"
 #include "flang/Parser/source.h"
-#include "clang/Serialization/PCHContainerOperations.h"
 
 using namespace Fortran::frontend;
 
@@ -43,3 +43,28 @@ void InputOutputTestAction::ExecuteAction() {
     ci.WriteOutputStream(fileContent.data());
   }
 }
+
+void PrintPreprocessedAction::ExecuteAction() {
+  std::string buf;
+  llvm::raw_string_ostream outForPP{buf};
+
+  // Run the preprocessor
+  CompilerInstance &ci = this->instance();
+  ci.parsing().DumpCookedChars(outForPP);
+
+  // If a pre-defined output stream exists, dump the preprocessed content there
+  if (!ci.IsOutputStreamNull()) {
+    // Send the output to the pre-defined output buffer.
+    ci.WriteOutputStream(outForPP.str());
+    return;
+  }
+
+  // Create a file and save the preprocessed output there
+  if (auto os{ci.CreateDefaultOutputFile(
+          /*Binary=*/true, /*InFile=*/GetCurrentFileOrBufferName())}) {
+    (*os) << outForPP.str();
+  } else {
+    llvm::errs() << "Unable to create the output file\n";
+    return;
+  }
+}

flang/lib/FrontendTool/ExecuteCompilerInvocation.cpp

@@ -29,6 +29,9 @@ static std::unique_ptr<FrontendAction> CreateFrontendBaseAction(
   case InputOutputTest:
     return std::make_unique<InputOutputTestAction>();
     break;
+  case PrintPreprocessedInput:
+    return std::make_unique<PrintPreprocessedAction>();
+    break;
   default:
     break;
     // TODO:

flang/test/Flang-Driver/driver-help-hidden.f90

@@ -18,6 +18,7 @@
 ! CHECK:USAGE: flang-new
 ! CHECK-EMPTY:
 ! CHECK-NEXT:OPTIONS:
+! CHECK-NEXT: -E        Only run the preprocessor
 ! CHECK-NEXT: -fcolor-diagnostics    Enable colors in diagnostics
 ! CHECK-NEXT: -fno-color-diagnostics Disable colors in diagnostics
 ! CHECK-NEXT: -help     Display available options

flang/test/Flang-Driver/driver-help.f90

@@ -12,24 +12,26 @@
 ! RUN: %flang-new -fc1 -help 2>&1 | FileCheck %s --check-prefix=HELP-FC1
 ! RUN: not %flang-new -fc1 -helps 2>&1 | FileCheck %s --check-prefix=ERROR
 
-!-----------------------------
-! EXPECTED OUTPUT (flang-new)
-!-----------------------------
+!----------------------------------------------------
+! EXPECTED OUTPUT FOR FLANG DRIVER (flang-new)
+!----------------------------------------------------
 ! HELP:USAGE: flang-new
 ! HELP-EMPTY:
 ! HELP-NEXT:OPTIONS:
+! HELP-NEXT: -E                     Only run the preprocessor
 ! HELP-NEXT: -fcolor-diagnostics    Enable colors in diagnostics
 ! HELP-NEXT: -fno-color-diagnostics Disable colors in diagnostics
 ! HELP-NEXT: -help                  Display available options
 ! HELP-NEXT: -o <file>              Write output to <file>
 ! HELP-NEXT: --version              Print version information
 
-!----------------------------------
-! EXPECTED OUTPUT (flang-new -fc1)
-!----------------------------------
+!-------------------------------------------------------------
+! EXPECTED OUTPUT FOR FLANG FRONTEND DRIVER (flang-new -fc1)
+!-------------------------------------------------------------
 ! HELP-FC1:USAGE: flang-new
 ! HELP-FC1-EMPTY:
 ! HELP-FC1-NEXT:OPTIONS:
+! HELP-FC1-NEXT: -E        Only run the preprocessor
 ! HELP-FC1-NEXT: -help     Display available options
 ! HELP-FC1-NEXT: -o <file> Write output to <file>
 ! HELP-FC1-NEXT: --version Print version information

flang/test/Frontend/Inputs/hello-world.c

@@ -0,0 +1 @@
+int main() { return 0; }

flang/test/Frontend/print-preprocess-C-file.f90

@@ -0,0 +1,13 @@
+! Test preprocessing for C files using Flang driver
+
+! REQUIRES: new-flang-driver
+
+!--------------------------
+! FLANG DRIVER (flang-new)
+!--------------------------
+! RUN: not %flang-new -E %S/Inputs/hello-world.c  2>&1 | FileCheck %s
+
+!-----------------------
+! EXPECTED OUTPUT
+!-----------------------
+! CHECK: error: unknown integrated tool '-cc1'. Valid tools include '-fc1'.

flang/test/Frontend/print-preprocessed-file.f90

@@ -0,0 +1,35 @@
+! Test printpreprocessed action
+
+! REQUIRES: new-flang-driver
+
+!--------------------------
+! FLANG DRIVER (flang-new)
+!--------------------------
+! RUN: %flang-new -E %s  2>&1 | FileCheck %s
+
+!-----------------------------------------
+!   FRONTEND FLANG DRIVER (flang-new -fc1)
+!-----------------------------------------
+! RUN: %flang-new -fc1 -E %s  2>&1 | FileCheck %s
+
+
+!-----------------------
+! EXPECTED OUTPUT
+!-----------------------
+! flang-new -E  %s
+! CHECK:program a
+! CHECK-NOT:program b
+! CHECK-NEXT:x = 1
+! CHECK-NEXT:write(*,*) x
+! CHECK-NEXT:end
+
+! Preprocessed-file.F:
+#define NEW
+#ifdef NEW
+  program A
+#else
+  program B
+#endif
+    x = 1
+    write(*,*) x
+  end

flang/unittests/Frontend/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_flang_unittest(FlangFrontendTests
   CompilerInstanceTest.cpp
   InputOutputTest.cpp
+  PrintPreprocessedTest.cpp
 )
 
 target_link_libraries(FlangFrontendTests
@@ -10,4 +11,4 @@ target_link_libraries(FlangFrontendTests
   flangFrontend
   flangFrontendTool
   FortranParser
-  )
+)

flang/unittests/Frontend/PrintPreprocessedTest.cpp

@@ -0,0 +1,79 @@
+//===- unittests/Frontend/PrintPreprocessedTest.cpp  FrontendAction tests--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gtest/gtest.h"
+#include "flang/Frontend/CompilerInstance.h"
+#include "flang/Frontend/FrontendOptions.h"
+#include "flang/FrontendTool/Utils.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace Fortran::frontend;
+
+namespace {
+
+TEST(FrontendAction, PrintPreprocessedInput) {
+  std::string inputFile = "test-file.f";
+  std::error_code ec;
+
+  // 1. Create the input file for the file manager
+  // AllSources (which is used to manage files inside every compiler instance),
+  // works with paths. This means that it requires a physical file. Create one.
+  std::unique_ptr<llvm::raw_fd_ostream> os{
+      new llvm::raw_fd_ostream(inputFile, ec, llvm::sys::fs::OF_None)};
+  if (ec)
+    FAIL() << "Fail to create the file need by the test";
+
+  // Populate the input file with the pre-defined input and flush it.
+  *(os) << "! test-file.F:\n"
+        << "#ifdef NEW\n"
+        << "  Program A \n"
+        << "#else\n"
+        << "  Program B\n"
+        << "#endif";
+  os.reset();
+
+  // Get the path of the input file
+  llvm::SmallString<64> cwd;
+  if (std::error_code ec = llvm::sys::fs::current_path(cwd))
+    FAIL() << "Failed to obtain the current working directory";
+  std::string testFilePath(cwd.c_str());
+  testFilePath += "/" + inputFile;
+
+  // 2. Prepare the compiler (CompilerInvocation + CompilerInstance)
+  CompilerInstance compInst;
+  compInst.CreateDiagnostics();
+  auto invocation = std::make_shared<CompilerInvocation>();
+  invocation->frontendOpts().programAction_ = PrintPreprocessedInput;
+
+  compInst.set_invocation(std::move(invocation));
+  compInst.frontendOpts().inputs_.push_back(
+      FrontendInputFile(testFilePath, Language::Fortran));
+
+  // 3. Set-up the output stream. Using output buffer wrapped as an output
+  // stream, as opposed to an actual file (or a file descriptor).
+  llvm::SmallVector<char, 256> outputFileBuffer;
+  std::unique_ptr<llvm::raw_pwrite_stream> outputFileStream(
+      new llvm::raw_svector_ostream(outputFileBuffer));
+  compInst.set_outputStream(std::move(outputFileStream));
+
+  // 4. Run the earlier defined FrontendAction
+  bool success = ExecuteCompilerInvocation(&compInst);
+
+  // 5. Validate the expected output
+  EXPECT_TRUE(success);
+  EXPECT_TRUE(!outputFileBuffer.empty());
+  EXPECT_TRUE(
+      llvm::StringRef(outputFileBuffer.data()).startswith("program b\n"));
+
+  // 6. Clear the input and the output files. Since we used an output buffer,
+  // there are no physical output files to delete.
+  llvm::sys::fs::remove(inputFile);
+  compInst.ClearOutputFiles(/*EraseFiles=*/true);
+}
+} // namespace