[flang] Disable Fortran free form line continuation in non-source lin… #94663

klausler · 2024-06-06T19:20:58Z

…e produced by keyword macro replacement

When later initial keyword macro replacement will yield a line that is not Fortran source, don't interpret "&" as a Fortran source line continuation marker during tokenization of the line.

Fixes #82579.

llvmbot · 2024-06-06T19:21:30Z

@llvm/pr-subscribers-flang-parser

Author: Peter Klausler (klausler)

Changes

…e produced by keyword macro replacement

When later initial keyword macro replacement will yield a line that is not Fortran source, don't interpret "&" as a Fortran source line continuation marker during tokenization of the line.

Fixes #82579.

Full diff: https://github.com/llvm/llvm-project/pull/94663.diff

5 Files Affected:

(modified) flang/include/flang/Parser/token-sequence.h (+1-1)
(modified) flang/lib/Parser/prescan.cpp (+39-10)
(modified) flang/lib/Parser/prescan.h (+4)
(modified) flang/lib/Parser/token-sequence.cpp (+4-1)
(modified) flang/test/Preprocessing/directive-contin-with-pp.F90 (+11)

diff --git a/flang/include/flang/Parser/token-sequence.h b/flang/include/flang/Parser/token-sequence.h
index 849240d8ec62c..ee5f71edd03c8 100644
--- a/flang/include/flang/Parser/token-sequence.h
+++ b/flang/include/flang/Parser/token-sequence.h
@@ -124,7 +124,7 @@ class TokenSequence {
   TokenSequence &RemoveRedundantBlanks(std::size_t firstChar = 0);
   TokenSequence &ClipComment(const Prescanner &, bool skipFirst = false);
   const TokenSequence &CheckBadFortranCharacters(
-      Messages &, const Prescanner &) const;
+      Messages &, const Prescanner &, bool allowAmpersand) const;
   const TokenSequence &CheckBadParentheses(Messages &) const;
   void Emit(CookedSource &) const;
   llvm::raw_ostream &Dump(llvm::raw_ostream &) const;
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index f9b9c3d2c6e56..cfc14b8f61854 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -180,6 +180,24 @@ void Prescanner::Statement() {
       }
     } else {
       SkipSpaces();
+      // Check for a leading identifier that might be a keyword macro
+      // that will expand to anything indicating a non-source line, like
+      // a comment marker or directive sentinel.  If so, disable line
+      // continuation, so that NextToken() won't consume anything from
+      // following lines.
+      if (IsLegalIdentifierStart(*at_)) {
+        CHECK(NextToken(tokens));
+        CHECK(tokens.SizeInTokens() == 1);
+        CharBlock id{tokens.TokenAt(0)};
+        if (preprocessor_.IsNameDefined(id) &&
+            !preprocessor_.IsFunctionLikeDefinition(id)) {
+          if (auto replaced{preprocessor_.MacroReplacement(tokens, *this)}) {
+            disableSourceContinuation_ =
+                ClassifyLine(*replaced, GetCurrentProvenance()).kind !=
+                LineClassification::Kind::Source;
+          }
+        }
+      }
     }
     break;
   }
@@ -197,17 +215,13 @@ void Prescanner::Statement() {
   Provenance newlineProvenance{GetCurrentProvenance()};
   if (std::optional<TokenSequence> preprocessed{
           preprocessor_.MacroReplacement(tokens, *this)}) {
-    // Reprocess the preprocessed line.  Append a newline temporarily.
-    preprocessed->PutNextTokenChar('\n', newlineProvenance);
-    preprocessed->CloseToken();
-    const char *ppd{preprocessed->ToCharBlock().begin()};
-    LineClassification ppl{ClassifyLine(ppd)};
-    preprocessed->pop_back(); // remove the newline
+    // Reprocess the preprocessed line.
+    LineClassification ppl{ClassifyLine(*preprocessed, newlineProvenance)};
     switch (ppl.kind) {
     case LineClassification::Kind::Comment:
       break;
     case LineClassification::Kind::IncludeLine:
-      FortranInclude(ppd + ppl.payloadOffset);
+      FortranInclude(preprocessed->TokenAt(0).begin() + ppl.payloadOffset);
       break;
     case LineClassification::Kind::ConditionalCompilationDirective:
     case LineClassification::Kind::IncludeDirective:
@@ -270,7 +284,8 @@ void Prescanner::Statement() {
 
 void Prescanner::CheckAndEmitLine(
     TokenSequence &tokens, Provenance newlineProvenance) {
-  tokens.CheckBadFortranCharacters(messages_, *this);
+  tokens.CheckBadFortranCharacters(
+      messages_, *this, disableSourceContinuation_);
   // Parenthesis nesting check does not apply while any #include is
   // active, nor on the lines before and after a top-level #include.
   // Applications play shenanigans with line continuation before and
@@ -1243,7 +1258,9 @@ bool Prescanner::IsImplicitContinuation() const {
 }
 
 bool Prescanner::Continuation(bool mightNeedFixedFormSpace) {
-  if (*at_ == '\n' || *at_ == '&') {
+  if (disableSourceContinuation_) {
+    return false;
+  } else if (*at_ == '\n' || *at_ == '&') {
     if (inFixedForm_) {
       return FixedFormContinuation(mightNeedFixedFormSpace);
     } else {
@@ -1255,8 +1272,9 @@ bool Prescanner::Continuation(bool mightNeedFixedFormSpace) {
     BeginSourceLine(nextLine_);
     NextLine();
     return true;
+  } else {
+    return false;
   }
-  return false;
 }
 
 std::optional<Prescanner::LineClassification>
@@ -1418,6 +1436,17 @@ Prescanner::LineClassification Prescanner::ClassifyLine(
   return {LineClassification::Kind::Source};
 }
 
+Prescanner::LineClassification Prescanner::ClassifyLine(
+    TokenSequence &tokens, Provenance newlineProvenance) const {
+  // Append a newline temporarily.
+  tokens.PutNextTokenChar('\n', newlineProvenance);
+  tokens.CloseToken();
+  const char *ppd{tokens.ToCharBlock().begin()};
+  LineClassification classification{ClassifyLine(ppd)};
+  tokens.pop_back(); // remove the newline
+  return classification;
+}
+
 void Prescanner::SourceFormChange(std::string &&dir) {
   if (dir == "!dir$ free") {
     inFixedForm_ = false;
diff --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h
index 491b1fe0a7517..1b414afcd2419 100644
--- a/flang/lib/Parser/prescan.h
+++ b/flang/lib/Parser/prescan.h
@@ -117,6 +117,7 @@ class Prescanner {
     parenthesisNesting_ = 0;
     continuationLines_ = 0;
     isPossibleMacroCall_ = false;
+    disableSourceContinuation_ = false;
   }
 
   Provenance GetProvenance(const char *sourceChar) const {
@@ -192,6 +193,8 @@ class Prescanner {
   std::optional<LineClassification> IsFreeFormCompilerDirectiveLine(
       const char *) const;
   LineClassification ClassifyLine(const char *) const;
+  LineClassification ClassifyLine(
+      TokenSequence &, Provenance newlineProvenance) const;
   void SourceFormChange(std::string &&);
   bool CompilerDirectiveContinuation(TokenSequence &, const char *sentinel);
   bool SourceLineContinuation(TokenSequence &);
@@ -211,6 +214,7 @@ class Prescanner {
   int continuationLines_{0};
   bool isPossibleMacroCall_{false};
   bool afterIncludeDirective_{false};
+  bool disableSourceContinuation_{false};
 
   Provenance startProvenance_;
   const char *start_{nullptr}; // beginning of current source file content
diff --git a/flang/lib/Parser/token-sequence.cpp b/flang/lib/Parser/token-sequence.cpp
index d0254ecd5aaef..40560bbacb54f 100644
--- a/flang/lib/Parser/token-sequence.cpp
+++ b/flang/lib/Parser/token-sequence.cpp
@@ -347,7 +347,8 @@ ProvenanceRange TokenSequence::GetProvenanceRange() const {
 }
 
 const TokenSequence &TokenSequence::CheckBadFortranCharacters(
-    Messages &messages, const Prescanner &prescanner) const {
+    Messages &messages, const Prescanner &prescanner,
+    bool allowAmpersand) const {
   std::size_t tokens{SizeInTokens()};
   for (std::size_t j{0}; j < tokens; ++j) {
     CharBlock token{TokenAt(j)};
@@ -362,6 +363,8 @@ const TokenSequence &TokenSequence::CheckBadFortranCharacters(
           ++j;
           continue;
         }
+      } else if (ch == '&' && allowAmpersand) {
+        continue;
       }
       if (ch < ' ' || ch >= '\x7f') {
         messages.Say(GetTokenProvenanceRange(j),
diff --git a/flang/test/Preprocessing/directive-contin-with-pp.F90 b/flang/test/Preprocessing/directive-contin-with-pp.F90
index 9a06ae8438210..7e30b03c349d1 100644
--- a/flang/test/Preprocessing/directive-contin-with-pp.F90
+++ b/flang/test/Preprocessing/directive-contin-with-pp.F90
@@ -5,6 +5,7 @@
 #define FIRST(x) DIR_START x
 #define NEXT(x) DIR_CONT x
 #define AMPER &
+#define COMMENT !
 
 subroutine s(x1, x2, x3, x4, x5, x6, x7)
 
@@ -28,6 +29,14 @@ subroutine s(x1, x2, x3, x4, x5, x6, x7)
 NEXT(x7 &)
 NEXT(x8)
 
+COMMENT blah &
+COMMENT & more
+stop 1
+
+DIR_START blah &
+DIR_START & more
+stop 2
+
 end
 
 !CHECK: subroutine s(x1, x2, x3, x4, x5, x6, x7)
@@ -38,4 +47,6 @@ subroutine s(x1, x2, x3, x4, x5, x6, x7)
 !CHECK: !dir$ ignore_tkr  x5
 !CHECK: !dir$ ignore_tkr  x6
 !CHECK: !dir$ ignore_tkr  x7  x8
+!CHECK: stop 1
+!CHECK: stop 2
 !CHECK: end

ye-luo · 2024-06-07T03:04:43Z

I'm testing this patch macro_omp.f90

#if defined(_OPENMP)
#define DEVOMP !$omp
#else
#define DEVOMP !!!
#endif

program omp
  implicit none
  integer i,j
 j = 0
 DEVOMP parallel &
 DEVOMP &do &
 DEVOMP & reduction(+:j)
 do i =1, 10
   j=j+i
 end do

 DEVOMP parallel &
 DEVOMP do &
 DEVOMP reduction(+:j)
 do i =1, 10
   j=j+i
 end do
 write(*,*) "expect 100 ans = ", j
end program

flang-new -cpp -fopenmp -E and flang-new -cpp -E got me desired output. That is good.

However, I'm hitting a very strange issue. When I do flang-new -cpp -fopenmp, I got error

yeluo@epyc-server:~$ flang-new -v -cpp -fopenmp macro_omp.f90
flang-new version 19.0.0git (https://github.com/llvm/llvm-project.git df6139812aac8ecfa342dbcb277a9e12a9693db2)
Target: x86_64-unknown-linux-gnu
Thread model: posix
InstalledDir: /soft/llvm/main-patched/bin
Build config: +assertions
Found candidate GCC installation: /usr/lib/gcc/x86_64-linux-gnu/7
Found candidate GCC installation: /usr/lib/gcc/x86_64-linux-gnu/7.5.0
Found candidate GCC installation: /usr/lib/gcc/x86_64-linux-gnu/8
Found candidate GCC installation: /usr/lib/gcc/x86_64-linux-gnu/9
Selected GCC installation: /usr/lib/gcc/x86_64-linux-gnu/9
Candidate multilib: .;@m64
Candidate multilib: 32;@m32
Candidate multilib: x32;@mx32
Selected multilib: .;@m64
Found CUDA installation: /usr/local/cuda-12.3, version 12.3
Found HIP installation: /opt/rocm-6.1.0, version 6.1.40091
 "/soft/llvm/main-patched/bin/flang-new" -fc1 -triple x86_64-unknown-linux-gnu -emit-obj -cpp -fcolor-diagnostics -mrelocation-model pic -pic-level 2 -pic-is-pie -target-cpu x86-64 -fopenmp -resource-dir /soft/llvm/main-patched/bin/.. -mframe-pointer=all -o /tmp/macro_omp-a30508.o -x f95-cpp-input macro_omp.f90
error: Could not parse macro_omp.f90
./macro_omp.f90:12:18: error: expected end of line
   DEVOMP parallel &
                   ^
./macro_omp.f90:12:2: in the context: OpenMP construct
   DEVOMP parallel &
   ^^^^^^
./macro_omp.f90:2:16: in a macro defined here
  #define DEVOMP !$omp
                 ^^^^^
that expanded to:
  !$omp
  ^
./macro_omp.f90:12:2: in the context: execution part construct
   DEVOMP parallel &
   ^^^^^^
./macro_omp.f90:2:16: in a macro defined here
  #define DEVOMP !$omp
                 ^^^^^
that expanded to:
  !$omp
  ^
./macro_omp.f90:11:2: in the context: execution part
   j = 0
   ^
./macro_omp.f90:7:1: in the context: main program
  program omp
  ^
./macro_omp.f90:13:9: error: expected 'END'
   DEVOMP &do &
          ^
./macro_omp.f90:13:2: in the context: execution part construct
   DEVOMP &do &
   ^^^^^^
./macro_omp.f90:2:16: in a macro defined here
  #define DEVOMP !$omp
                 ^^^^^
that expanded to:
  !$omp
  ^
./macro_omp.f90:11:2: in the context: execution part
   j = 0
   ^
./macro_omp.f90:7:1: in the context: main program
  program omp
  ^
./macro_omp.f90:13:10: error: expected OpenMP construct
   DEVOMP &do &
           ^
./macro_omp.f90:13:2: in the context: OpenMP construct
   DEVOMP &do &
   ^^^^^^
./macro_omp.f90:2:16: in a macro defined here
  #define DEVOMP !$omp
                 ^^^^^
that expanded to:
  !$omp
  ^
./macro_omp.f90:13:2: in the context: execution part construct
   DEVOMP &do &
   ^^^^^^
./macro_omp.f90:2:16: in a macro defined here
  #define DEVOMP !$omp
                 ^^^^^
that expanded to:
  !$omp
  ^
./macro_omp.f90:11:2: in the context: execution part
   j = 0
   ^
./macro_omp.f90:7:1: in the context: main program
  program omp
  ^
./macro_omp.f90:14:9: error: expected 'END'
   DEVOMP & reduction(+:j)
          ^
./macro_omp.f90:14:2: in the context: execution part construct
   DEVOMP & reduction(+:j)
   ^^^^^^
./macro_omp.f90:2:16: in a macro defined here
  #define DEVOMP !$omp
                 ^^^^^
that expanded to:
  !$omp
  ^
./macro_omp.f90:11:2: in the context: execution part
   j = 0
   ^
./macro_omp.f90:7:1: in the context: main program
  program omp
  ^
./macro_omp.f90:14:10: error: expected OpenMP construct
   DEVOMP & reduction(+:j)
           ^
./macro_omp.f90:14:2: in the context: OpenMP construct
   DEVOMP & reduction(+:j)
   ^^^^^^
./macro_omp.f90:2:16: in a macro defined here
  #define DEVOMP !$omp
                 ^^^^^
that expanded to:
  !$omp
  ^
./macro_omp.f90:14:2: in the context: execution part construct
   DEVOMP & reduction(+:j)
   ^^^^^^
./macro_omp.f90:2:16: in a macro defined here
  #define DEVOMP !$omp
                 ^^^^^
that expanded to:
  !$omp
  ^
./macro_omp.f90:11:2: in the context: execution part
   j = 0
   ^
./macro_omp.f90:7:1: in the context: main program
  program omp
  ^
./macro_omp.f90:19:18: error: expected end of line
   DEVOMP parallel &
                   ^
./macro_omp.f90:19:2: in the context: OpenMP construct
   DEVOMP parallel &
   ^^^^^^
./macro_omp.f90:2:16: in a macro defined here
  #define DEVOMP !$omp
                 ^^^^^
that expanded to:
  !$omp
  ^
./macro_omp.f90:19:2: in the context: execution part construct
   DEVOMP parallel &
   ^^^^^^
./macro_omp.f90:2:16: in a macro defined here
  #define DEVOMP !$omp
                 ^^^^^
that expanded to:
  !$omp
  ^
./macro_omp.f90:11:2: in the context: execution part
   j = 0
   ^
./macro_omp.f90:7:1: in the context: main program
  program omp
  ^
./macro_omp.f90:20:12: error: expected end of line
   DEVOMP do &
             ^
./macro_omp.f90:20:2: in the context: OpenMP construct
   DEVOMP do &
   ^^^^^^
./macro_omp.f90:2:16: in a macro defined here
  #define DEVOMP !$omp
                 ^^^^^
that expanded to:
  !$omp
  ^
./macro_omp.f90:20:2: in the context: execution part construct
   DEVOMP do &
   ^^^^^^
./macro_omp.f90:2:16: in a macro defined here
  #define DEVOMP !$omp
                 ^^^^^
that expanded to:
  !$omp
  ^
./macro_omp.f90:11:2: in the context: execution part
   j = 0
   ^
./macro_omp.f90:7:1: in the context: main program
  program omp
  ^
./macro_omp.f90:21:9: error: expected 'END'
   DEVOMP reduction(+:j)
          ^
./macro_omp.f90:21:2: in the context: execution part construct
   DEVOMP reduction(+:j)
   ^^^^^^
./macro_omp.f90:2:16: in a macro defined here
  #define DEVOMP !$omp
                 ^^^^^
that expanded to:
  !$omp
  ^
./macro_omp.f90:11:2: in the context: execution part
   j = 0
   ^
./macro_omp.f90:7:1: in the context: main program
  program omp
  ^
./macro_omp.f90:21:10: error: expected OpenMP construct
   DEVOMP reduction(+:j)
           ^
./macro_omp.f90:21:2: in the context: OpenMP construct
   DEVOMP reduction(+:j)
   ^^^^^^
./macro_omp.f90:2:16: in a macro defined here
  #define DEVOMP !$omp
                 ^^^^^
that expanded to:
  !$omp
  ^
./macro_omp.f90:21:2: in the context: execution part construct
   DEVOMP reduction(+:j)
   ^^^^^^
./macro_omp.f90:2:16: in a macro defined here
  #define DEVOMP !$omp
                 ^^^^^
that expanded to:
  !$omp
  ^
./macro_omp.f90:11:2: in the context: execution part
   j = 0
   ^
./macro_omp.f90:7:1: in the context: main program
  program omp
  ^

The error went crazy. actually there is a line error: Could not parse macro_omp.f90

However, if I extract preprocessed file flang-new -cpp -fopenmp -E macro_omp.f90 >& macro_omp_cpp.f90

#line "./macro_omp.f90" 7
      program omp

      implicit none
      integer i,j
      j = 0
 !$omp  parallel &
 !$omp  &do &
 !$omp  & reduction(+:j)
      do i =1, 10
      j=j+i
      end do

 !$omp  parallel &
 !$omp  do &
 !$omp  reduction(+:j)
      do i =1, 10
      j=j+i
      end do
      write(*,*) "expect 100 ans = ", j
      end program

and then flang-new -fopenmp macro_omp_cpp.f90, I got no error.

I also noticed the compiler seems behavior non-deterministicly when running flang-new -cpp -fopenmp macro_omp_cpp.f90. It actually succeeds sometimes but hard to reproduce. I'm wondering if the compiler is affected by uninitialized values.

…e produced by keyword macro replacement When later initial keyword macro replacement will yield a line that is not Fortran source, don't interpret "&" as a Fortran source line continuation marker during tokenization of the line. Fixes llvm#82579.

klausler · 2024-06-10T20:51:06Z

Patch has been updated.

ye-luo

Well fixed my reported issue.

klausler · 2024-06-11T15:31:14Z

Well fixed my reported issue.

Does that mean that you still have other bugs?

klausler · 2024-06-12T15:53:38Z

@ye-luo ping -- do you have other related bugs that should be fixed before I merge this?

ye-luo · 2024-06-12T15:55:33Z

@ye-luo ping -- do you have other related bugs that should be fixed before I merge this?

Everything looks good on my side from minimal reproducers to full applications.

klausler requested review from jeffhammond and ye-luo June 6, 2024 19:20

llvmbot added flang Flang issues not falling into any other category flang:parser labels Jun 6, 2024

klausler mentioned this pull request Jun 6, 2024

[Flang] macro + comment + line continuation #82579

Closed

klausler force-pushed the bug82579 branch from df61398 to 5194045 Compare June 10, 2024 20:50

ye-luo approved these changes Jun 11, 2024

View reviewed changes

klausler merged commit e286ecf into llvm:main Jun 12, 2024
7 checks passed

klausler deleted the bug82579 branch June 12, 2024 17:11

k-arrows mentioned this pull request Jun 16, 2024

[Flang] fatal internal error: CHECK(tokens.SizeInTokens() == 1) failed at /root/llvm-project/flang/lib/Parser/prescan.cpp(190) #95689

Closed

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

[flang] Disable Fortran free form line continuation in non-source lin… #94663

[flang] Disable Fortran free form line continuation in non-source lin… #94663

Uh oh!

klausler commented Jun 6, 2024

Uh oh!

llvmbot commented Jun 6, 2024

Uh oh!

ye-luo commented Jun 7, 2024 •

edited

Loading

Uh oh!

klausler commented Jun 10, 2024

Uh oh!

ye-luo left a comment

Uh oh!

klausler commented Jun 11, 2024

Uh oh!

klausler commented Jun 12, 2024

Uh oh!

ye-luo commented Jun 12, 2024

Uh oh!

Uh oh!

Uh oh!

[flang] Disable Fortran free form line continuation in non-source lin… #94663

[flang] Disable Fortran free form line continuation in non-source lin… #94663

Uh oh!

Conversation

klausler commented Jun 6, 2024

Uh oh!

llvmbot commented Jun 6, 2024

Uh oh!

ye-luo commented Jun 7, 2024 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

klausler commented Jun 10, 2024

Uh oh!

ye-luo left a comment

Choose a reason for hiding this comment

Uh oh!

klausler commented Jun 11, 2024

Uh oh!

klausler commented Jun 12, 2024

Uh oh!

ye-luo commented Jun 12, 2024

Uh oh!

Uh oh!

Uh oh!

ye-luo commented Jun 7, 2024 •

edited

Loading