[C++][Modules][P1857R3 2]: A module directive may only appear as the first preprocessing tokens in a file

Signed-off-by: yronglin <[email protected]>

Author: yronglin

clang/include/clang/Lex/Lexer.h

@@ -143,6 +143,9 @@ class Lexer : public PreprocessorLexer {
   /// True if this is the first time we're lexing the input file.
   bool IsFirstTimeLexingFile;
 
+  /// True if current lexing token is the first pp-token.
+  bool IsFirstPPToken;
+
   // NewLinePtr - A pointer to new line character '\n' being lexed. For '\r\n',
   // it also points to '\n.'
   const char *NewLinePtr;

clang/include/clang/Lex/Preprocessor.h

@@ -350,6 +350,9 @@ class Preprocessor {
   /// Whether the last token we lexed was an '@'.
   bool LastTokenWasAt = false;
 
+  /// First pp-token in current translation unit.
+  Token FirstPPToken;
+
   /// A position within a C++20 import-seq.
   class StdCXXImportSeq {
   public:
@@ -1766,6 +1769,8 @@ class Preprocessor {
   std::optional<LexEmbedParametersResult> LexEmbedParameters(Token &Current,
                                                              bool ForHasEmbed);
 
+  void setFirstPPToken(const Token &Tok) { FirstPPToken = Tok; }
+  Token getFirstPPToken() const { return FirstPPToken; }
   bool LexAfterModuleImport(Token &Result);
   void CollectPpImportSuffix(SmallVectorImpl<Token> &Toks);
 

clang/include/clang/Lex/Token.h

@@ -86,9 +86,12 @@ class Token {
                                 // macro stringizing or charizing operator.
     CommaAfterElided = 0x200, // The comma following this token was elided (MS).
     IsEditorPlaceholder = 0x400, // This identifier is a placeholder.
-    IsReinjected = 0x800, // A phase 4 token that was produced before and
-                          // re-added, e.g. via EnterTokenStream. Annotation
-                          // tokens are *not* reinjected.
+
+    IsReinjected = 0x800,  // A phase 4 token that was produced before and
+                           // re-added, e.g. via EnterTokenStream. Annotation
+                           // tokens are *not* reinjected.
+    FirstPPToken = 0x1000, // This token is the first pp token in the
+                           // translation unit.
   };
 
   tok::TokenKind getKind() const { return Kind; }
@@ -318,6 +321,9 @@ class Token {
   /// represented as characters between '<#' and '#>' in the source code. The
   /// lexer uses identifier tokens to represent placeholders.
   bool isEditorPlaceholder() const { return getFlag(IsEditorPlaceholder); }
+
+  /// Returns true if this token is the first pp-token.
+  bool isFirstPPToken() const { return getFlag(FirstPPToken); }
 };
 
 /// Information about the conditional stack (\#if directives)

clang/include/clang/Sema/Sema.h

@@ -9822,7 +9822,8 @@ class Sema final : public SemaBase {
   DeclGroupPtrTy ActOnModuleDecl(SourceLocation StartLoc,
                                  SourceLocation ModuleLoc, ModuleDeclKind MDK,
                                  ModuleIdPath Path, ModuleIdPath Partition,
-                                 ModuleImportState &ImportState);
+                                 ModuleImportState &ImportState,
+                                 bool IntroducerIsFirstPPToken);
 
   /// The parser has processed a global-module-fragment declaration that begins
   /// the definition of the global module fragment of the current module unit.

clang/lib/Lex/Lexer.cpp

@@ -174,6 +174,8 @@ void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
   ExtendedTokenMode = 0;
 
   NewLinePtr = nullptr;
+
+  IsFirstPPToken = true;
 }
 
 /// Lexer constructor - Create a new lexer object for the specified buffer
@@ -3725,13 +3727,21 @@ bool Lexer::Lex(Token &Result) {
     HasLeadingEmptyMacro = false;
   }
 
+  if (IsFirstPPToken) {
+    Result.setFlag(Token::FirstPPToken);
+    IsFirstPPToken = false;
+  }
+
   bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
   IsAtPhysicalStartOfLine = false;
   bool isRawLex = isLexingRawMode();
   (void) isRawLex;
   bool returnedToken = LexTokenInternal(Result, atPhysicalStartOfLine);
   // (After the LexTokenInternal call, the lexer might be destroyed.)
   assert((returnedToken || !isRawLex) && "Raw lex must succeed");
+
+  if (returnedToken && Result.isFirstPPToken() && PP)
+    PP->setFirstPPToken(Result);
   return returnedToken;
 }
 

clang/lib/Parse/Parser.cpp

@@ -2340,7 +2340,8 @@ void Parser::ParseMicrosoftIfExistsExternalDeclaration() {
 
 Parser::DeclGroupPtrTy
 Parser::ParseModuleDecl(Sema::ModuleImportState &ImportState) {
-  SourceLocation StartLoc = Tok.getLocation();
+  Token Introducer = Tok;
+  SourceLocation StartLoc = Introducer.getLocation();
 
   Sema::ModuleDeclKind MDK = TryConsumeToken(tok::kw_export)
                                  ? Sema::ModuleDeclKind::Interface
@@ -2359,7 +2360,7 @@ Parser::ParseModuleDecl(Sema::ModuleImportState &ImportState) {
   // Parse a global-module-fragment, if present.
   if (getLangOpts().CPlusPlusModules && Tok.is(tok::semi)) {
     SourceLocation SemiLoc = ConsumeToken();
-    if (ImportState != Sema::ModuleImportState::FirstDecl) {
+    if (!Introducer.isFirstPPToken()) {
       Diag(StartLoc, diag::err_global_module_introducer_not_at_start)
         << SourceRange(StartLoc, SemiLoc);
       return nullptr;
@@ -2416,7 +2417,7 @@ Parser::ParseModuleDecl(Sema::ModuleImportState &ImportState) {
   ExpectAndConsumeSemi(diag::err_module_expected_semi);
 
   return Actions.ActOnModuleDecl(StartLoc, ModuleLoc, MDK, Path, Partition,
-                                 ImportState);
+                                 ImportState, Introducer.isFirstPPToken());
 }
 
 Decl *Parser::ParseModuleImport(SourceLocation AtLoc,

clang/lib/Sema/SemaModule.cpp

@@ -258,11 +258,11 @@ static bool DiagReservedModuleName(Sema &S, const IdentifierInfo *II,
 Sema::DeclGroupPtrTy
 Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
                       ModuleDeclKind MDK, ModuleIdPath Path,
-                      ModuleIdPath Partition, ModuleImportState &ImportState) {
+                      ModuleIdPath Partition, ModuleImportState &ImportState,
+                      bool IntroducerIsFirstPPToken) {
   assert(getLangOpts().CPlusPlusModules &&
          "should only have module decl in standard C++ modules");
 
-  bool IsFirstDecl = ImportState == ModuleImportState::FirstDecl;
   bool SeenGMF = ImportState == ModuleImportState::GlobalFragment;
   // If any of the steps here fail, we count that as invalidating C++20
   // module state;
@@ -328,12 +328,14 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
           SeenGMF == (bool)this->TheGlobalModuleFragment) &&
          "mismatched global module state");
 
-  // In C++20, the module-declaration must be the first declaration if there
-  // is no global module fragment.
-  if (getLangOpts().CPlusPlusModules && !IsFirstDecl && !SeenGMF) {
+  // In C++20, A module directive may only appear as the first preprocessing
+  // tokens in a file (excluding the global module fragment.).
+  if (getLangOpts().CPlusPlusModules && !IntroducerIsFirstPPToken && !SeenGMF) {
     Diag(ModuleLoc, diag::err_module_decl_not_at_start);
     SourceLocation BeginLoc =
-        ModuleScopes.empty()
+        PP.getFirstPPToken().getLocation().isValid()
+            ? PP.getFirstPPToken().getLocation()
+        : ModuleScopes.empty()
             ? SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID())
             : ModuleScopes.back().BeginLoc;
     if (BeginLoc.isValid()) {

clang/test/CXX/basic/basic.link/p1.cpp

@@ -1,57 +1,128 @@
-// RUN: %clang_cc1 -std=c++2a -verify %s
-// RUN: %clang_cc1 -std=c++2a -verify -DNO_GLOBAL_FRAG %s
-// RUN: %clang_cc1 -std=c++2a -verify -DNO_MODULE_DECL %s
-// RUN: %clang_cc1 -std=c++2a -verify -DNO_PRIVATE_FRAG %s
-// RUN: %clang_cc1 -std=c++2a -verify -DNO_MODULE_DECL -DNO_PRIVATE_FRAG %s
-// RUN: %clang_cc1 -std=c++2a -verify -DNO_GLOBAL_FRAG -DNO_PRIVATE_FRAG %s
-// RUN: %clang_cc1 -std=c++2a -verify -DNO_GLOBAL_FRAG -DNO_MODULE_DECL %s
-// RUN: %clang_cc1 -std=c++2a -verify -DNO_GLOBAL_FRAG -DNO_MODULE_DECL -DNO_PRIVATE_FRAG %s
-// RUN: %clang_cc1 -std=c++2a -verify -DEXPORT_FRAGS %s
-
-#ifndef NO_GLOBAL_FRAG
-#ifdef EXPORT_FRAGS
-export // expected-error {{global module fragment cannot be exported}}
-#endif
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+
+// RUN: %clang_cc1 -std=c++2a -verify %t/M.cppm
+// RUN: %clang_cc1 -std=c++2a -verify %t/NoGlobalFrag.cppm
+// RUN: %clang_cc1 -std=c++2a -verify %t/NoModuleDecl.cppm
+// RUN: %clang_cc1 -std=c++2a -verify %t/NoPrivateFrag.cppm
+// RUN: %clang_cc1 -std=c++2a -verify %t/NoModuleDeclAndNoPrivateFrag.cppm
+// RUN: %clang_cc1 -std=c++2a -verify %t/NoGlobalFragAndNoPrivateFrag.cppm
+// RUN: %clang_cc1 -std=c++2a -verify %t/NoGlobalFragAndNoModuleDecl.cppm
+// RUN: %clang_cc1 -std=c++2a -verify %t/NoGlobalFragAndNoModuleDeclAndNoPrivateFrag.cppm
+// RUN: %clang_cc1 -std=c++2a -verify %t/ExportFrags.cppm
+
+//--- M.cppm
 module;
-#ifdef NO_MODULE_DECL
-// expected-error@-2 {{missing 'module' declaration at end of global module fragment introduced here}}
-#endif
-#endif
+extern int a; // #a1
+export module Foo;
+
+int a; // expected-error {{declaration of 'a' in module Foo follows declaration in the global module}}
+       // expected-note@#a1 {{previous decl}}
+extern int b;
+
+module; // expected-error {{'module;' introducing a global module fragment can appear only at the start of the translation unit}}
+module :private; // #priv-frag
+int b; // ok
+module :private; // expected-error {{private module fragment redefined}}
+                 // expected-note@#priv-frag {{previous definition is here}}
+
+//--- NoGlobalFrag.cppm
+
+extern int a; // #a1
+export module Foo; // expected-error {{module declaration must occur at the start of the translation unit}}
+                   // expected-note@-2 {{add 'module;' to the start of the file to introduce a global module fragment}}
+
+// expected-error@#a2 {{declaration of 'a' in module Foo follows declaration in the global module}}
+// expected-note@#a1 {{previous decl}}
+
+int a; // #a2
+extern int b;
+module; // expected-error {{'module;' introducing a global module fragment can appear only at the start of the translation unit}}
+module :private; // #priv-frag
+int b; // ok
+module :private; // expected-error {{private module fragment redefined}}
+// expected-note@#priv-frag {{previous definition is here}}
 
+//--- NoModuleDecl.cppm
+module; // expected-error {{missing 'module' declaration at end of global module fragment introduced here}}
 extern int a; // #a1
+int a; // #a2
+extern int b;
+module; // expected-error {{'module;' introducing a global module fragment can appear only at the start of the translation unit}}
+module :private; // expected-error {{private module fragment declaration with no preceding module declaration}}
+int b; // ok
 
-#ifndef NO_MODULE_DECL
+//--- NoPrivateFrag.cppm
+module;
+extern int a; // #a1
 export module Foo;
-#ifdef NO_GLOBAL_FRAG
-// expected-error@-2 {{module declaration must occur at the start of the translation unit}}
+
+// expected-error@#a2 {{declaration of 'a' in module Foo follows declaration in the global module}}
+// expected-note@#a1 {{previous decl}}
+int a; // #a2
+extern int b;
+
+module; // expected-error {{'module;' introducing a global module fragment can appear only at the start of the translation unit}}
+int b; // ok
+
+
+//--- NoModuleDeclAndNoPrivateFrag.cppm
+module; // expected-error {{missing 'module' declaration at end of global module fragment introduced here}}
+extern int a; // #a1
+int a; // #a2
+extern int b;
+
+module; // expected-error {{'module;' introducing a global module fragment can appear only at the start of the translation unit}}
+
+int b; // ok
+
+//--- NoGlobalFragAndNoPrivateFrag.cppm
+extern int a; // #a1
+export module Foo; // expected-error {{module declaration must occur at the start of the translation unit}}
 // expected-note@1 {{add 'module;' to the start of the file to introduce a global module fragment}}
-#endif
 
 // expected-error@#a2 {{declaration of 'a' in module Foo follows declaration in the global module}}
 // expected-note@#a1 {{previous decl}}
-#endif
 
 int a; // #a2
 extern int b;
 
 module; // expected-error {{'module;' introducing a global module fragment can appear only at the start of the translation unit}}
 
-#ifndef NO_PRIVATE_FRAG
-#ifdef EXPORT_FRAGS
-export // expected-error {{private module fragment cannot be exported}}
-#endif
+int b; // ok
+
+//--- NoGlobalFragAndNoModuleDecl.cppm
+extern int a; // #a1
+int a; // #a2
+extern int b;
+module; // expected-error {{'module;' introducing a global module fragment can appear only at the start of the translation unit}}
 module :private; // #priv-frag
-#ifdef NO_MODULE_DECL
-// expected-error@-2 {{private module fragment declaration with no preceding module declaration}}
-#endif
-#endif
+// expected-error@-1 {{private module fragment declaration with no preceding module declaration}}
+int b; // ok
 
+
+//--- NoGlobalFragAndNoModuleDeclAndNoPrivateFrag.cppm
+extern int a; // #a1
+int a; // #a2
+extern int b;
+
+module; // expected-error {{'module;' introducing a global module fragment can appear only at the start of the translation unit}}
 int b; // ok
 
+//--- ExportFrags.cppm
+export module; // expected-error {{global module fragment cannot be exported}}
+extern int a; // #a1
+export module Foo;
+// expected-error@#a2 {{declaration of 'a' in module Foo follows declaration in the global module}}
+// expected-note@#a1 {{previous decl}}
 
-#ifndef NO_PRIVATE_FRAG
-#ifndef NO_MODULE_DECL
+int a; // #a2
+extern int b;
+
+module; // expected-error {{'module;' introducing a global module fragment can appear only at the start of the translation unit}}
+
+module :private; // #priv-frag
+
+int b; // ok
 module :private; // expected-error {{private module fragment redefined}}
-// expected-note@#priv-frag {{previous definition is here}}
-#endif
-#endif
+                 // expected-note@#priv-frag {{previous definition is here}}

clang/test/CXX/basic/basic.link/p2.cpp

@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -std=c++2a -DEXPORT %s -verify
-// RUN: %clang_cc1 -std=c++2a -DEXPORT %s -emit-module-interface -o %t.pcm
-// RUN: %clang_cc1 -std=c++2a -UEXPORT %s -verify -fmodule-file=M=%t.pcm
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+
+// RUN: %clang_cc1 -std=c++2a %t/pmf_in_interface.cpp -verify
+// RUN: %clang_cc1 -std=c++2a %t/pmf_in_interface.cpp -emit-module-interface -o %t.pcm
+// RUN: %clang_cc1 -std=c++2a %t/pmf_in_implementation.cpp -verify -fmodule-file=M=%t.pcm
 
-#ifdef EXPORT
-// expected-no-diagnostics
-export
-#else
-// expected-note@+2 {{add 'export' here}}
-#endif
-module M;
 
-#ifndef EXPORT
-// expected-error@+2 {{private module fragment in module implementation unit}}
-#endif
+//--- pmf_in_interface.cpp
+// expected-no-diagnostics
+export module M;
 module :private;
+
+//--- pmf_in_implementation.cpp
+module M; // expected-note {{add 'export' here}}
+module :private; // expected-error {{private module fragment in module implementation unit}}