cxx20 module support for gcc

.github/workflows/runtest.yml

@@ -40,6 +40,9 @@ jobs:
           python -m pip install --upgrade pip setuptools wheel
           python -m pip install -r requirements-dev.txt
           # sudo apt-get update
+          sudo apt-get install gcc-12 g++-12
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100
 
       - name: runtest ${{ matrix.os }}
         run: |

SCons/Tool/cxx.py

@@ -29,9 +29,11 @@
 """
 
 import os.path
+import re
 
 import SCons.Defaults
 import SCons.Util
+import SCons.Scanner
 
 compilers = ['CC', 'c++']
 
@@ -50,6 +52,89 @@ def iscplusplus(source) -> bool:
                 return True
     return False
 
+def gen_module_map_file(root, module_map):
+    module_map_text = '$$root ' + str(root) + '\n'
+    for module, file in module_map.items():
+        module_map_text += str(module) + ' ' + str(file) + '\n'
+    return module_map_text
+
+#TODO filter out C++ whitespace and comments
+module_decl_re = re.compile("(module;)?(.|\n)*export module (.*);")
+
+def module_emitter(target, source, env):
+    if("CXXMODULEPATH" in env):
+        env["__CXXMODULEINIT__"](env)
+
+        export = module_decl_re.match(source[0].get_text_contents())
+        if export:
+            modulename = export[3].strip()
+            if modulename not in env["CXXMODULEMAP"]:
+                env["CXXMODULEMAP"][modulename] = modulename + env["CXXMODULESUFFIX"]
+            target.append(env.File("$CXXMODULEPATH/" + env["CXXMODULEMAP"][modulename]))
+    return (target, source, env)
+
+def module_emitter_static(target, source, env):
+    import SCons.Defaults
+    return SCons.Defaults.StaticObjectEmitter(*module_emitter(target, source, env))
+
+def module_emitter_shared(target, source, env):
+    import SCons.Defaults
+    return SCons.Defaults.SharedObjectEmitter(*module_emitter(target, source, env))
+
+#TODO filter out C++ whitespace and comments
+module_import_re = re.compile(r"\s*(?:(?:export)?\s*import\s*(\S*)\s*;)|(?:(export)?\s*module\s*(\S*)\s*;)")
+
+
+class CxxModuleScanner(SCons.Scanner.Current):
+    def scan(self, node, env, path):
+        result = self.c_scanner(node, env, path)
+
+        if not env.get("CXXMODULEPATH"):
+            return result
+
+        imports = module_import_re.findall(node.get_text_contents())
+        for module, export, impl in imports:
+            if not module:
+                if not export and impl:  # module implementation unit depends on module interface
+                    module = impl
+                else:
+                    continue
+            is_header_unit = False
+            if(module[0] == "<" or module[0] == '"'):
+                module_id_prefix = "@system-header/" if module[0] == "<" else "@header/"
+                cmi = module_id_prefix + module[1:-1] + "$CXXMODULESUFFIX"
+                is_header_unit = True
+            else:
+                cmi = env["CXXMODULEMAP"].get(
+                    module, module+"$CXXMODULESUFFIX")
+            cmi = env.File("$CXXMODULEPATH/" + cmi)
+
+            if(is_header_unit and not cmi.has_builder()):
+                source = self.c_scanner.find_include(
+                    (module[0], module[1:-1]), node.dir, path)
+                if source[0]:
+                    source = source[0]
+                else:
+                    source = env.Value(module[1:-1])
+                env.CxxHeaderUnit(
+                    cmi, source,
+                    CXXCOMOUTPUTSPEC="$CXXSYSTEMHEADERFLAGS" if module[0] == "<" else "$CXXUSERHEADERFLAGS",
+                    CPPPATH = [node.dir, "$CPPPATH"] if module[0] == '"' else "$CPPPATH",
+                    CXXMODULEIDPREFIX=module_id_prefix
+                )
+            result.append(cmi)
+        return result
+
+    def __init__(self, *args, **kwargs):
+        from SCons.Scanner import FindPathDirs
+        super().__init__(self.scan, recursive = True, path_function = FindPathDirs("CPPPATH"), *args, **kwargs)
+        from SCons.Tool import CScanner
+        self.c_scanner = CScanner
+
+
+header_unit = SCons.Builder.Builder(action="$CXXCOM",
+                                    source_scanner=CxxModuleScanner())
+
 def generate(env) -> None:
     """
     Add Builders and construction variables for Visual Age C++ compilers
@@ -58,22 +143,28 @@ def generate(env) -> None:
     import SCons.Tool
     import SCons.Tool.cc
     static_obj, shared_obj = SCons.Tool.createObjBuilders(env)
+    from SCons.Tool import SourceFileScanner
 
     for suffix in CXXSuffixes:
         static_obj.add_action(suffix, SCons.Defaults.CXXAction)
         shared_obj.add_action(suffix, SCons.Defaults.ShCXXAction)
-        static_obj.add_emitter(suffix, SCons.Defaults.StaticObjectEmitter)
-        shared_obj.add_emitter(suffix, SCons.Defaults.SharedObjectEmitter)
+        static_obj.add_emitter(suffix, module_emitter_static)
+        shared_obj.add_emitter(suffix, module_emitter_shared)
+        SourceFileScanner.add_scanner(suffix, CxxModuleScanner())
+
+    env['BUILDERS']['CxxHeaderUnit'] = header_unit
 
     SCons.Tool.cc.add_common_cc_variables(env)
 
     if 'CXX' not in env:
         env['CXX']    = env.Detect(compilers) or compilers[0]
     env['CXXFLAGS']   = SCons.Util.CLVar('')
-    env['CXXCOM']     = '$CXX -o $TARGET -c $CXXFLAGS $CCFLAGS $_CCCOMCOM $SOURCES'
+    env['CXXCOMOUTPUTSPEC'] = '-o $TARGET'
+    env['CXXCOM']     = '$CXX $CXXCOMOUTPUTSPEC -c ${ CXXMODULEFLAGS if CXXMODULEPATH else "" } $CXXFLAGS $CCFLAGS $_CCCOMCOM $SOURCES'
+    env['CXXMODULEMAP'] = {}
     env['SHCXX']      = '$CXX'
     env['SHCXXFLAGS'] = SCons.Util.CLVar('$CXXFLAGS')
-    env['SHCXXCOM']   = '$SHCXX -o $TARGET -c $SHCXXFLAGS $SHCCFLAGS $_CCCOMCOM $SOURCES'
+    env['SHCXXCOM']   = '$SHCXX -o $TARGET -c ${ CXXMODULEFLAGS if CXXMODULEPATH else "" } $SHCXXFLAGS $SHCCFLAGS $_CCCOMCOM $SOURCES'
 
     env['CPPDEFPREFIX']  = '-D'
     env['CPPDEFSUFFIX']  = ''

SCons/Tool/gcc.py

@@ -61,7 +61,6 @@ def generate(env) -> None:
     env["NINJA_DEPFILE_PARSE_FORMAT"] = 'gcc'
 
 
-
 def exists(env):
     # is executable, and is a GNU compiler (or accepts '--version' at least)
     return detect_version(env, env.Detect(env.get('CC', compilers)))

SCons/Tool/gxx.py

@@ -32,6 +32,10 @@
 """
 
 
+import os, sys
+import threading
+import asyncio
+import atexit
 import SCons.Tool
 import SCons.Util
 
@@ -41,6 +45,159 @@
 compilers = ['g++']
 
 
+def CODY_decode(input):
+    quoted = False
+    backslashed = False
+    result = []
+    output = ""
+
+    for c in input:
+        if quoted:
+            if backslashed:
+                output.append(c)
+                backslashed = False
+                continue
+            if c == "'":
+                quoted = False
+                continue
+            if c == "\\":
+                backslashed = True
+                continue
+            output += c
+            continue
+
+        if c == "'":
+            quoted = True
+            continue
+        if c == ' ' and output:
+            result.append(output)
+            output = ""
+            continue
+        output += c
+    if output:
+        result.append(output)
+
+    return result
+
+
+class module_mapper(threading.Thread):
+    def __init__(self, env, *args, **kw):
+        super().__init__(*args, **kw)
+        self.daemon = True
+        self.loop = asyncio.new_event_loop()
+
+        self.env = env
+        self.load_map()
+        atexit.register(self.save_map)
+
+        self.request_dispatch = {}
+        self.request_dispatch["HELLO"] = self.hello_response
+        self.request_dispatch["INCLUDE-TRANSLATE"] = self.include_translate_response
+        self.request_dispatch["MODULE-REPO"] = self.module_repo_response
+        self.request_dispatch["MODULE-EXPORT"] = self.module_export_response
+        self.request_dispatch["MODULE-COMPILED"] = self.module_compiled_response
+        self.request_dispatch["MODULE-IMPORT"] = self.module_import_response
+
+    def save_map(self):
+        save = open(self.env.subst("$CXXMODULEPATH/module.map"), "w")
+
+        save.writelines(
+            [self.env.subst("$$root $CXXMODULEPATH") + '\n'] +
+            [' '.join(pair) + '\n' for pair in self.env["CXXMODULEMAP"].items()])
+
+    def load_map(self):
+        try:
+            load = open(self.env.subst("$CXXMODULEPATH/module.map"), "r")
+        except FileNotFoundError:
+            return
+
+        saved_map = dict([tuple(line.rstrip("\n").split(maxsplit=1))
+                         for line in load.readlines() if not line[0] == '$'])
+        saved_map.update(self.env["CXXMODULEMAP"])
+        self.env["CXXMODULEMAP"] = saved_map
+
+    def run(self):
+        self.loop.run_forever()
+
+    def hello_response(self, request, sourcefile):
+        if(sourcefile != ""):
+            return ("ERROR", "'Unexpected handshake'")
+        if len(request) == 4 and request[1] == "1":
+            return (request[3], "HELLO", "1", "SCONS", "''")
+        else:
+            return ("ERROR", "'Invalid handshake'")
+
+    def module_repo_response(self, request, sourcefile):
+        return ("PATHNAME", self.env["CXXMODULEPATH"])
+
+    def include_translate_response(self, request, sourcefile):
+        return ("BOOL", "TRUE")
+
+    def module_export_response(self, request, sourcefile):
+        if sourcefile[0] == '@':
+            cmi = self.env.subst(sourcefile + "$CXXMODULESUFFIX")
+            self.env["CXXMODULEMAP"][request[1]] = cmi
+            return ("PATHNAME", cmi)
+        else:
+            return ("PATHNAME", self.env["CXXMODULEMAP"].get(request[1], self.env.subst(request[1] + "$CXXMODULESUFFIX")))
+
+    def module_compiled_response(self, request, sourcefile):
+        return ("OK")
+
+    def module_import_response(self, request, sourcefile):
+        return ("PATHNAME", self.env["CXXMODULEMAP"].get(request[1], self.env.subst(request[1] + "$CXXMODULESUFFIX")))
+
+    def default_response(self, request, sourcefile):
+        return ("ERROR", "'Unknown CODY request {}'".format(request[0]))
+
+    async def handle_connect(self, reader, writer):
+        sourcefile = ""
+        while True:
+            try:
+                request = await reader.readuntil()
+            except EOFError:
+                return
+
+            request = request.decode("utf-8")
+
+            separator = ''
+            request = request.rstrip('\n')
+            if request[-1] == ';':
+                request = request.rstrip(';')
+                separator = ' ;'
+
+            request = CODY_decode(request)
+
+            response = self.request_dispatch.get(
+                request[0], self.default_response)(request, sourcefile)
+            if(request[0] == "HELLO" and response[0] != "ERROR"):
+                sourcefile = response[0]
+                response = response[1:]
+            response = " ".join(response) + separator + "\n"
+
+            writer.write(response.encode())
+            await writer.drain()
+
+    async def listen(self, path):
+        if self.env["__GCCMAPPERMODE__"] == "domain_socket":
+            await asyncio.start_unix_server(self.handle_connect, path=path)
+        else:
+            srv = await asyncio.start_server(self.handle_connect, host="::1", port=0)
+            self.env["__GCCMAPPERPORT__"] = srv.sockets[0].getsockname()[1]
+
+
+def init_mapper(env):
+    if env.get("__GCCMODULEMAPPER__"):
+        return
+
+    mapper = module_mapper(env)
+    mapper.start()
+    os.makedirs(env.subst("$CXXMODULEPATH"), exist_ok=True)
+    asyncio.run_coroutine_threadsafe(mapper.listen(
+        env.subst("$CXXMODULEPATH/socket")), mapper.loop)
+
+    env["__GCCMODULEMAPPER__"] = mapper
+
 def generate(env) -> None:
     """Add Builders and construction variables for g++ to an Environment."""
     static_obj, shared_obj = SCons.Tool.createObjBuilders(env)
@@ -67,6 +224,17 @@ def generate(env) -> None:
     env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d'
     env["NINJA_DEPFILE_PARSE_FORMAT"] = 'gcc'
 
+    env['__CXXMODULEINIT__'] = init_mapper
+    if sys.platform != "win32":
+        env['__GCCMAPPERMODE__'] = "domain_socket"
+        env['__GCCMAPPERFLAGS__'] = "=$CXXMODULEPATH/socket"
+    else:
+        env['__GCCMAPPERMODE__'] = "tcp"
+        env['__GCCMAPPERFLAGS__'] = "::1:$($__GCCMAPPERPORT__$)"
+    env['CXXMODULEFLAGS'] = '-fmodules-ts -fmodule-mapper=$__GCCMAPPERFLAGS__?$CXXMODULEIDPREFIX$SOURCE'
+    env['CXXMODULESUFFIX'] = '.gcm'
+    env['CXXUSERHEADERFLAGS']    = '-x c++-user-header'
+    env['CXXSYSTEMHEADERFLAGS']  = '-x c++-system-header'
 
 
 def exists(env):

test/CXX/CXX-modules-fixture/SConstruct

@@ -0,0 +1,9 @@
+env = Environment(tools = ["link"])
+try:
+    env.Tool(ARGUMENTS["toolset"])
+except KeyError:
+    pass
+
+env.Append(CXXFLAGS = ["-std=c++20"])
+env['CXXMODULEPATH'] = "cxx-scons-modules"
+env.Program("scons-module-test", ["itest.cpp", "test.cpp", "main.cpp"])

test/CXX/CXX-modules-fixture/itest.cpp

@@ -0,0 +1,3 @@
+module test;
+
+int i = 42;

test/CXX/CXX-modules-fixture/main.cpp

@@ -0,0 +1,12 @@
+import test;
+
+import <iostream>;
+
+int main()
+{
+	std::cout << i << std::endl;
+	test();
+	int i = fact<4>::value;
+	std::cout << i << std::endl;
+	return 0;
+}

test/CXX/CXX-modules-fixture/test.cpp

@@ -0,0 +1,24 @@
+module;
+
+#include <iostream>
+#include "incl.h"
+
+export module test;
+export void test()
+{
+	std::cout << "hello, world\n";
+}
+
+export template<unsigned int> struct fact;
+
+export extern int i;
+
+template<unsigned int n>
+struct fact {
+    static constexpr unsigned int value = n * fact<n-1>::value;
+};
+
+template<>
+struct fact<0> {
+    static constexpr unsigned int value = 1;
+};