Introduce universal compilation host to execution framework

libsolidity/interface/CompilerStack.cpp

@@ -1256,6 +1256,16 @@ ContractDefinition const& CompilerStack::contractDefinition(std::string const& _
 	return *contract(_contractName).contract;
 }
 
+std::vector<ContractDefinition const*> CompilerStack::contractDefinitions(
+	std::string const& _sourceName
+) const
+{
+	auto const& source = m_sources.find(_sourceName);
+	solAssert(source != m_sources.end(), "No source found.");
+
+	return ASTNode::filteredNodes<ContractDefinition>(source->second.ast->nodes());
+}
+
 size_t CompilerStack::functionEntryPoint(
 	std::string const& _contractName,
 	FunctionDefinition const& _function

libsolidity/interface/CompilerStack.h

@@ -307,6 +307,9 @@ class CompilerStack: public langutil::CharStreamProvider, public evmasm::Abstrac
 	/// does not exist.
 	ContractDefinition const& contractDefinition(std::string const& _contractName) const;
 
+	/// @returns a list of parsed contracts per source unit with the supplied name.
+	std::vector<ContractDefinition const*> contractDefinitions(std::string const& _sourceName) const;
+
 	/// @returns a list of unhandled queries to the SMT solver (has to be supplied in a second run
 	/// by calling @a addSMTLib2Response).
 	std::vector<std::string> const& unhandledSMTLib2Queries() const { return m_unhandledSMTLib2Queries; }

test/CMakeLists.txt

@@ -121,6 +121,12 @@ set(libsolidity_sources
 detect_stray_source_files("${libsolidity_sources}" "libsolidity/")
 
 set(libsolidity_util_sources
+    libsolidity/util/compiler/Compiler.h
+    libsolidity/util/compiler/Compiler.cpp
+    libsolidity/util/compiler/CompilerHost.h
+    libsolidity/util/compiler/CompilerHost.cpp
+    libsolidity/util/compiler/InternalCompiler.h
+    libsolidity/util/compiler/InternalCompiler.cpp
     libsolidity/util/BytesUtils.cpp
     libsolidity/util/BytesUtilsTests.cpp
     libsolidity/util/BytesUtils.h

test/ExecutionFramework.h

@@ -65,7 +65,7 @@ class ExecutionFramework
 		std::string const& _contractName = "",
 		bytes const& _arguments = {},
 		std::map<std::string, util::h160> const& _libraryAddresses = {},
-		std::optional<std::string> const& _sourceName = std::nullopt
+		std::optional<std::string> const& _mainSourceName = std::nullopt
 	) = 0;
 
 	bytes const& compileAndRun(

test/libsolidity/GasCosts.cpp

@@ -19,7 +19,9 @@
  * Tests that check that the cost of certain operations stay within range.
  */
 
+#include <test/libsolidity/util/SoltestErrors.h>
 #include <test/libsolidity/SolidityExecutionFramework.h>
+
 #include <liblangutil/EVMVersion.h>
 #include <libsolutil/IpfsHash.h>
 #include <libevmasm/GasMeter.h>
@@ -38,7 +40,10 @@ namespace solidity::frontend::test
 #define CHECK_DEPLOY_GAS(_gasNoOpt, _gasOpt, _evmVersion) \
 	do \
 	{ \
-		u256 metaCost = GasMeter::dataGas(m_compiler.cborMetadata(m_compiler.lastContractName()), true, _evmVersion); \
+		auto const& output = m_compiler.output(); \
+		auto const* contract = output.contract(); \
+		soltestAssert(contract); \
+		u256 metaCost = GasMeter::dataGas(contract->cborMetadata, true, _evmVersion); \
 		u256 gasOpt{_gasOpt}; \
 		u256 gasNoOpt{_gasNoOpt}; \
 		u256 gas = m_optimiserSettings == OptimiserSettings::minimal() ? gasNoOpt : gasOpt; \
@@ -90,7 +95,7 @@ BOOST_AUTO_TEST_CASE(string_storage)
 			}
 		}
 	)";
-	m_compiler.setMetadataFormat(CompilerStack::MetadataFormat::NoMetadata);
+
 	m_appendCBORMetadata = false;
 	compileAndRun(sourceCode);
 
@@ -202,8 +207,15 @@ BOOST_AUTO_TEST_CASE(single_callvaluecheck)
 		}
 	)";
 	compileAndRun(sourceCode);
-	size_t bytecodeSizeNonpayable = m_compiler.object("Nonpayable").bytecode.size();
-	size_t bytecodeSizePayable = m_compiler.object("Payable").bytecode.size();
+	auto const& output = m_compiler.output();
+	auto const* nonpayable = output.contract("Nonpayable");
+	auto const* payable = output.contract("Payable");
+
+	soltestAssert(nonpayable);
+	soltestAssert(payable);
+
+	size_t bytecodeSizeNonpayable = nonpayable->object.size();
+	size_t bytecodeSizePayable = payable->object.size();
 
 	auto evmVersion = solidity::test::CommonOptions::get().evmVersion();
 	if (evmVersion < EVMVersion::shanghai())

test/libsolidity/GasMeter.cpp

@@ -22,9 +22,14 @@
  */
 
 #include <test/libsolidity/SolidityExecutionFramework.h>
+
+#include <test/libsolidity/util/Common.h>
+#include <test/libsolidity/util/SoltestErrors.h>
+
 #include <libevmasm/GasMeter.h>
 #include <libevmasm/KnownState.h>
 #include <libevmasm/PathGasMeter.h>
+
 #include <libsolidity/ast/AST.h>
 #include <libsolidity/interface/GasEstimator.h>
 
@@ -43,24 +48,39 @@ class GasMeterTestFramework: public SolidityExecutionFramework
 	void compile(std::string const& _sourceCode)
 	{
 		m_compiler.reset();
-		m_compiler.setSources({{"", "pragma solidity >=0.0;\n"
-				"// SPDX-License-Identifier: GPL-3.0\n" + _sourceCode}});
-		m_compiler.setOptimiserSettings(solidity::test::CommonOptions::get().optimize);
-		m_compiler.setEVMVersion(m_evmVersion);
-		BOOST_REQUIRE_MESSAGE(m_compiler.compile(), "Compiling contract failed");
+		m_compilerInput = CompilerInput{};
+
+		m_compilerInput.sourceCode = {{"", withPreamble(_sourceCode)}};
+		m_compilerInput.optimise = CommonOptions::get().optimize;
+		m_compilerInput.evmVersion = std::make_optional(m_evmVersion);
+
+		m_compiler.compile(m_compilerInput);
+
+		BOOST_REQUIRE_MESSAGE(m_compiler.output().success(), "Compiling contract failed");
 	}
 
 	void testCreationTimeGas(std::string const& _sourceCode, u256 const& _tolerance = u256(0))
 	{
 		compileAndRun(_sourceCode);
+
 		auto state = std::make_shared<KnownState>();
-		PathGasMeter meter(*m_compiler.assemblyItems(m_compiler.lastContractName()), solidity::test::CommonOptions::get().evmVersion());
+		auto const& output = m_compiler.output();
+		auto const* contract = output.contract();
+
+		soltestAssert(contract);
+		soltestAssert(contract->assemblyItems.has_value());
+
+		bytes const& object = contract->object;
+		bytes const& runtimeObject = contract->runtimeObject;
+		auto const& assemblyItems = contract->assemblyItems.value();
+
+		PathGasMeter meter(assemblyItems, CommonOptions::get().evmVersion());
 		GasMeter::GasConsumption gas = meter.estimateMax(0, state);
-		u256 bytecodeSize(m_compiler.runtimeObject(m_compiler.lastContractName()).bytecode.size());
+		u256 bytecodeSize(runtimeObject.size());
 		// costs for deployment
 		gas += bytecodeSize * GasCosts::createDataGas;
 		// costs for transaction
-		gas += gasForTransaction(m_compiler.object(m_compiler.lastContractName()).bytecode, true);
+		gas += gasForTransaction(object, true);
 
 		BOOST_REQUIRE(!gas.isInfinite);
 		BOOST_CHECK_LE(m_gasUsed, gas.value);
@@ -71,6 +91,14 @@ class GasMeterTestFramework: public SolidityExecutionFramework
 	/// against the actual gas usage computed by the VM on the given set of argument variants.
 	void testRunTimeGas(std::string const& _sig, std::vector<bytes> _argumentVariants, u256 const& _tolerance = u256(0))
 	{
+		auto const& output = m_compiler.output();
+		auto const* contract = output.contract();
+
+		soltestAssert(contract);
+		soltestAssert(contract->runtimeAssemblyItems.has_value());
+
+		auto const& runtimeAssemblyItems = contract->runtimeAssemblyItems.value();
+
 		u256 gasUsed = 0;
 		GasMeter::GasConsumption gas;
 		util::FixedHash<4> hash = util::selectorFromSignatureH32(_sig);
@@ -82,8 +110,8 @@ class GasMeterTestFramework: public SolidityExecutionFramework
 			gas = std::max(gas, gasForTransaction(hash.asBytes() + arguments, false));
 		}
 
-		gas += GasEstimator(solidity::test::CommonOptions::get().evmVersion()).functionalEstimation(
-			*m_compiler.runtimeAssemblyItems(m_compiler.lastContractName()),
+		gas += GasEstimator(CommonOptions::get().evmVersion()).functionalEstimation(
+			runtimeAssemblyItems,
 			_sig
 		);
 		BOOST_REQUIRE(!gas.isInfinite);
@@ -93,7 +121,7 @@ class GasMeterTestFramework: public SolidityExecutionFramework
 
 	static GasMeter::GasConsumption gasForTransaction(bytes const& _data, bool _isCreation)
 	{
-		auto evmVersion = solidity::test::CommonOptions::get().evmVersion();
+		auto evmVersion = CommonOptions::get().evmVersion();
 		GasMeter::GasConsumption gas = _isCreation ? GasCosts::txCreateGas : GasCosts::txGas;
 		for (auto i: _data)
 			gas += i != 0 ? GasCosts::txDataNonZeroGas(evmVersion) : GasCosts::txDataZeroGas;

test/libsolidity/SemanticTest.cpp

@@ -118,8 +118,8 @@ SemanticTest::SemanticTest(
 
 	if (m_enforceGasCost)
 	{
-		m_compiler.setMetadataFormat(CompilerStack::MetadataFormat::NoMetadata);
-		m_compiler.setMetadataHash(CompilerStack::MetadataHash::None);
+		m_compilerInput.metadataFormat = MetadataFormat::NoMetadata;
+		m_compilerInput.metadataHash = MetadataHash::None;
 	}
 }
 
@@ -196,7 +196,7 @@ std::vector<SideEffectHook> SemanticTest::makeSideEffectHooks() const
 	};
 }
 
-std::string SemanticTest::formatEventParameter(std::optional<AnnotatedEventSignature> _signature, bool _indexed, size_t _index, bytes const& _data)
+std::string SemanticTest::formatEventParameter(EventSignature const* _signature, bool _indexed, size_t _index, bytes const& _data)
 {
 	auto isPrintableASCII = [](bytes const& s)
 	{
@@ -217,7 +217,7 @@ std::string SemanticTest::formatEventParameter(std::optional<AnnotatedEventSigna
 	ABIType abiType(ABIType::Type::Hex);
 	if (isPrintableASCII(_data))
 		abiType = ABIType(ABIType::Type::String);
-	if (_signature.has_value())
+	if (_signature)
 	{
 		std::vector<std::string> const& types = _indexed ? _signature->indexedTypes : _signature->nonIndexedTypes;
 		if (_index < types.size())
@@ -231,17 +231,19 @@ std::string SemanticTest::formatEventParameter(std::optional<AnnotatedEventSigna
 
 std::vector<std::string> SemanticTest::eventSideEffectHook(FunctionCall const&) const
 {
+	auto const& output = m_compiler.output();
+
 	std::vector<std::string> sideEffects;
 	std::vector<LogRecord> recordedLogs = ExecutionFramework::recordedLogs();
 	for (LogRecord const& log: recordedLogs)
 	{
-		std::optional<AnnotatedEventSignature> eventSignature;
+		EventSignature const* eventSignature = nullptr;
 		if (!log.topics.empty())
-			eventSignature = matchEvent(log.topics[0]);
+			eventSignature = output.matchEvent(log.topics[0]);
 		std::stringstream sideEffect;
 		sideEffect << "emit ";
-		if (eventSignature.has_value())
-			sideEffect << eventSignature.value().signature;
+		if (eventSignature)
+			sideEffect << eventSignature->signature;
 		else
 			sideEffect << "<anonymous>";
 
@@ -252,7 +254,7 @@ std::vector<std::string> SemanticTest::eventSideEffectHook(FunctionCall const&)
 		size_t index{0};
 		for (h256 const& topic: log.topics)
 		{
-			if (!eventSignature.has_value() || index != 0)
+			if (!eventSignature || index != 0)
 				eventStrings.push_back("#" + formatEventParameter(eventSignature, true, index, topic.asBytes()));
 			++index;
 		}
@@ -273,31 +275,6 @@ std::vector<std::string> SemanticTest::eventSideEffectHook(FunctionCall const&)
 	return sideEffects;
 }
 
-std::optional<AnnotatedEventSignature> SemanticTest::matchEvent(util::h256 const& hash) const
-{
-	std::optional<AnnotatedEventSignature> result;
-	for (std::string& contractName: m_compiler.contractNames())
-	{
-		ContractDefinition const& contract = m_compiler.contractDefinition(contractName);
-		for (EventDefinition const* event: contract.events() + contract.usedInterfaceEvents())
-		{
-			FunctionTypePointer eventFunctionType = event->functionType(true);
-			if (!event->isAnonymous() && keccak256(eventFunctionType->externalSignature()) == hash)
-			{
-				AnnotatedEventSignature eventInfo;
-				eventInfo.signature = eventFunctionType->externalSignature();
-				for (auto const& param: event->parameters())
-					if (param->isIndexed())
-						eventInfo.indexedTypes.emplace_back(param->type()->toString(true));
-					else
-						eventInfo.nonIndexedTypes.emplace_back(param->type()->toString(true));
-				result = eventInfo;
-			}
-		}
-	}
-	return result;
-}
-
 frontend::OptimiserSettings SemanticTest::optimizerSettingsFor(RequiresYulOptimizer _requiresYulOptimizer)
 {
 	switch (_requiresYulOptimizer)
@@ -364,7 +341,7 @@ TestCase::TestResult SemanticTest::runTest(
 	for (TestFunctionCall& test: m_tests)
 		test.reset();
 
-	std::map<std::string, solidity::test::Address> libraries;
+	std::map<std::string, Address> libraries;
 
 	bool constructed = false;
 
@@ -383,13 +360,14 @@ TestCase::TestResult SemanticTest::runTest(
 		}
 		else if (test.call().kind == FunctionCall::Kind::Library)
 		{
+			std::string name = test.call().libraryFile + ":" + test.call().signature;
 			soltestAssert(
-				deploy(test.call().signature, 0, {}, libraries) && m_transactionSuccessful,
+				deploy(name, 0, {}, libraries) && m_transactionSuccessful,
 				"Failed to deploy library " + test.call().signature);
 			// For convenience, in semantic tests we assume that an unqualified name like `L` is equivalent to one
 			// with an empty source unit name (`:L`). This is fine because the compiler never uses unqualified
 			// names in the Yul code it produces and does not allow `linkersymbol()` at all in inline assembly.
-			libraries[test.call().libraryFile + ":" + test.call().signature] = m_contractAddress;
+			libraries[name] = m_contractAddress;
 			continue;
 		}
 		else
@@ -416,7 +394,13 @@ TestCase::TestResult SemanticTest::runTest(
 		}
 		else
 		{
+			ContractName contractName{m_sources.mainSourceFile, ""};
+
+			auto const* contract = m_compiler.output().contract(contractName);
+			soltestAssert(contract);
+
 			bytes output;
+
 			if (test.call().kind == FunctionCall::Kind::LowLevel)
 				output = callLowLevel(test.call().arguments.rawBytes(), test.call().value.value);
 			else if (test.call().kind == FunctionCall::Kind::Builtin)
@@ -432,9 +416,10 @@ TestCase::TestResult SemanticTest::runTest(
 			}
 			else
 			{
+				soltestAssert(contract->interfaceSymbols.has_value());
 				soltestAssert(
 					m_allowNonExistingFunctions ||
-					m_compiler.interfaceSymbols(m_compiler.lastContractName(m_sources.mainSourceFile))["methods"].contains(test.call().signature),
+					contract->interfaceSymbols.value()["methods"].contains(test.call().signature),
 					"The function " + test.call().signature + " is not known to the compiler"
 				);
 
@@ -462,7 +447,8 @@ TestCase::TestResult SemanticTest::runTest(
 			test.setFailure(!m_transactionSuccessful);
 			test.setRawBytes(std::move(output));
 			if (test.call().kind != FunctionCall::Kind::LowLevel)
-				test.setContractABI(m_compiler.contractABI(m_compiler.lastContractName(m_sources.mainSourceFile)));
+				if (contract->contractABI.has_value())
+					test.setContractABI(contract->contractABI.value());
 		}
 
 		std::vector<std::string> effects;

test/libsolidity/SemanticTest.h

@@ -30,13 +30,6 @@
 namespace solidity::frontend::test
 {
 
-struct AnnotatedEventSignature
-{
-	std::string signature;
-	std::vector<std::string> indexedTypes;
-	std::vector<std::string> nonIndexedTypes;
-};
-
 enum class RequiresYulOptimizer
 {
 	False,
@@ -107,8 +100,7 @@ class SemanticTest: public SolidityExecutionFramework, public EVMVersionRestrict
 	std::map<std::string, Builtin> makeBuiltins();
 	std::vector<SideEffectHook> makeSideEffectHooks() const;
 	std::vector<std::string> eventSideEffectHook(FunctionCall const&) const;
-	std::optional<AnnotatedEventSignature> matchEvent(util::h256 const& hash) const;
-	static std::string formatEventParameter(std::optional<AnnotatedEventSignature> _signature, bool _indexed, size_t _index, bytes const& _data);
+	static std::string formatEventParameter(EventSignature const* _signature, bool _indexed, size_t _index, bytes const& _data);
 
 	OptimiserSettings optimizerSettingsFor(RequiresYulOptimizer _requiresYulOptimizer);