From 43fe553ff002a60b1b2449250a530169b8db14fd Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Mon, 21 Feb 2022 18:56:05 -0500 Subject: [PATCH 01/10] [solc] Add --import-asm-json input mode. --- libsolidity/interface/CompilerStack.cpp | 112 ++++++++++++++-------- libsolidity/interface/CompilerStack.h | 5 + libsolidity/interface/OptimiserSettings.h | 2 + solc/CommandLineInterface.cpp | 57 ++++++++++- solc/CommandLineInterface.h | 2 + solc/CommandLineParser.cpp | 65 ++++++++----- solc/CommandLineParser.h | 3 +- 7 files changed, 173 insertions(+), 73 deletions(-) diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 3eb25987db42..48936e2d8c56 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -414,6 +414,28 @@ void CompilerStack::importASTs(map const& _sources) storeContractDefinitions(); } +void CompilerStack::importEvmAssemblyJson(std::map const& _sources) +{ + solAssert(_sources.size() == 1, ""); + solAssert(m_sources.empty(), ""); + solAssert(m_sourceOrder.empty(), ""); + if (m_stackState != Empty) + solThrow(CompilerError, "Must call importEvmAssemblyJson only before the SourcesSet state."); + + Json::Value jsonValue = _sources.begin()->second; + if (jsonValue.isMember("sourceList")) + for (auto const& item: jsonValue["sourceList"]) + { + Source source; + source.charStream = std::make_shared(item.asString(), ""); + m_sources.emplace(std::make_pair(item.asString(), source)); + m_sourceOrder.push_back(&m_sources[item.asString()]); + } + m_evmAssemblyJson[_sources.begin()->first] = jsonValue; + m_importedSources = true; + m_stackState = SourcesSet; +} + bool CompilerStack::analyze() { if (m_stackState != ParsedAndImported || m_stackState >= AnalysisPerformed) @@ -600,6 +622,9 @@ bool CompilerStack::parseAndAnalyze(State _stopAfter) { m_stopAfter = _stopAfter; + if (!m_evmAssemblyJson.empty()) + return true; + bool success = parse(); if (m_stackState >= m_stopAfter) return success; @@ -649,55 +674,58 @@ bool CompilerStack::compile(State _stopAfter) // Only compile contracts individually which have been requested. map> otherCompilers; - for (Source const* source: m_sourceOrder) - for (ASTPointer const& node: source->ast->nodes()) - if (auto contract = dynamic_cast(node.get())) - if (isRequestedContract(*contract)) - { - try + if (!m_evmAssemblyJson.empty()) + { + + } + else + { + for (Source const* source: m_sourceOrder) + for (ASTPointer const& node: source->ast->nodes()) + if (auto contract = dynamic_cast(node.get())) + if (isRequestedContract(*contract)) { - if (m_viaIR || m_generateIR || m_generateEwasm) - generateIR(*contract); - if (m_generateEvmBytecode) + try { - if (m_viaIR) - generateEVMFromIR(*contract); - else - compileContract(*contract, otherCompilers); + if (m_viaIR || m_generateIR || m_generateEwasm) + generateIR(*contract); + if (m_generateEvmBytecode) + { + if (m_viaIR) + generateEVMFromIR(*contract); + else + compileContract(*contract, otherCompilers); + } + if (m_generateEwasm) + generateEwasm(*contract); } - if (m_generateEwasm) - generateEwasm(*contract); - } - catch (Error const& _error) - { - if (_error.type() != Error::Type::CodeGenerationError) - throw; - m_errorReporter.error(_error.errorId(), _error.type(), SourceLocation(), _error.what()); - return false; - } - catch (UnimplementedFeatureError const& _unimplementedError) - { - if ( - SourceLocation const* sourceLocation = - boost::get_error_info(_unimplementedError) - ) + catch (Error const& _error) { - string const* comment = _unimplementedError.comment(); - m_errorReporter.error( - 1834_error, - Error::Type::CodeGenerationError, - *sourceLocation, - "Unimplemented feature error" + - ((comment && !comment->empty()) ? ": " + *comment : string{}) + - " in " + - _unimplementedError.lineInfo() - ); + if (_error.type() != Error::Type::CodeGenerationError) + throw; + m_errorReporter.error(_error.errorId(), _error.type(), SourceLocation(), _error.what()); return false; } - else - throw; + catch (UnimplementedFeatureError const& _unimplementedError) + { + if (SourceLocation const* sourceLocation + = boost::get_error_info(_unimplementedError)) + { + string const* comment = _unimplementedError.comment(); + m_errorReporter.error( + 1834_error, + Error::Type::CodeGenerationError, + *sourceLocation, + "Unimplemented feature error" + + ((comment && !comment->empty()) ? ": " + *comment : string{}) + " in " + + _unimplementedError.lineInfo()); + return false; + } + else + throw; + } } - } + } m_stackState = CompilationSuccessful; this->link(); return true; diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index c1f15a480426..5618a9574bc7 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -222,6 +222,10 @@ class CompilerStack: public langutil::CharStreamProvider /// Will throw errors if the import fails void importASTs(std::map const& _sources); + /// Imports given Evm Assembly Json. Leads to the same internal state as parse(). + /// Will throw errors if the import fails + void importEvmAssemblyJson(std::map const& _sources); + /// Performs the analysis steps (imports, scopesetting, syntaxCheck, referenceResolving, /// typechecking, staticAnalysis) on previously parsed sources. /// @returns false on error. @@ -499,6 +503,7 @@ class CompilerStack: public langutil::CharStreamProvider std::map m_sources; // if imported, store AST-JSONS for each filename std::map m_sourceJsons; + std::map m_evmAssemblyJson; std::vector m_unhandledSMTLib2Queries; std::map m_smtlib2Responses; std::shared_ptr m_globalContext; diff --git a/libsolidity/interface/OptimiserSettings.h b/libsolidity/interface/OptimiserSettings.h index e233048c18f1..2e960e8d3ca7 100644 --- a/libsolidity/interface/OptimiserSettings.h +++ b/libsolidity/interface/OptimiserSettings.h @@ -149,6 +149,8 @@ struct OptimiserSettings /// This specifies an estimate on how often each opcode in this assembly will be executed, /// i.e. use a small value to optimise for size and a large value to optimise for runtime gas usage. size_t expectedExecutionsPerDeployment = 200; + /// Flag reflecting whether optimizer is enabled. + bool enabled = false; }; } diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index 03679bd7c57d..09d305282a59 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -555,6 +555,25 @@ map CommandLineInterface::parseAstFromInput() return sourceJsons; } +map CommandLineInterface::parseEvmAssemblyJsonFromInput() +{ + solAssert(m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, ""); + solAssert(m_fileReader.sourceUnits().size() == 1, ""); + + map sourceJsons; + + for (auto const& iter: m_fileReader.sourceUnits()) + { + Json::Value evmAsmJson; + astAssert(jsonParseStrict(iter.second, evmAsmJson), "Input file could not be parsed to JSON"); + astAssert(evmAsmJson.isMember(".code"), "Invalid Format for assembly-JSON: Must have '.code'-object"); + astAssert(evmAsmJson.isMember(".data"), "Invalid Format for assembly-JSON: Must have '.data'-object"); + sourceJsons[iter.first] = evmAsmJson; + } + + return sourceJsons; +} + void CommandLineInterface::createFile(string const& _fileName, string const& _data) { namespace fs = boost::filesystem; @@ -658,6 +677,7 @@ void CommandLineInterface::processInput() break; case InputMode::Compiler: case InputMode::CompilerWithASTImport: + case InputMode::CompilerWithEvmAssemblyJsonImport: compile(); outputCompilationResults(); } @@ -678,7 +698,11 @@ void CommandLineInterface::printLicense() void CommandLineInterface::compile() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); m_compiler = make_unique(m_fileReader.reader()); @@ -725,7 +749,18 @@ void CommandLineInterface::compile() m_compiler->setOptimiserSettings(m_options.optimiserSettings()); - if (m_options.input.mode == InputMode::CompilerWithASTImport) + if (m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport) + { + try + { + m_compiler->importEvmAssemblyJson(parseEvmAssemblyJsonFromInput()); + } + catch (Exception const& _exc) + { + solThrow(CommandLineExecutionError, "Failed to import Evm Assembly JSON: "s + _exc.what()); + } + } + else if (m_options.input.mode == InputMode::CompilerWithASTImport) { try { @@ -785,7 +820,11 @@ void CommandLineInterface::compile() void CommandLineInterface::handleCombinedJSON() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); if (!m_options.compiler.combinedJsonRequests.has_value()) return; @@ -877,7 +916,11 @@ void CommandLineInterface::handleCombinedJSON() void CommandLineInterface::handleAst() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); if (!m_options.compiler.outputs.astCompactJson) return; @@ -1121,7 +1164,11 @@ void CommandLineInterface::assemble(yul::AssemblyStack::Language _language, yul: void CommandLineInterface::outputCompilationResults() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); handleCombinedJSON(); diff --git a/solc/CommandLineInterface.h b/solc/CommandLineInterface.h index 951731825cc6..de487aa107b6 100644 --- a/solc/CommandLineInterface.h +++ b/solc/CommandLineInterface.h @@ -115,6 +115,8 @@ class CommandLineInterface /// or standard-json output std::map parseAstFromInput(); + std::map parseEvmAssemblyJsonFromInput(); + /// Create a file in the given directory /// @arg _fileName the name of the file /// @arg _data to be written diff --git a/solc/CommandLineParser.cpp b/solc/CommandLineParser.cpp index 8f6d549f202c..a306820b167e 100644 --- a/solc/CommandLineParser.cpp +++ b/solc/CommandLineParser.cpp @@ -51,6 +51,7 @@ static string const g_strExperimentalViaIR = "experimental-via-ir"; static string const g_strGas = "gas"; static string const g_strHelp = "help"; static string const g_strImportAst = "import-ast"; +static string const g_strImportEvmAssemblerJson = "import-asm-json"; static string const g_strInputFile = "input-file"; static string const g_strYul = "yul"; static string const g_strYulDialect = "yul-dialect"; @@ -137,6 +138,7 @@ static map const g_inputModeName = { {InputMode::StandardJson, "standard JSON"}, {InputMode::Linker, "linker"}, {InputMode::LanguageServer, "language server (LSP)"}, + {InputMode::CompilerWithEvmAssemblyJsonImport, "assembler (EVM ASM JSON import)"} }; void CommandLineParser::checkMutuallyExclusive(vector const& _optionNames) @@ -166,7 +168,7 @@ ostream& operator<<(ostream& _out, CompilerOutputs const& _selection) if (_selection.*component) serializedSelection.push_back(CompilerOutputs::componentName(component)); - return _out << util::joinHumanReadable(serializedSelection, ","); + return _out << joinHumanReadable(serializedSelection, ","); } string const& CompilerOutputs::componentName(bool CompilerOutputs::* _component) @@ -197,7 +199,7 @@ ostream& operator<<(ostream& _out, CombinedJsonRequests const& _requests) if (_requests.*component) serializedRequests.push_back(CombinedJsonRequests::componentName(component)); - return _out << util::joinHumanReadable(serializedRequests, ","); + return _out << joinHumanReadable(serializedRequests, ","); } string const& CombinedJsonRequests::componentName(bool CombinedJsonRequests::* _component) @@ -267,6 +269,8 @@ OptimiserSettings CommandLineOptions::optimiserSettings() const if (optimizer.yulSteps.has_value()) settings.yulOptimiserSteps = optimizer.yulSteps.value(); + settings.enabled = optimizer.enabled; + return settings; } @@ -316,20 +320,20 @@ void CommandLineParser::parseInputPathsAndRemappings() m_options.input.paths.insert(positionalArg); } - if (m_options.input.mode == InputMode::StandardJson) + if (m_options.input.mode == InputMode::StandardJson || m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport) { if (m_options.input.paths.size() > 1 || (m_options.input.paths.size() == 1 && m_options.input.addStdin)) solThrow( CommandLineValidationError, - "Too many input files for --" + g_strStandardJSON + ".\n" + "Too many input files for --" + (m_options.input.mode == InputMode::StandardJson ? g_strStandardJSON : g_strImportEvmAssemblerJson) + ".\n" "Please either specify a single file name or provide its content on standard input." ); - else if (m_options.input.paths.size() == 0) + else if (m_options.input.paths.empty()) // Standard JSON mode input used to be handled separately and zero files meant "read from stdin". // Keep it working that way for backwards-compatibility. m_options.input.addStdin = true; } - else if (m_options.input.paths.size() == 0 && !m_options.input.addStdin) + else if (m_options.input.paths.empty() && !m_options.input.addStdin) solThrow( CommandLineValidationError, "No input files given. If you wish to use the standard input please specify \"-\" explicitly." @@ -343,17 +347,17 @@ void CommandLineParser::parseLibraryOption(string const& _input) try { if (fs::is_regular_file(_input)) - data = util::readFileAsString(_input); + data = readFileAsString(_input); } catch (fs::filesystem_error const&) { // Thrown e.g. if path is too long. } - catch (util::FileNotFound const&) + catch (FileNotFound const&) { // Should not happen if `fs::is_regular_file` is correct. } - catch (util::NotAFile const&) + catch (NotAFile const&) { // Should not happen if `fs::is_regular_file` is correct. } @@ -418,15 +422,15 @@ void CommandLineParser::parseLibraryOption(string const& _input) "Invalid length for address for library \"" + libName + "\": " + to_string(addrString.length()) + " instead of 40 characters." ); - if (!util::passesAddressChecksum(addrString, false)) + if (!passesAddressChecksum(addrString, false)) solThrow( CommandLineValidationError, "Invalid checksum on address for library \"" + libName + "\": " + addrString + "\n" - "The correct checksum is " + util::getChecksummedAddress(addrString) + "The correct checksum is " + getChecksummedAddress(addrString) ); - bytes binAddr = util::fromHex(addrString); - util::h160 address(binAddr, util::h160::AlignRight); - if (binAddr.size() > 20 || address == util::h160()) + bytes binAddr = fromHex(addrString); + h160 address(binAddr, h160::AlignRight); + if (binAddr.size() > 20 || address == h160()) solThrow( CommandLineValidationError, "Invalid address for library \"" + libName + "\": " + addrString @@ -461,6 +465,7 @@ void CommandLineParser::parseOutputSelection() solAssert(false); case InputMode::Compiler: case InputMode::CompilerWithASTImport: + case InputMode::CompilerWithEvmAssemblyJsonImport: return util::contains(compilerModeOutputs, _outputName); case InputMode::Assembler: return util::contains(assemblerModeOutputs, _outputName); @@ -582,15 +587,15 @@ General Information)").c_str(), ) ( g_strRevertStrings.c_str(), - po::value()->value_name(util::joinHumanReadable(g_revertStringsArgs, ",")), + po::value()->value_name(joinHumanReadable(g_revertStringsArgs, ",")), "Strip revert (and require) reason strings or add additional debugging information." ) ( g_strDebugInfo.c_str(), - po::value()->default_value(util::toString(DebugInfoSelection::Default())), + po::value()->default_value(toString(DebugInfoSelection::Default())), ("Debug info components to be included in the produced EVM assembly and Yul code. " "Value can be all, none or a comma-separated list containing one or more of the " - "following components: " + util::joinHumanReadable(DebugInfoSelection::componentMap() | ranges::views::keys) + ".").c_str() + "following components: " + joinHumanReadable(DebugInfoSelection::componentMap() | ranges::views::keys) + ".").c_str() ) ( g_strStopAfter.c_str(), @@ -636,6 +641,10 @@ General Information)").c_str(), "Supported Inputs is the output of the --" + g_strStandardJSON + " or the one produced by " "--" + g_strCombinedJson + " " + CombinedJsonRequests::componentName(&CombinedJsonRequests::ast)).c_str() ) + ( + g_strImportEvmAssemblerJson.c_str(), + "Import evm assembler json, assumes input holds the evm assembly in JSON format." + ) ( g_strLSP.c_str(), "Switch to language server mode (\"LSP\"). Allows the compiler to be used as an analysis backend " @@ -648,12 +657,12 @@ General Information)").c_str(), assemblyModeOptions.add_options() ( g_strMachine.c_str(), - po::value()->value_name(util::joinHumanReadable(g_machineArgs, ",")), + po::value()->value_name(joinHumanReadable(g_machineArgs, ",")), "Target machine in assembly or Yul mode." ) ( g_strYulDialect.c_str(), - po::value()->value_name(util::joinHumanReadable(g_yulDialectArgs, ",")), + po::value()->value_name(joinHumanReadable(g_yulDialectArgs, ",")), "Input dialect to use in assembly or yul mode." ) ; @@ -726,7 +735,7 @@ General Information)").c_str(), ) ( g_strCombinedJson.c_str(), - po::value()->value_name(util::joinHumanReadable(CombinedJsonRequests::componentMap() | ranges::views::keys, ",")), + po::value()->value_name(joinHumanReadable(CombinedJsonRequests::componentMap() | ranges::views::keys, ",")), "Output a single json document containing the specified information." ) ; @@ -736,7 +745,7 @@ General Information)").c_str(), metadataOptions.add_options() ( g_strMetadataHash.c_str(), - po::value()->value_name(util::joinHumanReadable(g_metadataHashArgs, ",")), + po::value()->value_name(joinHumanReadable(g_metadataHashArgs, ",")), "Choose hash method for the bytecode metadata or disable it." ) ( @@ -892,6 +901,8 @@ void CommandLineParser::processArgs() m_options.input.mode = InputMode::Linker; else if (m_args.count(g_strImportAst) > 0) m_options.input.mode = InputMode::CompilerWithASTImport; + else if (m_args.count(g_strImportEvmAssemblerJson) > 0) + m_options.input.mode = InputMode::CompilerWithEvmAssemblyJsonImport; else m_options.input.mode = InputMode::Compiler; @@ -1011,11 +1022,11 @@ void CommandLineParser::processArgs() if (m_args.count(g_strPrettyJson) > 0) { - m_options.formatting.json.format = util::JsonFormat::Pretty; + m_options.formatting.json.format = JsonFormat::Pretty; } if (!m_args[g_strJsonIndent].defaulted()) { - m_options.formatting.json.format = util::JsonFormat::Pretty; + m_options.formatting.json.format = JsonFormat::Pretty; m_options.formatting.json.indent = m_args[g_strJsonIndent].as(); } @@ -1260,7 +1271,11 @@ void CommandLineParser::processArgs() if (m_options.input.mode == InputMode::Compiler) m_options.input.errorRecovery = (m_args.count(g_strErrorRecovery) > 0); - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport + ); } void CommandLineParser::parseCombinedJsonOption() @@ -1289,7 +1304,7 @@ size_t CommandLineParser::countEnabledOptions(vector const& _optionNames string CommandLineParser::joinOptionNames(vector const& _optionNames, string _separator) { - return util::joinHumanReadable( + return joinHumanReadable( _optionNames | ranges::views::transform([](string const& _option){ return "--" + _option; }), _separator ); diff --git a/solc/CommandLineParser.h b/solc/CommandLineParser.h index 791e7f1c10cc..68eb51ed31d3 100644 --- a/solc/CommandLineParser.h +++ b/solc/CommandLineParser.h @@ -56,7 +56,8 @@ enum class InputMode StandardJson, Linker, Assembler, - LanguageServer + LanguageServer, + CompilerWithEvmAssemblyJsonImport }; struct CompilerOutputs From d71de24f9c73a843296dbbd46851fb997ad88e73 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Mon, 21 Feb 2022 20:32:02 -0500 Subject: [PATCH 02/10] [solc] Basic --import-asm-json import. --- libevmasm/Assembly.cpp | 441 +++++++++++++++++++----- libevmasm/Assembly.h | 32 +- libevmasm/AssemblyItem.cpp | 12 + libevmasm/AssemblyItem.h | 1 + libsolidity/interface/CompilerStack.cpp | 46 ++- libsolidity/interface/CompilerStack.h | 2 +- solc/CommandLineInterface.cpp | 60 +++- test/cmdlineTests/asm_json/output | 7 +- test/libevmasm/Assembler.cpp | 59 +++- 9 files changed, 544 insertions(+), 116 deletions(-) diff --git a/libevmasm/Assembly.cpp b/libevmasm/Assembly.cpp index 6d7b97cc4a7e..cb74364fcca7 100644 --- a/libevmasm/Assembly.cpp +++ b/libevmasm/Assembly.cpp @@ -34,7 +34,7 @@ #include #include -#include +#include #include #include @@ -222,13 +222,16 @@ string Assembly::assemblyString( return tmp.str(); } -Json::Value Assembly::createJsonValue(string _name, int _source, int _begin, int _end, string _value, string _jumpType) +Json::Value Assembly::createJsonValue( + string _name, int _sourceIndex, size_t _modifierDepth, int _begin, int _end, string _value, string _jumpType) { Json::Value value{Json::objectValue}; value["name"] = _name; - value["source"] = _source; + value["source"] = _sourceIndex; value["begin"] = _begin; value["end"] = _end; + if (_modifierDepth != 0) + value["modifierDepth"] = static_cast(_modifierDepth); if (!_value.empty()) value["value"] = _value; if (!_jumpType.empty()) @@ -243,12 +246,275 @@ string Assembly::toStringInHex(u256 _value) return hexStr.str(); } -Json::Value Assembly::assemblyJSON(map const& _sourceIndices) const +AssemblyItem Assembly::loadItemFromJSON(Json::Value const& _json) +{ + std::string name = _json["name"].isString() ? _json["name"].asString() : ""; + int begin = _json["begin"].isInt() ? _json["begin"].asInt() : -1; + int end = _json["end"].isInt() ? _json["end"].asInt() : -1; + int srcIndex = _json["source"].isInt() ? _json["source"].asInt() : -1; + size_t modifierDepth = _json["modifierDepth"].isInt() ? static_cast(_json["modifierDepth"].asInt()) : 0; + std::string value = _json["value"].isString() ? _json["value"].asString() : ""; + std::string jumpType = _json["jumpType"].isString() ? _json["jumpType"].asString() : ""; + solAssert(!name.empty(), ""); + + auto updateUsedTags = [&](u256 const& data) { + auto tag = static_cast(data); + if (this->m_usedTags <= tag) + this->m_usedTags = tag + 1; + }; + + auto updateImmutables = [&](string const& _immutableName) -> h256 { + h256 hash(util::keccak256(value)); + this->m_immutables[hash] = _immutableName; + return hash; + }; + + auto updateLibraries = [&](string const& _libraryName) -> h256 { + h256 hash(util::keccak256(_libraryName)); + this->m_libraries[hash] = _libraryName; + return hash; + }; + + SourceLocation location; + location.start = begin; + location.end = end; + if (srcIndex > -1 && srcIndex < (int) sources().size()) + location.sourceName = sources()[static_cast(srcIndex)]; + + AssemblyItem result(0); + + if (c_instructions.find(name) != c_instructions.end()) + { + AssemblyItem item{c_instructions.at(name), location}; + item.m_modifierDepth = modifierDepth; + if (!value.empty()) + item.setJumpType(value); + result = item; + } + else + { + u256 data; + if (name == "PUSH") + { + if (!value.empty()) + data = u256("0x" + value); + AssemblyItem item{AssemblyItemType::Push, data, location}; + if (!jumpType.empty()) + item.setJumpType(jumpType); + result = item; + } + else if (name == "PUSH [ErrorTag]") + result = {AssemblyItemType::PushTag, data, location}; + else if (name == "PUSH [tag]") + { + if (!value.empty()) + data = u256(value); + updateUsedTags(data); + result = {AssemblyItemType::PushTag, data, location}; + } + else if (name == "PUSH [$]") + { + if (!value.empty()) + data = u256("0x" + value); + result = {AssemblyItemType::PushSub, data, location}; + } + else if (name == "PUSH #[$]") + { + if (!value.empty()) + data = u256("0x" + value); + result = {AssemblyItemType::PushSubSize, data, location}; + } + else if (name == "PUSHSIZE") + result = {AssemblyItemType::PushProgramSize, data, location}; + else if (name == "PUSHLIB") + { + h256 hash = updateLibraries(value); + result = {AssemblyItemType::PushLibraryAddress, hash, location}; + } + else if (name == "PUSHDEPLOYADDRESS") + result = {AssemblyItemType::PushDeployTimeAddress, data, location}; + else if (name == "PUSHIMMUTABLE") + { + h256 hash = updateImmutables(value); + result = {AssemblyItemType::PushImmutable, hash, location}; + } + else if (name == "ASSIGNIMMUTABLE") + { + h256 hash = updateImmutables(value); + result = {AssemblyItemType::AssignImmutable, hash, location}; + } + else if (name == "tag") + { + if (!value.empty()) + data = u256(value); + result = {AssemblyItemType::Tag, data, location}; + } + else if (name == "PUSH data") + { + if (!value.empty()) + data = u256("0x" + value); + result = {AssemblyItemType::PushData, data, location}; + } + else if (name == "VERBATIM") + { + AssemblyItem item(fromHex(value), 0, 0); + item.setLocation(location); + result = item; + } + else + assertThrow(false, InvalidOpcode, ""); + } + result.m_modifierDepth = modifierDepth; + return result; +} + +vector Assembly::assemblyItemAsJSON(AssemblyItem const& _item, int _sourceIndex) const +{ + vector result; + + switch (_item.type()) + { + case Operation: + result.emplace_back(createJsonValue( + instructionInfo(_item.instruction()).name, + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + _item.getJumpTypeAsString())); + break; + case Push: + result.emplace_back(createJsonValue( + "PUSH", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toStringInHex(_item.data()), + _item.getJumpTypeAsString())); + break; + case PushTag: + if (_item.data() == 0) + result.emplace_back(createJsonValue( + "PUSH [ErrorTag]", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + "")); + else + result.emplace_back(createJsonValue( + "PUSH [tag]", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toString(_item.data()))); + break; + case PushSub: + result.emplace_back(createJsonValue( + "PUSH [$]", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toString(h256(_item.data())))); + break; + case PushSubSize: + result.emplace_back(createJsonValue( + "PUSH #[$]", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toString(h256(_item.data())))); + break; + case PushProgramSize: + result.emplace_back(createJsonValue( + "PUSHSIZE", _sourceIndex, _item.m_modifierDepth, _item.location().start, _item.location().end)); + break; + case PushLibraryAddress: + result.emplace_back(createJsonValue( + "PUSHLIB", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + m_libraries.at(h256(_item.data())))); + break; + case PushDeployTimeAddress: + result.emplace_back(createJsonValue( + "PUSHDEPLOYADDRESS", _sourceIndex, _item.m_modifierDepth, _item.location().start, _item.location().end)); + break; + case PushImmutable: + result.emplace_back(createJsonValue( + "PUSHIMMUTABLE", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + m_immutables.at(h256(_item.data())))); + break; + case AssignImmutable: + result.emplace_back(createJsonValue( + "ASSIGNIMMUTABLE", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + m_immutables.at(h256(_item.data())))); + break; + case Tag: + result.emplace_back(createJsonValue( + "tag", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toString(_item.data()))); + result.emplace_back(createJsonValue( + "JUMPDEST", _sourceIndex, _item.m_modifierDepth, _item.location().start, _item.location().end)); + break; + case PushData: + result.emplace_back(createJsonValue( + "PUSH data", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toStringInHex(_item.data()))); + break; + case VerbatimBytecode: + result.emplace_back(createJsonValue( + "VERBATIM", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + util::toHex(_item.verbatimData()))); + break; + default: + assertThrow(false, InvalidOpcode, ""); + } + return result; +} + +Json::Value Assembly::assemblyJSON(map const& _sourceIndices, bool _includeSourceList) const { Json::Value root; - root[".code"] = Json::arrayValue; + if (_includeSourceList) + { + root["sourceList"] = Json::arrayValue; + Json::Value& sourceList = root["sourceList"]; + std::vector sources(_sourceIndices.size()); + for (auto const& item: _sourceIndices) + sources[item.second] = item.first; + for (auto const& item: sources) + sourceList.append(item); + } - Json::Value& collection = root[".code"]; + root[".code"] = Json::arrayValue; + Json::Value& code = root[".code"]; for (AssemblyItem const& i: m_items) { int sourceIndex = -1; @@ -259,85 +525,8 @@ Json::Value Assembly::assemblyJSON(map const& _sourceIndices) sourceIndex = static_cast(iter->second); } - switch (i.type()) - { - case Operation: - collection.append( - createJsonValue( - instructionInfo(i.instruction()).name, - sourceIndex, - i.location().start, - i.location().end, - i.getJumpTypeAsString()) - ); - break; - case Push: - collection.append( - createJsonValue("PUSH", sourceIndex, i.location().start, i.location().end, toStringInHex(i.data()), i.getJumpTypeAsString())); - break; - case PushTag: - if (i.data() == 0) - collection.append( - createJsonValue("PUSH [ErrorTag]", sourceIndex, i.location().start, i.location().end, "")); - else - collection.append( - createJsonValue("PUSH [tag]", sourceIndex, i.location().start, i.location().end, toString(i.data()))); - break; - case PushSub: - collection.append( - createJsonValue("PUSH [$]", sourceIndex, i.location().start, i.location().end, toString(h256(i.data())))); - break; - case PushSubSize: - collection.append( - createJsonValue("PUSH #[$]", sourceIndex, i.location().start, i.location().end, toString(h256(i.data())))); - break; - case PushProgramSize: - collection.append( - createJsonValue("PUSHSIZE", sourceIndex, i.location().start, i.location().end)); - break; - case PushLibraryAddress: - collection.append( - createJsonValue("PUSHLIB", sourceIndex, i.location().start, i.location().end, m_libraries.at(h256(i.data()))) - ); - break; - case PushDeployTimeAddress: - collection.append( - createJsonValue("PUSHDEPLOYADDRESS", sourceIndex, i.location().start, i.location().end) - ); - break; - case PushImmutable: - collection.append(createJsonValue( - "PUSHIMMUTABLE", - sourceIndex, - i.location().start, - i.location().end, - m_immutables.at(h256(i.data())) - )); - break; - case AssignImmutable: - collection.append(createJsonValue( - "ASSIGNIMMUTABLE", - sourceIndex, - i.location().start, - i.location().end, - m_immutables.at(h256(i.data())) - )); - break; - case Tag: - collection.append( - createJsonValue("tag", sourceIndex, i.location().start, i.location().end, toString(i.data()))); - collection.append( - createJsonValue("JUMPDEST", sourceIndex, i.location().start, i.location().end)); - break; - case PushData: - collection.append(createJsonValue("PUSH data", sourceIndex, i.location().start, i.location().end, toStringInHex(i.data()))); - break; - case VerbatimBytecode: - collection.append(createJsonValue("VERBATIM", sourceIndex, i.location().start, i.location().end, util::toHex(i.verbatimData()))); - break; - default: - assertThrow(false, InvalidOpcode, ""); - } + for (Json::Value const& item: assemblyItemAsJSON(i, sourceIndex)) + code.append(item); } if (!m_data.empty() || !m_subs.empty()) @@ -352,16 +541,75 @@ Json::Value Assembly::assemblyJSON(map const& _sourceIndices) { std::stringstream hexStr; hexStr << hex << i; - data[hexStr.str()] = m_subs[i]->assemblyJSON(_sourceIndices); + data[hexStr.str()] = m_subs[i]->assemblyJSON(_sourceIndices, false); } } - if (m_auxiliaryData.size() > 0) + if (!m_auxiliaryData.empty()) root[".auxdata"] = util::toHex(m_auxiliaryData); return root; } +bool Assembly::addAssemblyItemsFromJSON(Json::Value const& _code) +{ + solAssert(_code.isArray(), ""); + for (auto const& it: _code) + this->m_items.emplace_back(loadItemFromJSON(it)); + + for (auto current = this->m_items.begin(); current != this->m_items.end(); ++current) + { + // During the assembly json export a `JUMPDEST` is always generated after a `tag`. + // So we just ignore exactly these `JUMPDEST`'s. + auto const next = std::next(current); + if ( + current->type() == AssemblyItemType::Tag && + next->type() == AssemblyItemType::Operation && + next->instruction() == Instruction::JUMPDEST + ) + this->m_items.erase(next); + } + + return true; +} + +bool Assembly::loadFromAssemblyJSON(Json::Value const& _json, bool _loadSources /* = true */) +{ + if (!_json[".code"].isArray()) + return false; + bool result{true}; + + if (_loadSources) + { + vector sourceList; + if (_json.isMember("sourceList")) + for (auto const& it: _json["sourceList"]) + sourceList.emplace_back(it.asString()); + setSources(sourceList); + } + + addAssemblyItemsFromJSON(_json[".code"]); + if (_json[".auxdata"].isString()) + this->m_auxiliaryData = fromHex(_json[".auxdata"].asString()); + Json::Value const& data = _json[".data"]; + for (Json::ValueConstIterator itr = data.begin(); itr != data.end(); itr++) + { + solAssert(itr.key().isString(), ""); + std::string key = itr.key().asString(); + Json::Value const& code = data[key]; + if (code.isString()) + this->m_data[h256(fromHex(key))] = fromHex(code.asString()); + else + { + shared_ptr subassembly = make_shared(); + subassembly->setSources(this->sources()); + result &= subassembly->loadFromAssemblyJSON(code, false); + this->m_subs.emplace_back(subassembly); + } + } + return result; +} + AssemblyItem Assembly::namedTag(string const& _name, size_t _params, size_t _returns, optional _sourceID) { assertThrow(!_name.empty(), AssemblyException, "Empty named tag."); @@ -397,6 +645,25 @@ AssemblyItem Assembly::newImmutableAssignment(string const& _identifier) return AssemblyItem{AssignImmutable, h}; } +Assembly& Assembly::optimise(bool _enable, EVMVersion _evmVersion, bool _isCreation, size_t _runs) +{ + OptimiserSettings settings; + settings.isCreation = _isCreation; + settings.runInliner = true; + settings.runJumpdestRemover = true; + settings.runPeephole = true; + if (_enable) + { + settings.runDeduplicate = true; + settings.runCSE = true; + settings.runConstantOptimiser = true; + } + settings.evmVersion = _evmVersion; + settings.expectedExecutionsPerDeployment = _runs; + optimise(settings); + return *this; +} + Assembly& Assembly::optimise(OptimiserSettings const& _settings) { optimiseInternal(_settings, {}); diff --git a/libevmasm/Assembly.h b/libevmasm/Assembly.h index 11bc16662979..1f46c3d68d62 100644 --- a/libevmasm/Assembly.h +++ b/libevmasm/Assembly.h @@ -39,6 +39,7 @@ #include #include #include +#include namespace solidity::evmasm { @@ -133,6 +134,13 @@ class Assembly /// is optimised according to the settings in @a _settings. Assembly& optimise(OptimiserSettings const& _settings); + /// Modify (if @a _enable is set) and return the current assembly such that creation and + /// execution gas usage is optimised. @a _isCreation should be true for the top-level assembly. + /// @a _runs specifes an estimate on how often each opcode in this assembly will be executed, + /// i.e. use a small value to optimise for size and a large value to optimise for runtime. + /// If @a _enable is not set, will perform some simple peephole optimizations. + Assembly& optimise(bool _enable, langutil::EVMVersion _evmVersion, bool _isCreation, size_t _runs); + /// Create a text representation of the assembly. std::string assemblyString( langutil::DebugInfoSelection const& _debugInfoSelection = langutil::DebugInfoSelection::Default(), @@ -147,9 +155,12 @@ class Assembly /// Create a JSON representation of the assembly. Json::Value assemblyJSON( - std::map const& _sourceIndices = std::map() + std::map const& _sourceIndices = std::map(), + bool _includeSourceList = true ) const; + bool loadFromAssemblyJSON(Json::Value const& _json, bool _loadSources = true); + /// Mark this assembly as invalid. Calling ``assemble`` on it will throw. void markAsInvalid() { m_invalid = true; } @@ -158,6 +169,16 @@ class Assembly bool isCreation() const { return m_creation; } + void setSources(std::vector> _sources) { + m_sources = std::move(_sources); + } + + void setSources(std::vector const& _sources) { + for (auto const& item: _sources) + m_sources.emplace_back(std::make_shared(item)); + } + std::vector> sources() const& { return m_sources; } + protected: /// Does the same operations as @a optimise, but should only be applied to a sub and /// returns the replaced tags. Also takes an argument containing the tags of this assembly @@ -166,10 +187,15 @@ class Assembly unsigned codeSize(unsigned subTagSize) const; + AssemblyItem loadItemFromJSON(Json::Value const& _json); + std::vector assemblyItemAsJSON(AssemblyItem const& _item, int _sourceIndex) const; + private: + bool addAssemblyItemsFromJSON(Json::Value const& _code); static Json::Value createJsonValue( std::string _name, - int _source, + int _sourceIndex, + size_t _modifierDepth, int _begin, int _end, std::string _value = std::string(), @@ -222,6 +248,8 @@ class Assembly std::string m_name; langutil::SourceLocation m_currentSourceLocation; + std::vector> m_sources; + public: size_t m_currentModifierDepth = 0; }; diff --git a/libevmasm/AssemblyItem.cpp b/libevmasm/AssemblyItem.cpp index 98405876535e..f8c73f2c1bc2 100644 --- a/libevmasm/AssemblyItem.cpp +++ b/libevmasm/AssemblyItem.cpp @@ -192,6 +192,18 @@ string AssemblyItem::getJumpTypeAsString() const } } +void AssemblyItem::setJumpType(std::string const& _jumpType) +{ + if (_jumpType == "[in]") + m_jumpType = JumpType::IntoFunction; + else if (_jumpType == "[out]") + m_jumpType = JumpType::OutOfFunction; + else if (_jumpType.empty()) + m_jumpType = JumpType::Ordinary; + else + assertThrow(false, AssemblyException, "Invalid jump type."); +} + string AssemblyItem::toAssemblyText(Assembly const& _assembly) const { string text; diff --git a/libevmasm/AssemblyItem.h b/libevmasm/AssemblyItem.h index 796a792e8a01..479a2d8f3c15 100644 --- a/libevmasm/AssemblyItem.h +++ b/libevmasm/AssemblyItem.h @@ -166,6 +166,7 @@ class AssemblyItem langutil::SourceLocation const& location() const { return m_location; } void setJumpType(JumpType _jumpType) { m_jumpType = _jumpType; } + void setJumpType(std::string const& _jumpType); JumpType getJumpType() const { return m_jumpType; } std::string getJumpTypeAsString() const; diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 48936e2d8c56..9cbe2baeb36c 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -676,7 +676,34 @@ bool CompilerStack::compile(State _stopAfter) if (!m_evmAssemblyJson.empty()) { - + solAssert(m_importedSources, ""); + solAssert(m_evmAssemblyJson.size() == 1, ""); + + string const evmSourceName = m_evmAssemblyJson.begin()->first; + Json::Value const evmJson = m_evmAssemblyJson.begin()->second; + + evmasm::Assembly::OptimiserSettings optimiserSettings; + optimiserSettings.evmVersion = m_evmVersion; + optimiserSettings.expectedExecutionsPerDeployment = m_optimiserSettings.expectedExecutionsPerDeployment; + optimiserSettings.runCSE = m_optimiserSettings.runCSE; + optimiserSettings.runConstantOptimiser = m_optimiserSettings.runConstantOptimiser; + optimiserSettings.runDeduplicate = m_optimiserSettings.runDeduplicate; + optimiserSettings.runInliner = m_optimiserSettings.runInliner; + optimiserSettings.runJumpdestRemover = m_optimiserSettings.runJumpdestRemover; + optimiserSettings.runPeephole = m_optimiserSettings.runPeephole; + + m_contracts[evmSourceName].evmAssembly = make_shared(evmSourceName); + m_contracts[evmSourceName].evmAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmSourceName]); + if (m_optimiserSettings.enabled) + m_contracts[evmSourceName].evmAssembly->optimise(optimiserSettings); + m_contracts[evmSourceName].object = m_contracts[evmSourceName].evmAssembly->assemble(); + + m_contracts[evmSourceName].evmRuntimeAssembly = make_shared(evmSourceName); + m_contracts[evmSourceName].evmRuntimeAssembly->setSources(m_contracts[evmSourceName].evmAssembly->sources()); + m_contracts[evmSourceName].evmRuntimeAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmSourceName][".data"]["0"], false); + if (m_optimiserSettings.enabled) + m_contracts[evmSourceName].evmRuntimeAssembly->optimise(optimiserSettings); + m_contracts[evmSourceName].runtimeObject = m_contracts[evmSourceName].evmRuntimeAssembly->assemble(); } else { @@ -961,14 +988,21 @@ vector CompilerStack::sourceNames() const return names; } -map CompilerStack::sourceIndices() const +map CompilerStack::sourceIndices(bool _includeInternalSources /* = true */) const { map indices; unsigned index = 0; - for (auto const& s: m_sources) - indices[s.first] = index++; - solAssert(!indices.count(CompilerContext::yulUtilityFileName()), ""); - indices[CompilerContext::yulUtilityFileName()] = index++; + if (m_evmAssemblyJson.empty()) + { + for (auto const& s: m_sources) + indices[s.first] = index++; + solAssert(!indices.count(CompilerContext::yulUtilityFileName()), ""); + if (_includeInternalSources) + indices[CompilerContext::yulUtilityFileName()] = index++; + } + else + for (auto const& s: m_sourceOrder) + indices[s->charStream->source()] = index++; return indices; } diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index 5618a9574bc7..e95eb76648d3 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -244,7 +244,7 @@ class CompilerStack: public langutil::CharStreamProvider /// @returns a mapping assigning each source name its index inside the vector returned /// by sourceNames(). - std::map sourceIndices() const; + std::map sourceIndices(bool _includeInternalSources = true) const; /// @returns the previously used character stream, useful for counting lines during error reporting. langutil::CharStream const& charStream(std::string const& _sourceName) const override; diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index 09d305282a59..c6f5a2e0cdb6 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -201,7 +201,11 @@ void CommandLineInterface::handleOpcode(string const& _contract) void CommandLineInterface::handleIR(string const& _contractName) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); if (!m_options.compiler.outputs.ir) return; @@ -217,7 +221,11 @@ void CommandLineInterface::handleIR(string const& _contractName) void CommandLineInterface::handleIROptimized(string const& _contractName) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); if (!m_options.compiler.outputs.irOptimized) return; @@ -233,7 +241,11 @@ void CommandLineInterface::handleIROptimized(string const& _contractName) void CommandLineInterface::handleEwasm(string const& _contractName) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); if (!m_options.compiler.outputs.ewasm) return; @@ -256,7 +268,11 @@ void CommandLineInterface::handleEwasm(string const& _contractName) void CommandLineInterface::handleBytecode(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); if (m_options.compiler.outputs.opcodes) handleOpcode(_contract); @@ -266,7 +282,11 @@ void CommandLineInterface::handleBytecode(string const& _contract) void CommandLineInterface::handleSignatureHashes(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); if (!m_options.compiler.outputs.signatureHashes) return; @@ -298,7 +318,11 @@ void CommandLineInterface::handleSignatureHashes(string const& _contract) void CommandLineInterface::handleMetadata(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); if (!m_options.compiler.outputs.metadata) return; @@ -312,7 +336,11 @@ void CommandLineInterface::handleMetadata(string const& _contract) void CommandLineInterface::handleABI(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); if (!m_options.compiler.outputs.abi) return; @@ -326,7 +354,11 @@ void CommandLineInterface::handleABI(string const& _contract) void CommandLineInterface::handleStorageLayout(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); if (!m_options.compiler.outputs.storageLayout) return; @@ -340,7 +372,11 @@ void CommandLineInterface::handleStorageLayout(string const& _contract) void CommandLineInterface::handleNatspec(bool _natspecDev, string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); bool enabled = false; std::string suffix; @@ -382,7 +418,11 @@ void CommandLineInterface::handleNatspec(bool _natspecDev, string const& _contra void CommandLineInterface::handleGasEstimation(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); Json::Value estimates = m_compiler->gasEstimates(_contract); sout() << "Gas estimation:" << endl; diff --git a/test/cmdlineTests/asm_json/output b/test/cmdlineTests/asm_json/output index 26ab87147a30..0128c541c474 100644 --- a/test/cmdlineTests/asm_json/output +++ b/test/cmdlineTests/asm_json/output @@ -1582,5 +1582,10 @@ EVM assembly: } ] } - } + }, + "sourceList": + [ + "asm_json/input.sol", + "#utility.yul" + ] } diff --git a/test/libevmasm/Assembler.cpp b/test/libevmasm/Assembler.cpp index 3a31678164b9..5a026d3456fc 100644 --- a/test/libevmasm/Assembler.cpp +++ b/test/libevmasm/Assembler.cpp @@ -56,7 +56,8 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) { map indices = { { "root.asm", 0 }, - { "sub.asm", 1 } + { "sub.asm", 1 }, + { "verbatim.asm", 2 } }; Assembly _assembly{false, {}}; auto root_asm = make_shared("root.asm"); @@ -65,11 +66,22 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) Assembly _subAsm{false, {}}; auto sub_asm = make_shared("sub.asm"); _subAsm.setSourceLocation({6, 8, sub_asm}); + + Assembly _verbatimAsm; + auto verbatim_asm = make_shared("verbatim.asm"); + _verbatimAsm.setSourceLocation({8, 18, verbatim_asm}); + // PushImmutable _subAsm.appendImmutable("someImmutable"); + _subAsm.append(AssemblyItem(PushTag, 0)); _subAsm.append(Instruction::INVALID); shared_ptr _subAsmPtr = make_shared(_subAsm); + _verbatimAsm.appendVerbatim({0xff,0xff}, 0, 0); + _verbatimAsm.appendVerbatim({0x74, 0x65, 0x73, 0x74}, 0, 1); + _verbatimAsm.append(Instruction::MSTORE); + shared_ptr _verbatimAsmPtr = make_shared(_verbatimAsm); + // Tag auto tag = _assembly.newTag(); _assembly.append(tag); @@ -90,6 +102,10 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) auto sub = _assembly.appendSubroutine(_subAsmPtr); // PushSub _assembly.pushSubroutineOffset(static_cast(sub.data())); + // PushSubSize + auto verbatim_sub = _assembly.appendSubroutine(_verbatimAsmPtr); + // PushSub + _assembly.pushSubroutineOffset(static_cast(verbatim_sub.data())); // PushDeployTimeAddress _assembly.append(PushDeployTimeAddress); // AssignImmutable. @@ -106,12 +122,12 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) BOOST_CHECK_EQUAL( _assembly.assemble().toHex(), - "5b6001600220606f73__$bf005014d9d0f534b8fcb268bd84c491a2$__" - "6000566067602260457300000000000000000000000000000000000000005050" + "5b6001600220607c73__$bf005014d9d0f534b8fcb268bd84c491a2$__" + "6000566074602460496007606d7300000000000000000000000000000000000000005050" "600260010152" "00fe" "7f0000000000000000000000000000000000000000000000000000000000000000" - "fe010203044266eeaa" + "6000feffff7465737452010203044266eeaa" ); BOOST_CHECK_EQUAL( _assembly.assemblyString(), @@ -124,6 +140,8 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) " data_a6885b3731702da62e8e4a8f584ac46a7f6822f4e2ba50fba902f67b1588d23b\n" " dataSize(sub_0)\n" " dataOffset(sub_0)\n" + " dataSize(sub_1)\n" + " dataOffset(sub_1)\n" " deployTimeAddress()\n" " assignImmutable(\"0xc3978657661c4d8e32e3d5f42597c009f0d3859e9f9d0d94325268f9799e2bfb\")\n" " 0x02\n" @@ -135,13 +153,20 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) "sub_0: assembly {\n" " /* \"sub.asm\":6:8 */\n" " immutable(\"0x26f2c0195e9d408feff3abd77d83f2971f3c9a18d1e8a9437c7835ae4211fc9f\")\n" + " tag_0\n" " invalid\n" "}\n" "\n" + "sub_1: assembly {\n" + " /* \"verbatim.asm\":8:18 */\n" + " verbatimbytecode_ffff\n" + " verbatimbytecode_74657374\n" + " mstore\n" + "}\n" + "\n" "auxdata: 0x4266eeaa\n" ); - BOOST_CHECK_EQUAL( - util::jsonCompactPrint(_assembly.assemblyJSON(indices)), + string json{ "{\".auxdata\":\"4266eeaa\",\".code\":[" "{\"begin\":1,\"end\":3,\"name\":\"tag\",\"source\":0,\"value\":\"1\"}," "{\"begin\":1,\"end\":3,\"name\":\"JUMPDEST\",\"source\":0}," @@ -155,6 +180,8 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) "{\"begin\":1,\"end\":3,\"name\":\"PUSH data\",\"source\":0,\"value\":\"A6885B3731702DA62E8E4A8F584AC46A7F6822F4E2BA50FBA902F67B1588D23B\"}," "{\"begin\":1,\"end\":3,\"name\":\"PUSH #[$]\",\"source\":0,\"value\":\"0000000000000000000000000000000000000000000000000000000000000000\"}," "{\"begin\":1,\"end\":3,\"name\":\"PUSH [$]\",\"source\":0,\"value\":\"0000000000000000000000000000000000000000000000000000000000000000\"}," + "{\"begin\":1,\"end\":3,\"name\":\"PUSH #[$]\",\"source\":0,\"value\":\"0000000000000000000000000000000000000000000000000000000000000001\"}," + "{\"begin\":1,\"end\":3,\"name\":\"PUSH [$]\",\"source\":0,\"value\":\"0000000000000000000000000000000000000000000000000000000000000001\"}," "{\"begin\":1,\"end\":3,\"name\":\"PUSHDEPLOYADDRESS\",\"source\":0}," "{\"begin\":1,\"end\":3,\"name\":\"ASSIGNIMMUTABLE\",\"source\":0,\"value\":\"someOtherImmutable\"}," "{\"begin\":1,\"end\":3,\"name\":\"PUSH\",\"source\":0,\"value\":\"2\"}," @@ -162,9 +189,23 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) "{\"begin\":1,\"end\":3,\"name\":\"STOP\",\"source\":0}" "],\".data\":{\"0\":{\".code\":[" "{\"begin\":6,\"end\":8,\"name\":\"PUSHIMMUTABLE\",\"source\":1,\"value\":\"someImmutable\"}," + "{\"begin\":6,\"end\":8,\"name\":\"PUSH [ErrorTag]\",\"source\":1}," "{\"begin\":6,\"end\":8,\"name\":\"INVALID\",\"source\":1}" - "]},\"A6885B3731702DA62E8E4A8F584AC46A7F6822F4E2BA50FBA902F67B1588D23B\":\"01020304\"}}" - ); + "]}," + "\"1\":{\".code\":[" + "{\"begin\":8,\"end\":18,\"name\":\"VERBATIM\",\"source\":2,\"value\":\"ffff\"}," + "{\"begin\":8,\"end\":18,\"name\":\"VERBATIM\",\"source\":2,\"value\":\"74657374\"}," + "{\"begin\":8,\"end\":18,\"name\":\"MSTORE\",\"source\":2}" + "]},\"A6885B3731702DA62E8E4A8F584AC46A7F6822F4E2BA50FBA902F67B1588D23B\":\"01020304\"},\"sourceList\":[\"root.asm\",\"sub.asm\",\"verbatim.asm\"]}" + }; + Json::Value jsonValue; + BOOST_CHECK(util::jsonParseStrict(json, jsonValue)); + BOOST_CHECK_EQUAL(util::jsonCompactPrint(_assembly.assemblyJSON(indices)), util::jsonCompactPrint(jsonValue)); + + Assembly _assemblyFromJson; + _assemblyFromJson.loadFromAssemblyJSON(_assembly.assemblyJSON(indices)); + BOOST_CHECK_EQUAL(util::jsonCompactPrint(_assemblyFromJson.assemblyJSON(indices)), util::jsonCompactPrint(jsonValue)); + BOOST_CHECK_EQUAL(_assembly.assemble().toHex(), _assemblyFromJson.assemble().toHex()); } BOOST_AUTO_TEST_CASE(immutables_and_its_source_maps) @@ -343,7 +384,7 @@ BOOST_AUTO_TEST_CASE(immutable) "{\"begin\":6,\"end\":8,\"name\":\"PUSHIMMUTABLE\",\"source\":1,\"value\":\"someImmutable\"}," "{\"begin\":6,\"end\":8,\"name\":\"PUSHIMMUTABLE\",\"source\":1,\"value\":\"someOtherImmutable\"}," "{\"begin\":6,\"end\":8,\"name\":\"PUSHIMMUTABLE\",\"source\":1,\"value\":\"someImmutable\"}" - "]}}}" + "]}},\"sourceList\":[\"root.asm\",\"sub.asm\"]}" ); } From e25a1a3879b8e17926a1b1cb890a94d9c7f7895d Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Thu, 24 Feb 2022 17:33:13 -0500 Subject: [PATCH 03/10] scripts/ImportExportTest.sh: Add generic script for import export. --- libevmasm/Assembly.cpp | 1 + scripts/ASTImportTest.sh | 155 --------------------- scripts/ImportExportTest.sh | 267 ++++++++++++++++++++++++++++++++++++ test/cmdlineTests.sh | 16 ++- 4 files changed, 282 insertions(+), 157 deletions(-) delete mode 100755 scripts/ASTImportTest.sh create mode 100755 scripts/ImportExportTest.sh diff --git a/libevmasm/Assembly.cpp b/libevmasm/Assembly.cpp index cb74364fcca7..3aebf57c00ec 100644 --- a/libevmasm/Assembly.cpp +++ b/libevmasm/Assembly.cpp @@ -347,6 +347,7 @@ AssemblyItem Assembly::loadItemFromJSON(Json::Value const& _json) { if (!value.empty()) data = u256(value); + updateUsedTags(data); result = {AssemblyItemType::Tag, data, location}; } else if (name == "PUSH data") diff --git a/scripts/ASTImportTest.sh b/scripts/ASTImportTest.sh deleted file mode 100755 index daf5a61632d7..000000000000 --- a/scripts/ASTImportTest.sh +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# Bash script to test the ast-import option of the compiler by -# first exporting a .sol file to JSON, then loading it into the compiler -# and exporting it again. The second JSON should be identical to the first -READLINK=readlink -if [[ "$OSTYPE" == "darwin"* ]]; then - READLINK=greadlink -fi -REPO_ROOT=$(${READLINK} -f "$(dirname "$0")"/..) -SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-${REPO_ROOT}/build} -SOLC=${SOLIDITY_BUILD_DIR}/solc/solc -SPLITSOURCES=${REPO_ROOT}/scripts/splitSources.py - -SYNTAXTESTS_DIR="${REPO_ROOT}/test/libsolidity/syntaxTests" -ASTJSONTESTS_DIR="${REPO_ROOT}/test/libsolidity/ASTJSON" -NSOURCES="$(find "$SYNTAXTESTS_DIR" -type f | wc -l)" - -# DEV_DIR="${REPO_ROOT}/../tmp/contracts/" -# NSOURCES="$(find $DEV_DIR -type f | wc -l)" #TODO use find command - -FAILED=0 -UNCOMPILABLE=0 -TESTED=0 - -if [[ "$(find . -maxdepth 0 -type d -empty)" == "" ]]; then - echo "Test directory not empty. Skipping!" - exit 1 -fi - -# function tests whether exporting and importing again leaves the JSON ast unchanged -# Results are recorded by adding to FAILED or UNCOMPILABLE. -# Also, in case of a mismatch a diff and the respective ASTs are printed -# Expected parameters: -# $1 name of the file to be exported and imported -# $2 any files needed to do so that might be in parent directories -function testImportExportEquivalence { - local nth_input_file="$1" - IFS=" " read -r -a all_input_files <<< "$2" - - if $SOLC "$nth_input_file" "${all_input_files[@]}" > /dev/null 2>&1 - then - ! [[ -e stderr.txt ]] || { echo "stderr.txt already exists. Refusing to overwrite."; exit 1; } - - # save exported json as expected result (silently) - $SOLC --combined-json ast --pretty-json "$nth_input_file" "${all_input_files[@]}" > expected.json 2> /dev/null - # import it, and export it again as obtained result (silently) - if ! $SOLC --import-ast --combined-json ast --pretty-json expected.json > obtained.json 2> stderr.txt - then - # For investigating, use exit 1 here so the script stops at the - # first failing test - # exit 1 - FAILED=$((FAILED + 1)) - echo -e "ERROR: AST reimport failed for input file $nth_input_file" - echo - echo "Compiler stderr:" - cat ./stderr.txt - echo - echo "Compiler stdout:" - cat ./obtained.json - return 1 - fi - DIFF="$(diff expected.json obtained.json)" - if [ "$DIFF" != "" ] - then - if [ "$DIFFVIEW" == "" ] - then - echo -e "ERROR: JSONS differ for $1: \n $DIFF \n" - echo "Expected:" - cat ./expected.json - echo "Obtained:" - cat ./obtained.json - else - # Use user supplied diff view binary - $DIFFVIEW expected.json obtained.json - fi - FAILED=$((FAILED + 1)) - return 2 - fi - TESTED=$((TESTED + 1)) - rm expected.json obtained.json - rm -f stderr.txt - else - # echo "contract $solfile could not be compiled " - UNCOMPILABLE=$((UNCOMPILABLE + 1)) - fi - # return 0 -} -echo "Looking at $NSOURCES .sol files..." - -WORKINGDIR=$PWD - -# for solfile in $(find $DEV_DIR -name *.sol) -# boost_filesystem_bug specifically tests a local fix for a boost::filesystem -# bug. Since the test involves a malformed path, there is no point in running -# AST tests on it. See https://github.com/boostorg/filesystem/issues/176 -# shellcheck disable=SC2044 -for solfile in $(find "$SYNTAXTESTS_DIR" "$ASTJSONTESTS_DIR" -name "*.sol" -and -not -name "boost_filesystem_bug.sol") -do - echo -n "." - # create a temporary sub-directory - FILETMP=$(mktemp -d) - cd "$FILETMP" - - set +e - OUTPUT=$("$SPLITSOURCES" "$solfile") - SPLITSOURCES_RC=$? - set -e - if [ ${SPLITSOURCES_RC} == 0 ] - then - # echo $OUTPUT - NSOURCES=$((NSOURCES - 1)) - for i in $OUTPUT; - do - testImportExportEquivalence "$i" "$OUTPUT" - NSOURCES=$((NSOURCES + 1)) - done - elif [ ${SPLITSOURCES_RC} == 1 ] - then - testImportExportEquivalence "$solfile" - elif [ ${SPLITSOURCES_RC} == 2 ] - then - # The script will exit with return code 2, if an UnicodeDecodeError occurred. - # This is the case if e.g. some tests are using invalid utf-8 sequences. We will ignore - # these errors, but print the actual output of the script. - echo -e "\n${OUTPUT}\n" - testImportExportEquivalence "$solfile" - else - # All other return codes will be treated as critical errors. The script will exit. - echo -e "\nGot unexpected return code ${SPLITSOURCES_RC} from ${SPLITSOURCES}. Aborting." - echo -e "\n${OUTPUT}\n" - - cd "$WORKINGDIR" - # Delete temporary files - rm -rf "$FILETMP" - - exit 1 - fi - - cd "$WORKINGDIR" - # Delete temporary files - rm -rf "$FILETMP" -done - -echo "" - -if [ "$FAILED" = 0 ] -then - echo "SUCCESS: $TESTED syntaxTests passed, $FAILED failed, $UNCOMPILABLE could not be compiled ($NSOURCES sources total)." -else - echo "FAILURE: Out of $NSOURCES sources, $FAILED failed, ($UNCOMPILABLE could not be compiled)." - exit 1 -fi diff --git a/scripts/ImportExportTest.sh b/scripts/ImportExportTest.sh new file mode 100755 index 000000000000..afa67d0300da --- /dev/null +++ b/scripts/ImportExportTest.sh @@ -0,0 +1,267 @@ +#!/usr/bin/env bash + +set -e + +# Bash script to test the ast-import option of the compiler by +# first exporting a .sol file to JSON, then loading it into the compiler +# and exporting it again. The second JSON should be identical to the first +READLINK=readlink +if [[ "$OSTYPE" == "darwin"* ]]; then + READLINK=greadlink +fi +IMPORT_TEST_TYPE=${1} +REPO_ROOT=$(${READLINK} -f "$(dirname "$0")"/..) +SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-${REPO_ROOT}/build} +SOLC=${SOLIDITY_BUILD_DIR}/solc/solc +SPLITSOURCES=${REPO_ROOT}/scripts/splitSources.py + +SYNTAXTESTS_DIR="${REPO_ROOT}/test/libsolidity/syntaxTests" +SEMANTICTESTS_DIR="${REPO_ROOT}/test/libsolidity/semanticTests" +ASTJSONTESTS_DIR="${REPO_ROOT}/test/libsolidity/ASTJSON" + +# DEV_DIR="${REPO_ROOT}/../tmp/contracts/" +# NSOURCES="$(find $DEV_DIR -type f | wc -l)" #TODO use find command + +FAILED=0 +UNCOMPILABLE=0 +TESTED=0 + +if [[ "$(find . -maxdepth 0 -type d -empty)" == "" ]]; then + echo "Test directory not empty. Skipping!" + exit 1 +fi + +# function tests whether exporting and importing again leaves the JSON ast unchanged +# Results are recorded by adding to FAILED or UNCOMPILABLE. +# Also, in case of a mismatch a diff and the respective ASTs are printed +# Expected parameters: +# $1 name of the file to be exported and imported +# $2 any files needed to do so that might be in parent directories +function testImportExportEquivalence { + local nth_input_file="$1" + IFS=" " read -r -a all_input_files <<< "$2" + + if $SOLC --bin "$nth_input_file" "${all_input_files[@]}" > /dev/null 2>&1 + then + ! [[ -e stderr.txt ]] || { echo "stderr.txt already exists. Refusing to overwrite."; exit 1; } + + if [ "${IMPORT_TEST_TYPE}" == "ast" ] + then + # save exported json as expected result (silently) + $SOLC --combined-json ast --pretty-json "$nth_input_file" "${all_input_files[@]}" > expected.json 2> /dev/null + # import it, and export it again as obtained result (silently) + if ! $SOLC --import-ast --combined-json ast --pretty-json expected.json > obtained.json 2> stderr.txt + then + # For investigating, use exit 1 here so the script stops at the + # first failing test + # exit 1 + FAILED=$((FAILED + 1)) + echo -e "ERROR: AST reimport failed for input file $nth_input_file" + echo + echo "Compiler stderr:" + cat ./stderr.txt + echo + echo "Compiler stdout:" + cat ./obtained.json + return 1 + fi + set +e + DIFF="$(diff expected.json obtained.json)" + set +e + if [ "$DIFF" != "" ] + then + if [ "$DIFFVIEW" == "" ] + then + echo -e "ERROR: JSONS differ for $1: \n $DIFF \n" + echo "Expected:" + cat ./expected.json + echo "Obtained:" + cat ./obtained.json + else + # Use user supplied diff view binary + $DIFFVIEW expected.json obtained.json + fi + FAILED=$((FAILED + 1)) + return 2 + fi + TESTED=$((TESTED + 1)) + rm expected.json obtained.json + rm -f stderr.txt + elif [ "${IMPORT_TEST_TYPE}" == "evm-assembly" ] + then + local types=( "asm" "bin" "bin-runtime" "opcodes" "srcmap" "srcmap-runtime" ) + local _TESTED=1 + if ! $SOLC --combined-json bin,bin-runtime,opcodes,asm,srcmap,srcmap-runtime --pretty-json "$nth_input_file" "${all_input_files[@]}" > expected.json 2> expected.error + then + printf "\n" + echo "$nth_input_file" + cat expected.error + UNCOMPILABLE=$((UNCOMPILABLE + 1)) + return 0 + else + for contract in $(jq '.contracts | keys | .[]' expected.json 2> /dev/null) + do + for type in "${types[@]}" + do + jq --raw-output ".contracts.${contract}.\"${type}\"" expected.json > "expected.${type}" + done + + assembly=$(cat expected.asm) + if [ "$assembly" != "" ] && [ "$assembly" != "null" ] + then + if ! $SOLC --combined-json bin,bin-runtime,opcodes,asm,srcmap,srcmap-runtime --pretty-json --import-asm-json expected.asm > obtained.json 2> obtained.error + then + printf "\n" + echo "$nth_input_file" + cat obtained.error + FAILED=$((FAILED + 1)) + return 0 + else + for type in "${types[@]}" + do + for obtained_contract in $(jq '.contracts | keys | .[]' obtained.json 2> /dev/null) + do + jq --raw-output ".contracts.${obtained_contract}.\"${type}\"" obtained.json > "obtained.${type}" + set +e + DIFF="$(diff "expected.${type}" "obtained.${type}")" + set -e + if [ "$DIFF" != "" ] + then + if [ "$DIFFVIEW" == "" ] + then + echo -e "ERROR: JSONS differ for $1: \n $DIFF \n" + echo "Expected:" + cat "expected.${type}" + echo "Obtained:" + cat "obtained.${type}" + else + # Use user supplied diff view binary + $DIFFVIEW expected.json obtained.json + fi + _TESTED= + FAILED=$((FAILED + 1)) + return 0 + fi + done + done + + rm obtained.json + rm -f obtained.error + for type in "${types[@]}" + do + rm "obtained.${type}" + done + fi + + for type in "${types[@]}" + do + rm "expected.${type}" + done + fi + done + rm expected.json + fi + if [ -n "${_TESTED}" ] + then + TESTED=$((TESTED + 1)) + fi + else + echo "unknown import test type. aborting." + exit 1 + fi + else + UNCOMPILABLE=$((UNCOMPILABLE + 1)) + fi +} + +WORKINGDIR=$PWD +NSOURCES=0 + +# check whether SOLC works. +if ! $SOLC --version > /dev/null 2>&1 +then + echo "$SOLC not found. aborting." + exit 1 +fi + +# check whether jq can be found. +if ! jq --version > /dev/null 2>&1 +then + echo "jq needed. please install. aborting." + exit 1 +fi + +# for solfile in $(find $DEV_DIR -name *.sol) +# boost_filesystem_bug specifically tests a local fix for a boost::filesystem +# bug. Since the test involves a malformed path, there is no point in running +# AST tests on it. See https://github.com/boostorg/filesystem/issues/176 +if [ "${IMPORT_TEST_TYPE}" == "ast" ] +then + IMPORT_TEST_FILES=$(find "${SYNTAXTESTS_DIR}" "${ASTJSONTESTS_DIR}" -name "*.sol" -and -not -name "boost_filesystem_bug.sol") +elif [ "${IMPORT_TEST_TYPE}" == "evm-assembly" ] +then + IMPORT_TEST_FILES=$(find "${SYNTAXTESTS_DIR}" "${SEMANTICTESTS_DIR}" -name "*.sol" -and -not -name "boost_filesystem_bug.sol") +else + echo "unknown import test type. aborting. please specify $0 [ast|evm-assembly]." + exit 1 +fi + +NSOURCES="$(echo "$IMPORT_TEST_FILES" | wc -l)" +echo "Looking at $NSOURCES .sol files..." + +for solfile in ${IMPORT_TEST_FILES} +do + echo -n "." + # create a temporary sub-directory + FILETMP=$(mktemp -d) + cd "$FILETMP" + + set +e + OUTPUT=$("$SPLITSOURCES" "$solfile") + SPLITSOURCES_RC=$? + set -e + if [ ${SPLITSOURCES_RC} == 0 ] + then + # echo $OUTPUT + NSOURCES=$((NSOURCES - 1)) + for i in $OUTPUT; + do + testImportExportEquivalence "$i" "$OUTPUT" + NSOURCES=$((NSOURCES + 1)) + done + elif [ ${SPLITSOURCES_RC} == 1 ] + then + testImportExportEquivalence "$solfile" + elif [ ${SPLITSOURCES_RC} == 2 ] + then + # The script will exit with return code 2, if an UnicodeDecodeError occurred. + # This is the case if e.g. some tests are using invalid utf-8 sequences. We will ignore + # these errors, but print the actual output of the script. + echo -e "\n${OUTPUT}\n" + testImportExportEquivalence "$solfile" + else + # All other return codes will be treated as critical errors. The script will exit. + echo -e "\nGot unexpected return code ${SPLITSOURCES_RC} from ${SPLITSOURCES}. Aborting." + echo -e "\n${OUTPUT}\n" + + cd "$WORKINGDIR" + # Delete temporary files + rm -rf "$FILETMP" + + exit 1 + fi + + cd "$WORKINGDIR" + # Delete temporary files + rm -rf "$FILETMP" +done + +echo "" + +if [ "$FAILED" = 0 ] +then + echo "SUCCESS: $TESTED tests passed, $FAILED failed, $UNCOMPILABLE could not be compiled ($NSOURCES sources total)." +else + echo "FAILURE: Out of $NSOURCES sources, $FAILED failed, ($UNCOMPILABLE could not be compiled)." + exit 1 +fi diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index d33dd208c13b..cfbb8ffefa08 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -645,11 +645,23 @@ SOLTMPDIR=$(mktemp -d) ) rm -r "$SOLTMPDIR" -printTask "Testing AST import..." +printTask "Testing AST import/export..." SOLTMPDIR=$(mktemp -d) ( cd "$SOLTMPDIR" - if ! "$REPO_ROOT/scripts/ASTImportTest.sh" + if ! "$REPO_ROOT/scripts/ImportExportTest.sh" ast + then + rm -r "$SOLTMPDIR" + fail + fi +) +rm -r "$SOLTMPDIR" + +printTask "Testing EVM Assembly JSON import/export..." +SOLTMPDIR=$(mktemp -d) +( + cd "$SOLTMPDIR" + if ! "$REPO_ROOT/scripts/ImportExportTest.sh" evm-assembly then rm -r "$SOLTMPDIR" fail From 00f034362b787f7be40a41cb9af3b6e761deb5eb Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Wed, 9 Mar 2022 16:50:19 -0500 Subject: [PATCH 04/10] scripts/ImportExportTest.sh: Add direct import/export tests. --- scripts/ImportExportTest.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/scripts/ImportExportTest.sh b/scripts/ImportExportTest.sh index afa67d0300da..e2b5b39619bc 100755 --- a/scripts/ImportExportTest.sh +++ b/scripts/ImportExportTest.sh @@ -145,6 +145,40 @@ function testImportExportEquivalence { done done + if ! $SOLC --combined-json asm --pretty-json --import-asm-json expected.asm > obtained_direct_import_export.json 2> obtained_direct_import_export.error + then + printf "\n" + echo "$nth_input_file" + cat obtained_direct_import_export.error + FAILED=$((FAILED + 1)) + return 0 + else + for obtained_contract in $(jq '.contracts | keys | .[]' obtained_direct_import_export.json 2> /dev/null) + do + jq --raw-output ".contracts.${obtained_contract}.\"asm\"" obtained_direct_import_export.json > obtained_direct_import_export.asm + set +e + DIFF="$(diff expected.asm obtained_direct_import_export.asm)" + set -e + if [ "$DIFF" != "" ] + then + if [ "$DIFFVIEW" == "" ] + then + echo -e "ERROR: JSONS differ for $1: \n $DIFF \n" + echo "Expected:" + cat expected.asm + echo "Obtained:" + cat obtained_direct_import_export.asm + else + # Use user supplied diff view binary + $DIFFVIEW expected.asm obtained_direct_import_export.asm + fi + _TESTED= + FAILED=$((FAILED + 1)) + return 0 + fi + done + fi + rm obtained.json rm -f obtained.error for type in "${types[@]}" From 0c0853e635166dc13ada17e2b1a0d19460e0fd8e Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Wed, 9 Mar 2022 16:51:08 -0500 Subject: [PATCH 05/10] CLI: Add checks for invalid output-modes for --import-asm-json. --- solc/CommandLineInterface.cpp | 12 ++++++++++-- solc/CommandLineParser.cpp | 29 +++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index c6f5a2e0cdb6..6d3bdb7d7e42 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -161,7 +161,11 @@ static bool coloredOutput(CommandLineOptions const& _options) void CommandLineInterface::handleBinary(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); if (m_options.compiler.outputs.binary) { @@ -187,7 +191,11 @@ void CommandLineInterface::handleBinary(string const& _contract) void CommandLineInterface::handleOpcode(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); if (!m_options.output.dir.empty()) createFile(m_compiler->filesystemFriendlyName(_contract) + ".opcode", evmasm::disassemble(m_compiler->object(_contract).bytecode)); diff --git a/solc/CommandLineParser.cpp b/solc/CommandLineParser.cpp index a306820b167e..2a139c8cbd7c 100644 --- a/solc/CommandLineParser.cpp +++ b/solc/CommandLineParser.cpp @@ -455,6 +455,13 @@ void CommandLineParser::parseOutputSelection() CompilerOutputs::componentName(&CompilerOutputs::ewasm), CompilerOutputs::componentName(&CompilerOutputs::ewasmIR), }; + static set const evmAssemblyJsonImportModeOutputs = { + CompilerOutputs::componentName(&CompilerOutputs::asm_), + CompilerOutputs::componentName(&CompilerOutputs::binary), + CompilerOutputs::componentName(&CompilerOutputs::binaryRuntime), + CompilerOutputs::componentName(&CompilerOutputs::opcodes), + CompilerOutputs::componentName(&CompilerOutputs::asmJson), + }; switch (_mode) { @@ -465,8 +472,9 @@ void CommandLineParser::parseOutputSelection() solAssert(false); case InputMode::Compiler: case InputMode::CompilerWithASTImport: - case InputMode::CompilerWithEvmAssemblyJsonImport: return util::contains(compilerModeOutputs, _outputName); + case InputMode::CompilerWithEvmAssemblyJsonImport: + return util::contains(evmAssemblyJsonImportModeOutputs, _outputName); case InputMode::Assembler: return util::contains(assemblerModeOutputs, _outputName); case InputMode::StandardJson: @@ -936,7 +944,6 @@ void CommandLineParser::processArgs() return; checkMutuallyExclusive({g_strColor, g_strNoColor}); - array const conflictingWithStopAfter{ CompilerOutputs::componentName(&CompilerOutputs::binary), CompilerOutputs::componentName(&CompilerOutputs::ir), @@ -952,9 +959,27 @@ void CommandLineParser::processArgs() for (auto& option: conflictingWithStopAfter) checkMutuallyExclusive({g_strStopAfter, option}); + array const conflictingWithAsmJsonImport{ + CompilerOutputs::componentName(&CompilerOutputs::ir), + CompilerOutputs::componentName(&CompilerOutputs::irOptimized), + CompilerOutputs::componentName(&CompilerOutputs::ewasm), + CompilerOutputs::componentName(&CompilerOutputs::ewasmIR), + g_strGas, + CompilerOutputs::componentName(&CompilerOutputs::metadata), + CompilerOutputs::componentName(&CompilerOutputs::natspecDev), + CompilerOutputs::componentName(&CompilerOutputs::natspecUser), + CompilerOutputs::componentName(&CompilerOutputs::signatureHashes), + CompilerOutputs::componentName(&CompilerOutputs::storageLayout), + CompilerOutputs::componentName(&CompilerOutputs::astCompactJson), + }; + + for (auto& option: conflictingWithAsmJsonImport) + checkMutuallyExclusive({g_strImportEvmAssemblerJson, option}); + if ( m_options.input.mode != InputMode::Compiler && m_options.input.mode != InputMode::CompilerWithASTImport && + m_options.input.mode != InputMode::CompilerWithEvmAssemblyJsonImport && m_options.input.mode != InputMode::Assembler ) { From c097842455d21357b74879212cc5552313b339a5 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Mon, 14 Mar 2022 09:24:26 -0500 Subject: [PATCH 06/10] rebase --- libevmasm/Assembly.cpp | 21 +------------ libevmasm/Assembly.h | 7 ----- libsolidity/interface/CompilerStack.cpp | 4 +-- solc/CommandLineParser.cpp | 40 ++++++++++++------------- test/libevmasm/Assembler.cpp | 4 +-- 5 files changed, 25 insertions(+), 51 deletions(-) diff --git a/libevmasm/Assembly.cpp b/libevmasm/Assembly.cpp index 3aebf57c00ec..881107cf251e 100644 --- a/libevmasm/Assembly.cpp +++ b/libevmasm/Assembly.cpp @@ -602,7 +602,7 @@ bool Assembly::loadFromAssemblyJSON(Json::Value const& _json, bool _loadSources this->m_data[h256(fromHex(key))] = fromHex(code.asString()); else { - shared_ptr subassembly = make_shared(); + shared_ptr subassembly = make_shared(false, ""); subassembly->setSources(this->sources()); result &= subassembly->loadFromAssemblyJSON(code, false); this->m_subs.emplace_back(subassembly); @@ -646,25 +646,6 @@ AssemblyItem Assembly::newImmutableAssignment(string const& _identifier) return AssemblyItem{AssignImmutable, h}; } -Assembly& Assembly::optimise(bool _enable, EVMVersion _evmVersion, bool _isCreation, size_t _runs) -{ - OptimiserSettings settings; - settings.isCreation = _isCreation; - settings.runInliner = true; - settings.runJumpdestRemover = true; - settings.runPeephole = true; - if (_enable) - { - settings.runDeduplicate = true; - settings.runCSE = true; - settings.runConstantOptimiser = true; - } - settings.evmVersion = _evmVersion; - settings.expectedExecutionsPerDeployment = _runs; - optimise(settings); - return *this; -} - Assembly& Assembly::optimise(OptimiserSettings const& _settings) { optimiseInternal(_settings, {}); diff --git a/libevmasm/Assembly.h b/libevmasm/Assembly.h index 1f46c3d68d62..5f0f98abc4ac 100644 --- a/libevmasm/Assembly.h +++ b/libevmasm/Assembly.h @@ -134,13 +134,6 @@ class Assembly /// is optimised according to the settings in @a _settings. Assembly& optimise(OptimiserSettings const& _settings); - /// Modify (if @a _enable is set) and return the current assembly such that creation and - /// execution gas usage is optimised. @a _isCreation should be true for the top-level assembly. - /// @a _runs specifes an estimate on how often each opcode in this assembly will be executed, - /// i.e. use a small value to optimise for size and a large value to optimise for runtime. - /// If @a _enable is not set, will perform some simple peephole optimizations. - Assembly& optimise(bool _enable, langutil::EVMVersion _evmVersion, bool _isCreation, size_t _runs); - /// Create a text representation of the assembly. std::string assemblyString( langutil::DebugInfoSelection const& _debugInfoSelection = langutil::DebugInfoSelection::Default(), diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 9cbe2baeb36c..a5190cd822db 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -692,13 +692,13 @@ bool CompilerStack::compile(State _stopAfter) optimiserSettings.runJumpdestRemover = m_optimiserSettings.runJumpdestRemover; optimiserSettings.runPeephole = m_optimiserSettings.runPeephole; - m_contracts[evmSourceName].evmAssembly = make_shared(evmSourceName); + m_contracts[evmSourceName].evmAssembly = make_shared(true, evmSourceName); m_contracts[evmSourceName].evmAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmSourceName]); if (m_optimiserSettings.enabled) m_contracts[evmSourceName].evmAssembly->optimise(optimiserSettings); m_contracts[evmSourceName].object = m_contracts[evmSourceName].evmAssembly->assemble(); - m_contracts[evmSourceName].evmRuntimeAssembly = make_shared(evmSourceName); + m_contracts[evmSourceName].evmRuntimeAssembly = make_shared(false, evmSourceName); m_contracts[evmSourceName].evmRuntimeAssembly->setSources(m_contracts[evmSourceName].evmAssembly->sources()); m_contracts[evmSourceName].evmRuntimeAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmSourceName][".data"]["0"], false); if (m_optimiserSettings.enabled) diff --git a/solc/CommandLineParser.cpp b/solc/CommandLineParser.cpp index 2a139c8cbd7c..bbb0c2f99e23 100644 --- a/solc/CommandLineParser.cpp +++ b/solc/CommandLineParser.cpp @@ -168,7 +168,7 @@ ostream& operator<<(ostream& _out, CompilerOutputs const& _selection) if (_selection.*component) serializedSelection.push_back(CompilerOutputs::componentName(component)); - return _out << joinHumanReadable(serializedSelection, ","); + return _out << util::joinHumanReadable(serializedSelection, ","); } string const& CompilerOutputs::componentName(bool CompilerOutputs::* _component) @@ -199,7 +199,7 @@ ostream& operator<<(ostream& _out, CombinedJsonRequests const& _requests) if (_requests.*component) serializedRequests.push_back(CombinedJsonRequests::componentName(component)); - return _out << joinHumanReadable(serializedRequests, ","); + return _out << util::joinHumanReadable(serializedRequests, ","); } string const& CombinedJsonRequests::componentName(bool CombinedJsonRequests::* _component) @@ -347,17 +347,17 @@ void CommandLineParser::parseLibraryOption(string const& _input) try { if (fs::is_regular_file(_input)) - data = readFileAsString(_input); + data = util::readFileAsString(_input); } catch (fs::filesystem_error const&) { // Thrown e.g. if path is too long. } - catch (FileNotFound const&) + catch (util::FileNotFound const&) { // Should not happen if `fs::is_regular_file` is correct. } - catch (NotAFile const&) + catch (util::NotAFile const&) { // Should not happen if `fs::is_regular_file` is correct. } @@ -422,15 +422,15 @@ void CommandLineParser::parseLibraryOption(string const& _input) "Invalid length for address for library \"" + libName + "\": " + to_string(addrString.length()) + " instead of 40 characters." ); - if (!passesAddressChecksum(addrString, false)) + if (!util::passesAddressChecksum(addrString, false)) solThrow( CommandLineValidationError, "Invalid checksum on address for library \"" + libName + "\": " + addrString + "\n" - "The correct checksum is " + getChecksummedAddress(addrString) + "The correct checksum is " + util::getChecksummedAddress(addrString) ); - bytes binAddr = fromHex(addrString); - h160 address(binAddr, h160::AlignRight); - if (binAddr.size() > 20 || address == h160()) + bytes binAddr = util::fromHex(addrString); + util::h160 address(binAddr, util::h160::AlignRight); + if (binAddr.size() > 20 || address == util::h160()) solThrow( CommandLineValidationError, "Invalid address for library \"" + libName + "\": " + addrString @@ -595,15 +595,15 @@ General Information)").c_str(), ) ( g_strRevertStrings.c_str(), - po::value()->value_name(joinHumanReadable(g_revertStringsArgs, ",")), + po::value()->value_name(util::joinHumanReadable(g_revertStringsArgs, ",")), "Strip revert (and require) reason strings or add additional debugging information." ) ( g_strDebugInfo.c_str(), - po::value()->default_value(toString(DebugInfoSelection::Default())), + po::value()->default_value(util::toString(DebugInfoSelection::Default())), ("Debug info components to be included in the produced EVM assembly and Yul code. " "Value can be all, none or a comma-separated list containing one or more of the " - "following components: " + joinHumanReadable(DebugInfoSelection::componentMap() | ranges::views::keys) + ".").c_str() + "following components: " + util::joinHumanReadable(DebugInfoSelection::componentMap() | ranges::views::keys) + ".").c_str() ) ( g_strStopAfter.c_str(), @@ -665,12 +665,12 @@ General Information)").c_str(), assemblyModeOptions.add_options() ( g_strMachine.c_str(), - po::value()->value_name(joinHumanReadable(g_machineArgs, ",")), + po::value()->value_name(util::joinHumanReadable(g_machineArgs, ",")), "Target machine in assembly or Yul mode." ) ( g_strYulDialect.c_str(), - po::value()->value_name(joinHumanReadable(g_yulDialectArgs, ",")), + po::value()->value_name(util::joinHumanReadable(g_yulDialectArgs, ",")), "Input dialect to use in assembly or yul mode." ) ; @@ -743,7 +743,7 @@ General Information)").c_str(), ) ( g_strCombinedJson.c_str(), - po::value()->value_name(joinHumanReadable(CombinedJsonRequests::componentMap() | ranges::views::keys, ",")), + po::value()->value_name(util::joinHumanReadable(CombinedJsonRequests::componentMap() | ranges::views::keys, ",")), "Output a single json document containing the specified information." ) ; @@ -753,7 +753,7 @@ General Information)").c_str(), metadataOptions.add_options() ( g_strMetadataHash.c_str(), - po::value()->value_name(joinHumanReadable(g_metadataHashArgs, ",")), + po::value()->value_name(util::joinHumanReadable(g_metadataHashArgs, ",")), "Choose hash method for the bytecode metadata or disable it." ) ( @@ -1047,11 +1047,11 @@ void CommandLineParser::processArgs() if (m_args.count(g_strPrettyJson) > 0) { - m_options.formatting.json.format = JsonFormat::Pretty; + m_options.formatting.json.format = util::JsonFormat::Pretty; } if (!m_args[g_strJsonIndent].defaulted()) { - m_options.formatting.json.format = JsonFormat::Pretty; + m_options.formatting.json.format = util::JsonFormat::Pretty; m_options.formatting.json.indent = m_args[g_strJsonIndent].as(); } @@ -1329,7 +1329,7 @@ size_t CommandLineParser::countEnabledOptions(vector const& _optionNames string CommandLineParser::joinOptionNames(vector const& _optionNames, string _separator) { - return joinHumanReadable( + return util::joinHumanReadable( _optionNames | ranges::views::transform([](string const& _option){ return "--" + _option; }), _separator ); diff --git a/test/libevmasm/Assembler.cpp b/test/libevmasm/Assembler.cpp index 5a026d3456fc..1dd52485140a 100644 --- a/test/libevmasm/Assembler.cpp +++ b/test/libevmasm/Assembler.cpp @@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) auto sub_asm = make_shared("sub.asm"); _subAsm.setSourceLocation({6, 8, sub_asm}); - Assembly _verbatimAsm; + Assembly _verbatimAsm(true, ""); auto verbatim_asm = make_shared("verbatim.asm"); _verbatimAsm.setSourceLocation({8, 18, verbatim_asm}); @@ -202,7 +202,7 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) BOOST_CHECK(util::jsonParseStrict(json, jsonValue)); BOOST_CHECK_EQUAL(util::jsonCompactPrint(_assembly.assemblyJSON(indices)), util::jsonCompactPrint(jsonValue)); - Assembly _assemblyFromJson; + Assembly _assemblyFromJson(true, ""); _assemblyFromJson.loadFromAssemblyJSON(_assembly.assemblyJSON(indices)); BOOST_CHECK_EQUAL(util::jsonCompactPrint(_assemblyFromJson.assemblyJSON(indices)), util::jsonCompactPrint(jsonValue)); BOOST_CHECK_EQUAL(_assembly.assemble().toHex(), _assemblyFromJson.assemble().toHex()); From 39d56ee9971891feca7f8336e148cd22136f7cb2 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Mon, 14 Mar 2022 10:22:25 -0500 Subject: [PATCH 07/10] scripts/ImportExportTest.sh: fix direct export via --asm-json. --- scripts/ImportExportTest.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ImportExportTest.sh b/scripts/ImportExportTest.sh index e2b5b39619bc..d0f0c5a3325c 100755 --- a/scripts/ImportExportTest.sh +++ b/scripts/ImportExportTest.sh @@ -145,7 +145,8 @@ function testImportExportEquivalence { done done - if ! $SOLC --combined-json asm --pretty-json --import-asm-json expected.asm > obtained_direct_import_export.json 2> obtained_direct_import_export.error + # direct export via --asm-json, if imported with --import-asm-json. + if ! $SOLC --asm-json --import-asm-json expected.asm | tail -n+4 > obtained_direct_import_export.json 2> obtained_direct_import_export.error then printf "\n" echo "$nth_input_file" From 43faf23ed88d67b6c26450c98d75a0069361dfc1 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Mon, 14 Mar 2022 15:51:34 -0500 Subject: [PATCH 08/10] Minor optimization. --- libevmasm/Assembly.cpp | 2 +- libevmasm/Assembly.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libevmasm/Assembly.cpp b/libevmasm/Assembly.cpp index 881107cf251e..6fcc17d12429 100644 --- a/libevmasm/Assembly.cpp +++ b/libevmasm/Assembly.cpp @@ -223,7 +223,7 @@ string Assembly::assemblyString( } Json::Value Assembly::createJsonValue( - string _name, int _sourceIndex, size_t _modifierDepth, int _begin, int _end, string _value, string _jumpType) + string _name, int _sourceIndex, size_t _modifierDepth, int _begin, int _end, string const& _value, string const& _jumpType) { Json::Value value{Json::objectValue}; value["name"] = _name; diff --git a/libevmasm/Assembly.h b/libevmasm/Assembly.h index 5f0f98abc4ac..3e8002652d6f 100644 --- a/libevmasm/Assembly.h +++ b/libevmasm/Assembly.h @@ -191,8 +191,8 @@ class Assembly size_t _modifierDepth, int _begin, int _end, - std::string _value = std::string(), - std::string _jumpType = std::string() + std::string const& _value = std::string(), + std::string const& _jumpType = std::string() ); static std::string toStringInHex(u256 _value); From e8f76ff661f893dbe7fd3b18f85036051bdd662c Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Mon, 14 Mar 2022 15:51:55 -0500 Subject: [PATCH 09/10] scripts/ImportExportTest.sh: adding test for --optimized --- scripts/ImportExportTest.sh | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/scripts/ImportExportTest.sh b/scripts/ImportExportTest.sh index d0f0c5a3325c..11ff9726373d 100755 --- a/scripts/ImportExportTest.sh +++ b/scripts/ImportExportTest.sh @@ -99,11 +99,13 @@ function testImportExportEquivalence { UNCOMPILABLE=$((UNCOMPILABLE + 1)) return 0 else + $SOLC --optimize --combined-json bin,bin-runtime,opcodes,asm,srcmap,srcmap-runtime --pretty-json "$nth_input_file" "${all_input_files[@]}" > expected_optimized.json 2> expected_optimized.error for contract in $(jq '.contracts | keys | .[]' expected.json 2> /dev/null) do for type in "${types[@]}" do jq --raw-output ".contracts.${contract}.\"${type}\"" expected.json > "expected.${type}" + jq --raw-output ".contracts.${contract}.\"${type}\"" expected_optimized.json > "expected_optimized.${type}" done assembly=$(cat expected.asm) @@ -117,27 +119,47 @@ function testImportExportEquivalence { FAILED=$((FAILED + 1)) return 0 else + $SOLC --combined-json bin,bin-runtime,opcodes,asm,srcmap,srcmap-runtime --pretty-json --import-asm-json expected.asm > obtained_optimized.json 2> obtained_optimized.error for type in "${types[@]}" do for obtained_contract in $(jq '.contracts | keys | .[]' obtained.json 2> /dev/null) do jq --raw-output ".contracts.${obtained_contract}.\"${type}\"" obtained.json > "obtained.${type}" + jq --raw-output ".contracts.${obtained_contract}.\"${type}\"" obtained_optimized.json > "obtained_optimized.${type}" set +e DIFF="$(diff "expected.${type}" "obtained.${type}")" + # DIFF_OPTIMIZED="$(diff "expected_optimized.${type}" "obtained_optimized.${type}")" set -e - if [ "$DIFF" != "" ] + if [ "$DIFFVIEW" == "" ] then - if [ "$DIFFVIEW" == "" ] + if [ "$DIFF" != "" ] then echo -e "ERROR: JSONS differ for $1: \n $DIFF \n" echo "Expected:" cat "expected.${type}" echo "Obtained:" cat "obtained.${type}" - else - # Use user supplied diff view binary + fi + if [ "$DIFF_OPTIMIZED" != "" ] + then + echo -e "ERROR: JSONS (optimized) differ for $1: \n $DIFF \n" + echo "Expected (optimized):" + cat "expected_optimized.${type}" + echo "Obtained (optimized):" + cat "obtained_optimized.${type}" + fi + else + # Use user supplied diff view binary + if [ "$DIFF" != "" ] + then + echo "$DIFFVIEW expected.json obtained.json" $DIFFVIEW expected.json obtained.json fi + if [ "$DIFF_OPTIMIZED" != "" ] + then + echo "$DIFFVIEW expected_optimized.json obtained_optimized.json" + $DIFFVIEW expected_optimized.json obtained_optimized.json + fi _TESTED= FAILED=$((FAILED + 1)) return 0 From 6700a65f52e637101c24599c8018ffd61ac7522b Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Mon, 14 Mar 2022 17:40:06 -0500 Subject: [PATCH 10/10] scripts/ImportExportTest.sh: disable --optimize tests for evm-assembly. --- scripts/ImportExportTest.sh | 67 ++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/scripts/ImportExportTest.sh b/scripts/ImportExportTest.sh index 11ff9726373d..f20fd46351ec 100755 --- a/scripts/ImportExportTest.sh +++ b/scripts/ImportExportTest.sh @@ -99,13 +99,13 @@ function testImportExportEquivalence { UNCOMPILABLE=$((UNCOMPILABLE + 1)) return 0 else - $SOLC --optimize --combined-json bin,bin-runtime,opcodes,asm,srcmap,srcmap-runtime --pretty-json "$nth_input_file" "${all_input_files[@]}" > expected_optimized.json 2> expected_optimized.error + # $SOLC --optimize --combined-json bin,bin-runtime,opcodes,asm,srcmap,srcmap-runtime --pretty-json "$nth_input_file" "${all_input_files[@]}" > expected_optimized.json 2> expected_optimized.error || true for contract in $(jq '.contracts | keys | .[]' expected.json 2> /dev/null) do for type in "${types[@]}" do jq --raw-output ".contracts.${contract}.\"${type}\"" expected.json > "expected.${type}" - jq --raw-output ".contracts.${contract}.\"${type}\"" expected_optimized.json > "expected_optimized.${type}" + # jq --raw-output ".contracts.${contract}.\"${type}\"" expected_optimized.json > "expected_optimized.${type}" done assembly=$(cat expected.asm) @@ -119,46 +119,51 @@ function testImportExportEquivalence { FAILED=$((FAILED + 1)) return 0 else - $SOLC --combined-json bin,bin-runtime,opcodes,asm,srcmap,srcmap-runtime --pretty-json --import-asm-json expected.asm > obtained_optimized.json 2> obtained_optimized.error + # try to optimize asm json import. + # (may fail, because of e.g. "Some immutables were read from but never assigned, possibly because of optimization") + # $SOLC --combined-json bin,bin-runtime,opcodes,asm,srcmap,srcmap-runtime --pretty-json --import-asm-json expected.asm > obtained_optimized.json 2> obtained_optimized.error || true for type in "${types[@]}" do for obtained_contract in $(jq '.contracts | keys | .[]' obtained.json 2> /dev/null) do jq --raw-output ".contracts.${obtained_contract}.\"${type}\"" obtained.json > "obtained.${type}" - jq --raw-output ".contracts.${obtained_contract}.\"${type}\"" obtained_optimized.json > "obtained_optimized.${type}" + # jq --raw-output ".contracts.${obtained_contract}.\"${type}\"" obtained_optimized.json > "obtained_optimized.${type}" set +e DIFF="$(diff "expected.${type}" "obtained.${type}")" # DIFF_OPTIMIZED="$(diff "expected_optimized.${type}" "obtained_optimized.${type}")" set -e - if [ "$DIFFVIEW" == "" ] + if [ "$DIFF" != "" ] || [ "$DIFF_OPTIMIZED" != "" ] then - if [ "$DIFF" != "" ] - then - echo -e "ERROR: JSONS differ for $1: \n $DIFF \n" - echo "Expected:" - cat "expected.${type}" - echo "Obtained:" - cat "obtained.${type}" - fi - if [ "$DIFF_OPTIMIZED" != "" ] - then - echo -e "ERROR: JSONS (optimized) differ for $1: \n $DIFF \n" - echo "Expected (optimized):" - cat "expected_optimized.${type}" - echo "Obtained (optimized):" - cat "obtained_optimized.${type}" - fi - else - # Use user supplied diff view binary - if [ "$DIFF" != "" ] - then - echo "$DIFFVIEW expected.json obtained.json" - $DIFFVIEW expected.json obtained.json - fi - if [ "$DIFF_OPTIMIZED" != "" ] + if [ "$DIFFVIEW" == "" ] then - echo "$DIFFVIEW expected_optimized.json obtained_optimized.json" - $DIFFVIEW expected_optimized.json obtained_optimized.json + if [ "$DIFF" != "" ] + then + echo -e "ERROR: EVM Assembly JSON differ for $1: \n $DIFF \n" + echo "Expected:" + cat "expected.${type}" + echo "Obtained:" + cat "obtained.${type}" + fi + if [ "$DIFF_OPTIMIZED" != "" ] + then + echo -e "ERROR: EVM Assembly JSON (optimization was enabled) differ for $1: \n $DIFF \n" + echo "Expected (optimized):" + cat "expected_optimized.${type}" + echo "Obtained (optimized):" + cat "obtained_optimized.${type}" + fi + else + # Use user supplied diff view binary + if [ "$DIFF" != "" ] + then + echo "$DIFFVIEW expected.json obtained.json" + $DIFFVIEW expected.json obtained.json + fi + if [ "$DIFF_OPTIMIZED" != "" ] + then + echo "$DIFFVIEW expected_optimized.json obtained_optimized.json" + $DIFFVIEW expected_optimized.json obtained_optimized.json + fi fi _TESTED= FAILED=$((FAILED + 1))