From afa974463933bcf69c7570b74ec2f4704e807400 Mon Sep 17 00:00:00 2001 From: Nikola Matic Date: Tue, 21 Apr 2026 12:24:59 +0200 Subject: [PATCH] Migrate ethdebug CLI tests to isolest --- test/CMakeLists.txt | 4 + test/InteractiveTests.h | 2 + test/libsolidity/EthdebugTest.cpp | 162 ++++++++ test/libsolidity/EthdebugTest.h | 72 ++++ test/libsolidity/JSONExpectationTest.cpp | 391 ++++++++++++++++++ test/libsolidity/JSONExpectationTest.h | 117 ++++++ .../ethdebugTests/abstract_and_interface.sol | 16 + .../ethdebugTests/basic_contract.sol | 24 ++ .../ethdebugTests/context_source_mapping.sol | 23 ++ .../isoltestTesting/compilation_metadata.sol | 14 + .../ethdebugTests/isoltestTesting/filters.sol | 11 + .../isoltestTesting/not_found.sol | 7 + .../isoltestTesting/object_matching.sol | 36 ++ .../isoltestTesting/paren_path_syntax.sol | 8 + .../isoltestTesting/resources_only.sol | 8 + .../isoltestTesting/value_syntax_forms.sol | 13 + .../ethdebugTests/multi_source.sol | 15 + .../multi_source_name_collision.sol | 19 + .../libsolidity/ethdebugTests/via_ssa_cfg.sol | 8 + .../via_ssa_cfg_with_function.sol | 13 + test/tools/CMakeLists.txt | 2 + 21 files changed, 965 insertions(+) create mode 100644 test/libsolidity/EthdebugTest.cpp create mode 100644 test/libsolidity/EthdebugTest.h create mode 100644 test/libsolidity/JSONExpectationTest.cpp create mode 100644 test/libsolidity/JSONExpectationTest.h create mode 100644 test/libsolidity/ethdebugTests/abstract_and_interface.sol create mode 100644 test/libsolidity/ethdebugTests/basic_contract.sol create mode 100644 test/libsolidity/ethdebugTests/context_source_mapping.sol create mode 100644 test/libsolidity/ethdebugTests/isoltestTesting/compilation_metadata.sol create mode 100644 test/libsolidity/ethdebugTests/isoltestTesting/filters.sol create mode 100644 test/libsolidity/ethdebugTests/isoltestTesting/not_found.sol create mode 100644 test/libsolidity/ethdebugTests/isoltestTesting/object_matching.sol create mode 100644 test/libsolidity/ethdebugTests/isoltestTesting/paren_path_syntax.sol create mode 100644 test/libsolidity/ethdebugTests/isoltestTesting/resources_only.sol create mode 100644 test/libsolidity/ethdebugTests/isoltestTesting/value_syntax_forms.sol create mode 100644 test/libsolidity/ethdebugTests/multi_source.sol create mode 100644 test/libsolidity/ethdebugTests/multi_source_name_collision.sol create mode 100644 test/libsolidity/ethdebugTests/via_ssa_cfg.sol create mode 100644 test/libsolidity/ethdebugTests/via_ssa_cfg_with_function.sol diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3cceb9a47282..920cec2dc4af 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -79,6 +79,8 @@ set(libsolidity_sources libsolidity/ASTJSONTest.h libsolidity/ErrorCheck.cpp libsolidity/ErrorCheck.h + libsolidity/EthdebugTest.cpp + libsolidity/EthdebugTest.h libsolidity/FunctionDependencyGraphTest.cpp libsolidity/FunctionDependencyGraphTest.h libsolidity/GasCosts.cpp @@ -87,6 +89,8 @@ set(libsolidity_sources libsolidity/GasTest.h libsolidity/Imports.cpp libsolidity/InlineAssembly.cpp + libsolidity/JSONExpectationTest.cpp + libsolidity/JSONExpectationTest.h libsolidity/LibSolc.cpp libsolidity/Metadata.cpp libsolidity/MemoryGuardTest.cpp diff --git a/test/InteractiveTests.h b/test/InteractiveTests.h index a232494d6222..edc6b9d1fc8a 100644 --- a/test/InteractiveTests.h +++ b/test/InteractiveTests.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -93,6 +94,7 @@ Testsuite const g_interactiveTestsuites[] = { {"JSON ABI", "libsolidity", "ABIJson", false, false, &ABIJsonTest::create}, {"JSON Natspec", "libsolidity", "natspecJSON", false, false, &NatspecJSONTest::create}, {"SMT Checker", "libsolidity", "smtCheckerTests", true, false, &SMTCheckerTest::create}, + {"Ethdebug", "libsolidity", "ethdebugTests", false, false, &EthdebugTest::create}, {"Gas Estimates", "libsolidity", "gasTests", false, false, &GasTest::create}, {"Memory Guard", "libsolidity", "memoryGuardTests", false, false, &MemoryGuardTest::create}, {"AST Properties", "libsolidity", "astPropertyTests", false, false, &ASTPropertyTest::create}, diff --git a/test/libsolidity/EthdebugTest.cpp b/test/libsolidity/EthdebugTest.cpp new file mode 100644 index 000000000000..8681c2d4f3c4 --- /dev/null +++ b/test/libsolidity/EthdebugTest.cpp @@ -0,0 +1,162 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +// SPDX-License-Identifier: GPL-3.0 + +#include +#include + +#include + +#include + +#include + +#include +#include + +using namespace solidity; +using namespace solidity::frontend; +using namespace solidity::frontend::test; +using namespace solidity::langutil; +using namespace std::string_literals; + +EthdebugTest::EthdebugTest(std::string const& _filename): + JSONExpectationTest(_filename) +{ + m_optimise = m_reader.boolSetting("optimize", false); + m_optimiseYul = m_reader.boolSetting("optimize-yul", false); + m_useSSACFG = m_reader.boolSetting("compileViaSSACFG", false); + + auto const revertStrings = revertStringsFromString(m_reader.stringSetting("revertStrings", "default")); + if (!revertStrings) + BOOST_THROW_EXCEPTION(std::runtime_error("Invalid revertStrings setting.")); + m_revertStrings = *revertStrings; +} + +void EthdebugTest::setupCompiler(CompilerStack& _compiler) +{ + AnalysisFramework::setupCompiler(_compiler); + + _compiler.setViaIR(true); + _compiler.setExperimental(true); + if (m_useSSACFG) + _compiler.setViaSSACFG(true); + + DebugInfoSelection selection = DebugInfoSelection::Default(); + selection.enable("ethdebug"); + _compiler.selectDebugInfo(selection); + + _compiler.setMetadataFormat(CompilerStack::MetadataFormat::NoMetadata); + _compiler.setRevertStringBehaviour(m_revertStrings); + + OptimiserSettings settings = m_optimise ? OptimiserSettings::standard() : OptimiserSettings::minimal(); + if (m_optimiseYul) + settings.runYulOptimiser = true; + _compiler.setOptimiserSettings(settings); +} + +JSONExpectationTest::PathParts EthdebugTest::splitPath(std::string_view _path) const +{ + if (_path.empty()) + BOOST_THROW_EXCEPTION(std::runtime_error("Empty path in expectation")); + if (_path[0] == ' ' || _path[0] == '\t') + BOOST_THROW_EXCEPTION(std::runtime_error( + "Path must not have leading whitespace: "s.append(_path) + )); + + if (_path[0] == '(') + return JSONExpectationTest::splitPath(_path); + + if (_path[0] == '.') + { + auto const secondDot = _path.find('.', 1); + if (secondDot == std::string_view::npos) + return {std::string_view{}, _path.substr(1), std::string_view{}}; + return { + std::string_view{}, + _path.substr(1, secondDot - 1), + _path.substr(secondDot + 1) + }; + } + + auto const colonPos = _path.find(':'); + size_t const searchStart = colonPos == std::string_view::npos ? 0 : colonPos + 1; + + auto const outputDot = _path.find('.', searchStart); + if (outputDot == std::string_view::npos) + return {_path, std::string_view{}, std::string_view{}}; + + auto const pathDot = _path.find('.', outputDot + 1); + if (pathDot == std::string_view::npos) + return { + _path.substr(0, outputDot), + _path.substr(outputDot + 1), + std::string_view{} + }; + return { + _path.substr(0, outputDot), + _path.substr(outputDot + 1, pathDot - outputDot - 1), + _path.substr(pathDot + 1) + }; +} + +std::optional EthdebugTest::fetchOutput( + std::string_view _qualifiedContract, + std::string_view _outputName +) const +{ + if (compiler().state() < CompilerStack::State::CompilationSuccessful) + return std::nullopt; + + if (_qualifiedContract.empty()) + { + if (_outputName == "compilation") + return compiler().ethdebugCompilation(); + if (_outputName == "resources") + return compiler().ethdebug(); + return std::nullopt; + } + + if (auto const resolved = resolveContractName(_qualifiedContract)) + { + if (_outputName == "creation") + return compiler().ethdebug(*resolved); + if (_outputName == "runtime") + return compiler().ethdebugRuntime(*resolved); + if (_outputName == "contract") + { + Json const creation = compiler().ethdebug(*resolved); + if (creation.is_null() || !creation.contains("contract")) + return std::nullopt; + return creation["contract"]; + } + } + return std::nullopt; +} + +std::optional EthdebugTest::resolveContractName(std::string_view _name) const +{ + auto const& contractNames = compiler().contractNames(); + + auto const isMatch = [&](std::string const& contractName) { + return contractName == _name || compiler().contractDefinition(contractName).name() == _name; + }; + + if (std::ranges::count_if(contractNames, isMatch) != 1) + return std::nullopt; + return *std::ranges::find_if(contractNames, isMatch); +} diff --git a/test/libsolidity/EthdebugTest.h b/test/libsolidity/EthdebugTest.h new file mode 100644 index 000000000000..7a0955230f01 --- /dev/null +++ b/test/libsolidity/EthdebugTest.h @@ -0,0 +1,72 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +// SPDX-License-Identifier: GPL-3.0 + +#pragma once + +#include + +#include + +namespace solidity::frontend::test +{ +/// Isoltest test case for asserting against the compiler's ethdebug JSON output. +/// +/// Settings (in the `// ====` block): +/// - `EVMVersion: ` — gates whether the test runs (inherited). +/// - `optimize: ` — enable optimizer (default: false). +/// - `optimize-yul: ` — additionally run the Yul optimizer (default: false). +/// - `compileViaSSACFG: ` — route codegen through the SSA-CFG (default: false). +/// - `revertStrings: default|strip|debug|verboseDebug` +/// — revert string behavior (default: default). +/// +/// Compilation is always performed with `viaIR=true`, `experimental=true`, ethdebug +/// debug info enabled, and metadata stripped, so the output is stable across builds. +/// +/// Scope keys exposed to expectations: +/// - Globals: `.resources`, `.compilation`. +/// - Per contract: `Contract.creation`, `Contract.runtime`, `Contract.contract`. +/// - Source-qualified per contract (when needed to disambiguate same-named +/// contracts in different sources): `source.sol:Contract.creation`, etc. +class EthdebugTest: public JSONExpectationTest +{ +public: + static std::unique_ptr create(Config const& _config) + { + return std::make_unique(_config.filename); + } + + EthdebugTest(std::string const& _filename); + +protected: + void setupCompiler(CompilerStack& _compiler) override; + PathParts splitPath(std::string_view _path) const override; + std::optional fetchOutput( + std::string_view _qualifiedContract, + std::string_view _outputName + ) const override; + +private: + std::optional resolveContractName(std::string_view _name) const; + + bool m_optimise = false; + bool m_optimiseYul = false; + bool m_useSSACFG = false; + RevertStrings m_revertStrings = RevertStrings::Default; +}; + +} diff --git a/test/libsolidity/JSONExpectationTest.cpp b/test/libsolidity/JSONExpectationTest.cpp new file mode 100644 index 000000000000..40c731cc9145 --- /dev/null +++ b/test/libsolidity/JSONExpectationTest.cpp @@ -0,0 +1,391 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +// SPDX-License-Identifier: GPL-3.0 + +#include +#include + +#include +#include +#include + +#include + +#include +#include + +using namespace solidity; +using namespace solidity::frontend; +using namespace solidity::frontend::test; +using namespace solidity::util; +using namespace std::string_literals; + +JSONExpectationTest::JSONExpectationTest(std::string const& _filename): + EVMVersionRestrictedTestCase(_filename), + m_sources(m_reader.sources()) +{ + parseExpectations(m_reader.stream()); +} + +void JSONExpectationTest::parseExpectations(std::istream& _stream) +{ + std::string line; + while (getline(_stream, line)) + { + if (!line.starts_with("// ")) + { + if (line == "//") + continue; + BOOST_THROW_EXCEPTION(std::runtime_error("Invalid expectation: expected \"// \" prefix in line: " + line)); + } + + std::string_view stripped(line); + stripped.remove_prefix(3); + + auto const startPos = stripped.find_first_not_of(" \t"); + if (startPos == std::string_view::npos) + continue; + stripped.remove_prefix(startPos); + + auto const colonPos = stripped.rfind(':'); + if (colonPos != std::string_view::npos && + stripped.find_first_not_of(" \t", colonPos + 1) == std::string_view::npos) + { + Expectation exp = parsePathPart(stripped.substr(0, colonPos)); + exp.value = extractMultiLineExpectationJSON(_stream); + m_expectations.push_back(std::move(exp)); + } + else + m_expectations.push_back(parseSingleLineExpectation(stripped, _stream)); + } +} + +// TODO: O(N^2) — Json::accept() reparses the whole block per line. Acceptable at current +// fixture sizes; switch to indent-based termination if it becomes a problem. +Json JSONExpectationTest::extractMultiLineExpectationJSON(std::istream& _stream, std::string _rawJSON) +{ + if (!_rawJSON.empty()) + _rawJSON += "\n"; + + auto const finalize = [&]() { + Json parsed; + std::string errors; + if (!jsonParseStrict(_rawJSON, parsed, &errors)) + BOOST_THROW_EXCEPTION(std::runtime_error( + "Failed to parse JSON expectation block:\n" + _rawJSON + "\nErrors:\n" + errors + )); + return parsed; + }; + + if (Json::accept(_rawJSON)) + return finalize(); + + std::string line; + while (getline(_stream, line)) + { + std::string_view stripped; + if (line.starts_with("// ")) + stripped = std::string_view(line).substr(3); + else if (line == "//") + stripped = ""; + else + BOOST_THROW_EXCEPTION(std::runtime_error("Invalid expectation: expected \"// \" prefix in JSON block: " + line)); + + _rawJSON += stripped; + _rawJSON += "\n"; + + if (Json::accept(_rawJSON)) + return finalize(); + } + std::string errors; + Json discarded; + jsonParseStrict(_rawJSON, discarded, &errors); + BOOST_THROW_EXCEPTION(std::runtime_error( + "Unterminated or malformed multi-line JSON expectation block:\n" + _rawJSON + + (errors.empty() ? "" : "\nErrors:\n" + errors) + )); +} + +JSONExpectationTest::Expectation JSONExpectationTest::parseSingleLineExpectation(std::string_view _line, std::istream& _stream) +{ + auto const colonPos = _line.find(": "); + if (colonPos == std::string_view::npos) + BOOST_THROW_EXCEPTION(std::runtime_error( + "Missing ': ' separator in expectation line: "s.append(_line) + )); + + std::string_view const pathPart = _line.substr(0, colonPos); + std::string const rawValue(_line.substr(colonPos + 2)); + + Expectation result = parsePathPart(pathPart); + Json parsed; + if (jsonParseStrict(rawValue, parsed)) + result.value = std::move(parsed); + else if (!rawValue.empty() && (rawValue.front() == '{' || rawValue.front() == '[')) + result.value = extractMultiLineExpectationJSON(_stream, rawValue); + else + result.value = Json(rawValue); + return result; +} + +JSONExpectationTest::Expectation JSONExpectationTest::parsePathPart(std::string_view _pathPart) +{ + Expectation result; + + auto const pipePos = _pathPart.find(" | "); + if (pipePos != std::string_view::npos) + { + result.filter = std::string(_pathPart.substr(pipePos + 3)); + _pathPart = _pathPart.substr(0, pipePos); + } + + result.fullPath = std::string(_pathPart); + return result; +} + +JSONExpectationTest::PathParts JSONExpectationTest::splitPath(std::string_view _path) const +{ + auto const open = _path.find('('); + if (open == std::string_view::npos) + BOOST_THROW_EXCEPTION(std::runtime_error( + "Path must use `(scope) jsonPath` form: "s.append(_path) + )); + auto const close = _path.find(')', open); + if (close == std::string_view::npos) + BOOST_THROW_EXCEPTION(std::runtime_error( + "Unterminated `(...)` scope in path: "s.append(_path) + )); + + auto const trimLeading = [](std::string_view _sv) { + _sv.remove_prefix(std::min(_sv.find_first_not_of(" \t"), _sv.size())); + return _sv; + }; + + std::string_view const inside = trimLeading(_path.substr(open + 1, close - open - 1)); + std::string_view const jsonPath = trimLeading(_path.substr(close + 1)); + + auto const sep = inside.find_first_of(" \t"); + if (sep == std::string_view::npos) + return {std::string_view{}, inside, jsonPath}; + return {inside.substr(0, sep), trimLeading(inside.substr(sep)), jsonPath}; +} + +namespace +{ +// Convert expectation path, e.g. "instructions[0].offset" to RFC 6901 JSON pointer, i.e. +// /instructions/0/offset to be later used in JSON lookups. +nlohmann::json::json_pointer pathToPointer(std::string_view _path) +{ + if (_path.empty()) + return nlohmann::json::json_pointer{}; + + std::string ptr = "/"; + for (char c: _path) + { + if (c == '.' || c == '[') + ptr += '/'; + else if (c != ']') + ptr += c; + } + return nlohmann::json::json_pointer(ptr); +} +} + +std::optional JSONExpectationTest::resolvePath(Json const& _json, std::string_view _path) const +{ + auto const ptr = pathToPointer(_path); + if (!_json.contains(ptr)) + return std::nullopt; + return _json.at(ptr); +} + +Json JSONExpectationTest::applyFilter(Json const& _json, std::string_view _filter) const +{ + if (_filter == "length") + { + if (_json.is_array() || _json.is_object()) + return Json(static_cast(_json.size())); + if (_json.is_string()) + return Json(static_cast(_json.get().size())); + BOOST_THROW_EXCEPTION(std::runtime_error( + "'length' filter requires an array, object, or string; got "s + _json.type_name() + )); + } + if (_filter == "type") + return Json(_json.type_name()); + if (_filter == "keys") + { + if (!_json.is_object()) + BOOST_THROW_EXCEPTION(std::runtime_error( + "'keys' filter requires an object; got "s + _json.type_name() + )); + Json result = Json::array(); + // Json uses std::map for object storage, so items() iterates keys in + // lexicographic order. Tests rely on this for stable output. + for (auto const& [key, _]: _json.items()) + result.push_back(key); + return result; + } + + BOOST_THROW_EXCEPTION(std::runtime_error("Unknown filter: "s.append(_filter))); +} + +std::string JSONExpectationTest::formatPathPrefix(Expectation const& _exp) +{ + return _exp.filter.empty() ? _exp.fullPath : _exp.fullPath + " | " + _exp.filter; +} + +std::string JSONExpectationTest::formatValueAs(Json const& _value) const +{ + if (_value.is_string()) + { + auto const& content = _value.get_ref(); + Json probe; + if (jsonParseStrict(content, probe)) + return _value.dump(); + return content; + } + if (_value.is_object() || _value.is_array()) + { + bool const hasNested = std::any_of(_value.begin(), _value.end(), [](Json const& _el) { + return _el.is_structured(); + }); + return hasNested + ? jsonPrint(_value, {JsonFormat::Pretty, 4}) + : jsonCompactPrint(_value); + } + return _value.dump(); +} + +void JSONExpectationTest::applyPlaceholders(Json const& _expected, Json& _obtained) const +{ + if (_expected.is_string() && _expected.get() == "") + { + _obtained = _expected; + return; + } + if (_expected.type() != _obtained.type()) + return; + if (_expected.is_object()) + { + for (auto const& [key, expectedVal]: _expected.items()) + if (_obtained.contains(key)) + applyPlaceholders(expectedVal, _obtained[key]); + } + else if (_expected.is_array()) + for (size_t i = 0; i < std::min(_expected.size(), _obtained.size()); ++i) + applyPlaceholders(_expected[i], _obtained[i]); +} + +TestCase::TestResult JSONExpectationTest::run(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) +{ + if (!runFramework(withPreamble(m_sources.sources), PipelineStage::Compilation)) + { + printPrefixed(_stream, formatErrors(filteredErrors(false /* _includeWarningsAndInfos */), _formatted), _linePrefix); + return TestResult::FatalError; + } + + bool const allMatch = std::ranges::all_of(m_expectations, [&](Expectation const& _expectation) { + auto const parts = splitPath(_expectation.fullPath); + auto const isExpected = [&](std::string_view _sentinel) { + return _expectation.value.is_string() && _expectation.value.get() == _sentinel; + }; + + auto const output = fetchOutput(parts.qualifiedContract, parts.outputName); + if (!output) + return isExpected(""); + + auto const actualLookup = resolvePath(*output, parts.jsonPath); + if (!actualLookup) + return isExpected(""); + + Json actualJson = *actualLookup; + if (!_expectation.filter.empty()) + actualJson = applyFilter(actualJson, _expectation.filter); + + Json patched = actualJson; + applyPlaceholders(_expectation.value, patched); + return patched == _expectation.value; + }); + + if (allMatch) + return TestResult::Success; + + _stream << _linePrefix << "Expected:" << std::endl; + printPrefixed(_stream, formatExpectations(m_expectations), _linePrefix + " "); + _stream << _linePrefix << "Obtained:" << std::endl; + printUpdatedExpectations(_stream, _linePrefix + " "); + return TestResult::Failure; +} + +void JSONExpectationTest::printSource(std::ostream& _stream, std::string const& _linePrefix, bool /*_formatted*/) const +{ + if (m_sources.sources.empty()) + return; + + bool const outputSourceNames = m_sources.sources.size() != 1 || !m_sources.sources.begin()->first.empty(); + for (auto const& [name, source]: m_sources.sources) + { + if (outputSourceNames) + _stream << _linePrefix << "==== Source: " << name << " ====" << std::endl; + printPrefixed(_stream, source, _linePrefix); + } +} + +std::string JSONExpectationTest::formatExpectations(std::vector const& _expectations) const +{ + std::string output; + for (auto const& exp: _expectations) + { + std::string const formatted = formatValueAs(exp.value); + output += formatPathPrefix(exp) + ": " + formatted; + if (formatted.empty() || formatted.back() != '\n') + output += '\n'; + } + return output; +} + +void JSONExpectationTest::printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const +{ + for (auto const& exp: m_expectations) + { + auto const parts = splitPath(exp.fullPath); + std::string const pathPrefix = formatPathPrefix(exp); + + auto const output = fetchOutput(parts.qualifiedContract, parts.outputName); + if (!output) + { + _stream << _linePrefix << pathPrefix << ": " << std::endl; + continue; + } + + auto const resolved = resolvePath(*output, parts.jsonPath); + if (!resolved) + { + _stream << _linePrefix << pathPrefix << ": " << std::endl; + continue; + } + + Json value = *resolved; + if (!exp.filter.empty()) + value = applyFilter(value, exp.filter); + applyPlaceholders(exp.value, value); + + std::string const formatted = formatValueAs(value); + printPrefixed(_stream, pathPrefix + ": " + formatted, _linePrefix); + } +} diff --git a/test/libsolidity/JSONExpectationTest.h b/test/libsolidity/JSONExpectationTest.h new file mode 100644 index 000000000000..f260131b50ee --- /dev/null +++ b/test/libsolidity/JSONExpectationTest.h @@ -0,0 +1,117 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +// SPDX-License-Identifier: GPL-3.0 + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace solidity::frontend::test +{ +/// Base class for isoltest test cases that drive a Solidity compilation and assert +/// against a JSON output. +/// +/// Expectations +/// ------------ +/// // path [| filter]: expected +/// +/// `expected` is a JSON value and may continue across subsequent `// ` lines. Bare +/// scalars like `solc`, `PUSH1`, `0x80` or `` are accepted without quoting; +/// use the quoted form (e.g. `"42"`) for strings whose content looks like another +/// JSON type. Lines without `: ` are rejected. +/// +/// Filters (applied to the resolved value before comparison): +/// - `| length` — array/object/string size as a number +/// - `| type` — JSON type name ("object", "array", "number", …) +/// - `| keys` — list of keys (object only); always lexicographically sorted +/// +/// Sentinels (assert absence): +/// - `` — passes when the referenced contract/output is absent +/// - `` — passes when the inner JSON path is absent +/// +/// Placeholder: +/// - `` matches any value at the position where it appears. +/// Limitations: only works at string leaves (not as a dict key); matches whole +/// values (not fragments like `v`); has no escape; multiple occurrences +/// match independently. +/// +/// Example: +/// // C.instructions[0].offset: 0 +/// // C.instructions | length: 3 +/// // C.compilation: { +/// // "compiler": {"name": "solc", "version": ""} +/// // } +/// // Missing.contract.name: +/// // C.contract.nope: +class JSONExpectationTest: public AnalysisFramework, public EVMVersionRestrictedTestCase +{ +public: + JSONExpectationTest(std::string const& _filename); + + TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override; + void printSource(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) const override; + void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override; + +protected: + struct Expectation + { + std::string fullPath; + std::string filter; // optional filter, e.g. "length" + Json value; // parsed expected value (placeholder leaves stored as their literal name) + }; + + struct PathParts + { + std::string_view qualifiedContract; + std::string_view outputName; + std::string_view jsonPath; + }; + + /// Splits a path into (qualifiedContract, outputName, jsonPath). Default parses `(scope) jsonPath`. + virtual PathParts splitPath(std::string_view _path) const; + + virtual std::optional fetchOutput( + std::string_view _qualifiedContract, + std::string_view _outputName + ) const = 0; + +private: + void parseExpectations(std::istream& _stream); + Expectation parseSingleLineExpectation(std::string_view _line, std::istream& _stream); + Expectation parsePathPart(std::string_view _pathPart); + Json extractMultiLineExpectationJSON(std::istream& _stream, std::string _rawJSON = ""); + + std::optional resolvePath(Json const& _json, std::string_view _path) const; + Json applyFilter(Json const& _json, std::string_view _filter) const; + std::string formatValueAs(Json const& _value) const; + void applyPlaceholders(Json const& _expected, Json& _obtained) const; + std::string formatExpectations(std::vector const& _expectations) const; + static std::string formatPathPrefix(Expectation const& _exp); + + SourceMap m_sources; + std::vector m_expectations; +}; + +} // namespace diff --git a/test/libsolidity/ethdebugTests/abstract_and_interface.sol b/test/libsolidity/ethdebugTests/abstract_and_interface.sol new file mode 100644 index 000000000000..5f4feacc8249 --- /dev/null +++ b/test/libsolidity/ethdebugTests/abstract_and_interface.sol @@ -0,0 +1,16 @@ +interface I { + function f() external; +} + +abstract contract A { + function g() public virtual; +} + +contract C is A, I { + function f() external override {} + function g() public override {} +} +// ---- +// C.contract.name: C +// C.creation.environment: create +// C.runtime.environment: call diff --git a/test/libsolidity/ethdebugTests/basic_contract.sol b/test/libsolidity/ethdebugTests/basic_contract.sol new file mode 100644 index 000000000000..69637daaffa0 --- /dev/null +++ b/test/libsolidity/ethdebugTests/basic_contract.sol @@ -0,0 +1,24 @@ +contract C { + uint public x; + function set(uint _x) public { + x = _x; + } + function get() public view returns (uint) { + return x; + } +} +// ---- +// .compilation.compiler.name: solc +// .resources.compilation.sources | length: 1 +// .resources.compilation.sources[0].id: 0 +// +// C.contract.name: C +// C.creation.environment: create +// C.contract.definition.source.id: 0 +// C.creation.instructions[0].offset: 0 +// C.creation.instructions[0].operation.mnemonic: PUSH1 +// C.creation.instructions[0].operation.arguments[0]: 0x80 +// C.creation.instructions[2].operation.mnemonic: MSTORE +// +// C.runtime.contract.name: C +// C.runtime.environment: call diff --git a/test/libsolidity/ethdebugTests/context_source_mapping.sol b/test/libsolidity/ethdebugTests/context_source_mapping.sol new file mode 100644 index 000000000000..075d66322395 --- /dev/null +++ b/test/libsolidity/ethdebugTests/context_source_mapping.sol @@ -0,0 +1,23 @@ +contract C { + uint public x; + function set(uint _x) public { + x = _x; + } + function get() public view returns (uint) { + return x; + } +} +// ---- +// C.creation.instructions[0].context.code.range.offset: 59 +// C.creation.instructions[0].context.code.range.length: 162 +// C.creation.instructions[0].context.code.source.id: 0 +// C.runtime.instructions[0].context.code: +// { +// "range": { +// "length": 162, +// "offset": 59 +// }, +// "source": { +// "id": 0 +// } +// } diff --git a/test/libsolidity/ethdebugTests/isoltestTesting/compilation_metadata.sol b/test/libsolidity/ethdebugTests/isoltestTesting/compilation_metadata.sol new file mode 100644 index 000000000000..cfcdb89b9ec7 --- /dev/null +++ b/test/libsolidity/ethdebugTests/isoltestTesting/compilation_metadata.sol @@ -0,0 +1,14 @@ +contract C { + function f() public pure {} +} +// ---- +// .compilation.compiler.name: solc +// .compilation.compiler | type: object +// .compilation.compiler | keys: ["name", "version"] +// .compilation: +// { +// "compiler": { +// "name": "solc", +// "version": "" +// } +// } diff --git a/test/libsolidity/ethdebugTests/isoltestTesting/filters.sol b/test/libsolidity/ethdebugTests/isoltestTesting/filters.sol new file mode 100644 index 000000000000..5d17a0fb33d8 --- /dev/null +++ b/test/libsolidity/ethdebugTests/isoltestTesting/filters.sol @@ -0,0 +1,11 @@ +contract C { + function f() public pure returns (uint) { + return 42; + } +} +// ---- +// .resources.compilation.sources | length: 1 +// C.contract | type: object +// C.contract | keys: ["definition", "name"] +// C.creation.instructions | type: array +// C.creation.environment | type: string diff --git a/test/libsolidity/ethdebugTests/isoltestTesting/not_found.sol b/test/libsolidity/ethdebugTests/isoltestTesting/not_found.sol new file mode 100644 index 000000000000..e1cb724e43e4 --- /dev/null +++ b/test/libsolidity/ethdebugTests/isoltestTesting/not_found.sol @@ -0,0 +1,7 @@ +contract C { + function f() public {} +} +// ---- +// C.contract.name: C +// C.contract.nonexistentKey: +// Missing.contract.name: diff --git a/test/libsolidity/ethdebugTests/isoltestTesting/object_matching.sol b/test/libsolidity/ethdebugTests/isoltestTesting/object_matching.sol new file mode 100644 index 000000000000..9be3947dc14c --- /dev/null +++ b/test/libsolidity/ethdebugTests/isoltestTesting/object_matching.sol @@ -0,0 +1,36 @@ +contract C { + function f() public pure returns (uint) { + return 42; + } +} +// ---- +// C.contract: +// { +// "definition": { +// "source": { +// "id": 0 +// } +// }, +// "name": "C" +// } +// C.creation.instructions[0]: +// { +// "context": { +// "code": { +// "range": { +// "length": 85, +// "offset": 59 +// }, +// "source": { +// "id": 0 +// } +// } +// }, +// "offset": 0, +// "operation": { +// "arguments": [ +// "0x80" +// ], +// "mnemonic": "PUSH1" +// } +// } diff --git a/test/libsolidity/ethdebugTests/isoltestTesting/paren_path_syntax.sol b/test/libsolidity/ethdebugTests/isoltestTesting/paren_path_syntax.sol new file mode 100644 index 000000000000..8ffc50a2f07e --- /dev/null +++ b/test/libsolidity/ethdebugTests/isoltestTesting/paren_path_syntax.sol @@ -0,0 +1,8 @@ +contract C { + function f() public pure {} +} +// ---- +// (compilation) compiler.name: solc +// (C contract) name: C +// (C creation) instructions[0].offset: 0 +// (C runtime) instructions | type: array diff --git a/test/libsolidity/ethdebugTests/isoltestTesting/resources_only.sol b/test/libsolidity/ethdebugTests/isoltestTesting/resources_only.sol new file mode 100644 index 000000000000..939a336f2d0b --- /dev/null +++ b/test/libsolidity/ethdebugTests/isoltestTesting/resources_only.sol @@ -0,0 +1,8 @@ +contract C { + function f() public {} +} +// ---- +// .compilation.compiler.name: solc +// .compilation.compiler.version: +// .resources.compilation.sources | length: 1 +// .resources.compilation.sources[0].id: 0 diff --git a/test/libsolidity/ethdebugTests/isoltestTesting/value_syntax_forms.sol b/test/libsolidity/ethdebugTests/isoltestTesting/value_syntax_forms.sol new file mode 100644 index 000000000000..bc434474ed73 --- /dev/null +++ b/test/libsolidity/ethdebugTests/isoltestTesting/value_syntax_forms.sol @@ -0,0 +1,13 @@ +contract C { + uint x; + function f() public { x = 1; } +} +// ---- +// .compilation.compiler.name: solc +// .compilation.compiler.name: "solc" +// C.contract.name: C +// C.contract.name: "C" +// C.creation.instructions[0].operation.mnemonic: PUSH1 +// C.creation.instructions[0].operation.mnemonic: "PUSH1" +// C.creation.instructions[0].operation.arguments[0]: 0x80 +// C.creation.instructions[0].operation.arguments[0]: "0x80" diff --git a/test/libsolidity/ethdebugTests/multi_source.sol b/test/libsolidity/ethdebugTests/multi_source.sol new file mode 100644 index 000000000000..1ba4970e8424 --- /dev/null +++ b/test/libsolidity/ethdebugTests/multi_source.sol @@ -0,0 +1,15 @@ +==== Source: lib.sol ==== +contract Lib { + function helper() public pure returns (uint) { return 1; } +} +==== Source: main.sol ==== +import "lib.sol"; +contract Main { + function f() public pure returns (uint) { return 42; } +} +// ---- +// .resources.compilation.sources | length: 2 +// Lib.contract.name: Lib +// Main.contract.name: Main +// Lib.creation.environment: create +// Main.creation.environment: create diff --git a/test/libsolidity/ethdebugTests/multi_source_name_collision.sol b/test/libsolidity/ethdebugTests/multi_source_name_collision.sol new file mode 100644 index 000000000000..11569827be83 --- /dev/null +++ b/test/libsolidity/ethdebugTests/multi_source_name_collision.sol @@ -0,0 +1,19 @@ +==== Source: a.sol ==== +contract C { + function fromA() public pure returns (uint) { return 1; } +} +contract Unique { + function only() public pure returns (uint) { return 99; } +} +==== Source: b.sol ==== +contract C { + function fromB() public pure returns (uint) { return 2; } +} +// ---- +// .resources.compilation.sources | length: 2 +// a.sol:C.contract.name: C +// b.sol:C.contract.name: C +// a.sol:C.creation.environment: create +// b.sol:C.creation.environment: create +// Unique.contract.name: Unique +// Unique.creation.environment: create diff --git a/test/libsolidity/ethdebugTests/via_ssa_cfg.sol b/test/libsolidity/ethdebugTests/via_ssa_cfg.sol new file mode 100644 index 000000000000..2d6ab4e24ab7 --- /dev/null +++ b/test/libsolidity/ethdebugTests/via_ssa_cfg.sol @@ -0,0 +1,8 @@ +contract C {} +// ==== +// compileViaSSACFG: true +// ---- +// .compilation.compiler.name: solc +// C.contract.name: C +// C.creation.environment: create +// C.runtime.environment: call diff --git a/test/libsolidity/ethdebugTests/via_ssa_cfg_with_function.sol b/test/libsolidity/ethdebugTests/via_ssa_cfg_with_function.sol new file mode 100644 index 000000000000..9dd9fed9037b --- /dev/null +++ b/test/libsolidity/ethdebugTests/via_ssa_cfg_with_function.sol @@ -0,0 +1,13 @@ +contract C { + function f() public {} +} +// ==== +// compileViaSSACFG: true +// ---- +// .compilation.compiler.name: solc +// .resources.compilation.sources | length: 1 +// C.contract.name: C +// C.creation.environment: create +// C.runtime.environment: call +// C.creation.instructions | type: array +// C.runtime.instructions | type: array diff --git a/test/tools/CMakeLists.txt b/test/tools/CMakeLists.txt index e01c644d8c9c..419de4e08147 100644 --- a/test/tools/CMakeLists.txt +++ b/test/tools/CMakeLists.txt @@ -25,6 +25,7 @@ add_executable(isoltest ../libsolidity/util/ContractABIUtils.cpp ../libsolidity/util/TestFileParser.cpp ../libsolidity/util/TestFunctionCall.cpp + ../libsolidity/EthdebugTest.cpp ../libsolidity/GasTest.cpp ../libsolidity/MemoryGuardTest.cpp ../libsolidity/NatspecJSONTest.cpp @@ -39,6 +40,7 @@ add_executable(isoltest ../libsolidity/ASTJSONTest.cpp ../libsolidity/ASTPropertyTest.cpp ../libsolidity/FunctionDependencyGraphTest.cpp + ../libsolidity/JSONExpectationTest.cpp ../libsolidity/SMTCheckerTest.cpp ../libyul/Common.cpp ../libyul/ControlFlowGraphTest.cpp