Merge pull request #16229 from argotorg/introduce-experimental-flag

Introduce experimental flag/option

Author: nikola-matic

Changelog.md

@@ -3,6 +3,10 @@
 Language Features:
 
 Compiler Features:
+* Commandline Interface: Introduce `--experimental` flag required for enabling the experimental mode.
+* General: Restrict the existing experimental features (`generic-solidity`, `lsp`, `ethdebug`, `eof`, `evm`, `ast-import`, `evmasm-import`, `ir-ast`, `ssa-cfg`) to experimental mode.
+* Metadata: Store the state of the experimental mode in JSON and CBOR metadata. In CBOR this broadens the meaning of the existing `experimental` field, which used to indicate only the presence of certain experimental pragmas in the source.
+* Standard JSON Interface: Introduce `settings.experimental` setting required for enabling the experimental mode.
 
 Bugfixes:
 

docs/metadata.rst

@@ -136,6 +136,12 @@ explanatory purposes.
         "compilationTarget": {
           "myDirectory/myFile.sol": "MyContract"
         },
+        // Optional (false if omitted): Indicates whether experimental mode has been enabled.
+        // Always matches the value of the `experimental` flag in CBOR metadata.
+        // Note that experimental mode being enabled does not necessarily mean that any
+        // experimental features were actually used, or if they were, that those features
+        // affected the bytecode.
+        "experimental": true,
         // Required for Solidity: Addresses for libraries used.
         // Note that metadata has a different format for "libraries" field than the standard JSON input.
         // metadata format = { "MyLib.sol:MyLib": "0x123123..." }
@@ -197,12 +203,14 @@ Below are all the possible fields:
 .. code-block:: javascript
 
     {
+      // Present if "bytecodeHash" was "ipfs" in compiler settings
       "ipfs": "<metadata hash>",
-      // If "bytecodeHash" was "bzzr1" in compiler settings not "ipfs" but "bzzr1"
+      // Present if "bytecodeHash" was "bzzr1" in compiler settings
       "bzzr1": "<metadata hash>",
       // Previous versions were using "bzzr0" instead of "bzzr1"
       "bzzr0": "<metadata hash>",
-      // If any experimental features that affect code generation are used
+      // Present if experimental mode has been enabled either via "--experimental" flag or
+      // "settings.experimental" option in Standard JSON
       "experimental": true,
       "solc": "<compiler version>"
     }

docs/using-the-compiler.rst

@@ -281,6 +281,10 @@ Input Description
         "stopAfter": "parsing",
         // Optional: List of remappings
         "remappings": [ ":g=/dir" ],
+        // Optional: Experimental mode toggle (Default: false)
+        // Makes it possible to use experimental features (but does not enable any such feature by itself).
+        // The use of this mode is recorded in contract metadata.
+        "experimental": true,
         // Optional: Optimizer settings
         "optimizer": {
           // Turn on the optimizer. Optional. Default: false.
@@ -383,8 +387,11 @@ Input Description
           //     - `<end>` is the index of the first byte after that location.
           // - `snippet`: A single-line code snippet from the location indicated by `@src`.
           //     The snippet is quoted and follows the corresponding `@src` annotation.
-          // - `*`: Wildcard value that can be used to request everything.
-          "debugInfo": ["location", "snippet"]
+          // - `ast-id`: Annotations of the form `@ast-id <id>` over elements that can be mapped back to a definition in the original Solidity file.
+          //   `<id>` is a node ID in the Solidity AST ('ast' output).
+          // - `ethdebug`: Ethdebug annotations (experimental).
+          // - `*`: Wildcard value that can be used to request all non-experimental components.
+          "debugInfo": ["location", "snippet", "ast-id", "ethdebug"]
         },
         // Metadata settings (optional)
         "metadata": {
@@ -435,13 +442,15 @@ Input Description
         //   userdoc - User documentation (natspec)
         //   metadata - Metadata
         //   ir - Yul intermediate representation of the code before optimization
-        //   irAst - AST of Yul intermediate representation of the code before optimization
+        //   irAst - AST of Yul intermediate representation of the code before optimization (experimental)
         //   irOptimized - Intermediate representation after optimization
-        //   irOptimizedAst - AST of intermediate representation after optimization
-        //   storageLayout - Slots, offsets and types of the contract's state variables in storage.
-        //   transientStorageLayout - Slots, offsets and types of the contract's state variables in transient storage.
+        //   irOptimizedAst - AST of intermediate representation after optimization (experimental)
+        //   storageLayout - Slots, offsets and types of the contract's state variables in storage
+        //   transientStorageLayout - Slots, offsets and types of the contract's state variables in transient storage
         //   evm.assembly - New assembly format
         //   evm.legacyAssembly - Old-style assembly format in JSON
+        //   evm.bytecode.ethdebug - Debug information in ethdebug format (ethdebug/format/program schema). Can only be requested when compiling via IR. (experimental)
+        //   evm.deployedBytecode.ethdebug - Like evm.bytecode.ethdebug, but for the runtime part of the contract (experimental)
         //   evm.bytecode.functionDebugData - Debugging information at function level
         //   evm.bytecode.object - Bytecode object
         //   evm.bytecode.opcodes - Opcodes list
@@ -452,6 +461,7 @@ Input Description
         //   evm.deployedBytecode.immutableReferences - Map from AST ids to bytecode ranges that reference immutables
         //   evm.methodIdentifiers - The list of function hashes
         //   evm.gasEstimates - Function gas estimates
+        //   yulCFGJson - Control Flow Graph (CFG) of the Single Static Assignment (SSA) form of the contract (experimental)
         //
         // Note that using `evm`, `evm.bytecode`, etc. will select every
         // target part of that output. Additionally, `*` can be used as a wildcard to request everything.
@@ -604,6 +614,8 @@ Output Description
               "legacyAssembly": {},
               // Bytecode and related details.
               "bytecode": {
+                // Ethdebug output (experimental)
+                "ethdebug": {/* ... */},
                 // Debugging data at the level of functions.
                 "functionDebugData": {
                   // Now follows a set of functions including compiler-internal and
@@ -646,6 +658,8 @@ Output Description
                 }
               },
               "deployedBytecode": {
+                // Ethdebug output (experimental)
+                "ethdebug": {/* ... */},
                 /* ..., */ // The same layout as above.
                 "immutableReferences": {
                   // There are two references to the immutable with AST ID 3, both 32 bytes long. One is
@@ -670,11 +684,15 @@ Output Description
                 "internal": {
                   "heavyLifting()": "infinite"
                 }
-              }
+              },
+              // Yul CFG representation of the SSA form (experimental)
+              "yulCFGJson": {/* ... */}
             }
           }
         }
-      }
+      },
+      // Global Ethdebug output (experimental)
+      "ethdebug": {/* ... */ }
     }
 
 
@@ -696,3 +714,57 @@ Error Types
 13. ``YulException``: Error during Yul code generation - this should be reported as an issue.
 14. ``Warning``: A warning, which didn't stop the compilation, but should be addressed if possible.
 15. ``Info``: Information that the compiler thinks the user might find useful, but is not dangerous and does not necessarily need to be addressed.
+
+.. index:: ! Experimental mode, ! --experimental
+.. _experimental-mode:
+
+Experimental Mode
+*****************
+
+Some language and compiler features included in stable releases are not themselves considered stable.
+They are sparsely documented, if at all, often not adequately tested, and thus not yet intended for production use.
+In many cases it is possible to develop a big feature incrementally, with each iteration being already stable.
+Sometimes, however, it is preferable to start with a prototype and stabilize it over multiple releases, while receiving feedback from users.
+To prevent accidental use, such features can be only accessed by enabling the experimental mode.
+
+There are no backwards compatibility guarantees for experimental features.
+They are subject to change in breaking ways in non-breaking releases of the compiler.
+Only major changes affecting them are recorded in the changelog.
+
+To enable the experimental mode, use the ``--experimental`` flag on the command line,
+or the analogous ``settings.experimental`` boolean setting in the Standard JSON input.
+
+Note that the use of this mode is recorded in the metadata:
+
+- ``experimental`` flag in CBOR metadata is set to ``true``,
+- ``settings.experimental`` in JSON metadata is set to ``true``,
+
+.. note::
+    Prior to version 0.8.34, most of the experimental features were usable without any extra safeguards.
+    Some were gated behind ``pragma experimental``, but this was not done consistently.
+    The information about them was also only recorded in CBOR metadata and even then not always.
+    The main goal of the experimental mode is to systematize this and make users fully aware when relying on features which are unfinished or not production-ready.
+
+The table below details all currently available experimental features.
+
++-----------------------+--------------------------+------------------+-------------------------------------------------------------------+
+| Feature               | ID                       | Affects bytecode | Flag/pragma                                                       |
++=======================+==========================+==================+===================================================================+
+| AST import            | ``ast-import``           | yes              | ``--import-ast``                                                  |
++-----------------------+--------------------------+------------------+-------------------------------------------------------------------+
+| LSP                   | ``lsp``                  | no               | ``--lsp``                                                         |
++-----------------------+--------------------------+------------------+-------------------------------------------------------------------+
+| EVM Assembly import   | ``evmasm-import``        | yes              | ``--import-asm-json``                                             |
++-----------------------+--------------------------+------------------+-------------------------------------------------------------------+
+| Generic Solidity      | ``generic-solidity``     | yes              | ``pragma experimental solidity``                                  |
++-----------------------+--------------------------+------------------+-------------------------------------------------------------------+
+| IR AST                | ``ir-ast``               | no               | ``--ir-ast-json``, ``--ir-optimized-ast-json``                    |
++-----------------------+--------------------------+------------------+-------------------------------------------------------------------+
+| EOF                   | ``eof``                  | yes              | ``--experimental-eof-version``                                    |
++-----------------------+--------------------------+------------------+-------------------------------------------------------------------+
+| Non-mainnet EVMs      | ``evm``                  | yes              | ``--evm-version <version name>``                                  |
++-----------------------+--------------------------+------------------+-------------------------------------------------------------------+
+| Ethdebug              | ``ethdebug``             | no               | ``--ethdebug``, ``--ethdebug-runtime``, ``--debug-info ethdebug`` |
++-----------------------+--------------------------+------------------+-------------------------------------------------------------------+
+| Yul SSA CFG exporter  | ``ssa-cfg``              | no               | ``--yul-cfg-json``                                                |
++-----------------------+--------------------------+------------------+-------------------------------------------------------------------+

libsolidity/analysis/SyntaxChecker.cpp

@@ -110,7 +110,16 @@ bool SyntaxChecker::visit(PragmaDirective const& _pragma)
 				auto feature = ExperimentalFeatureNames.at(literal);
 				m_sourceUnit->annotation().experimentalFeatures.insert(feature);
 				if (!ExperimentalFeatureWithoutWarning.count(feature))
-					m_errorReporter.warning(2264_error, _pragma.location(), "Experimental features are turned on. Do not use experimental features on live deployments.");
+				{
+					if (!m_experimental)
+						m_errorReporter.syntaxError(
+							2816_error,
+							_pragma.location(),
+							"Experimental pragmas can only be used if experimental mode is enabled. To enable experimental mode, use the --experimental flag."
+						);
+					else
+						m_errorReporter.warning(2264_error, _pragma.location(), "Experimental features are turned on. Do not use experimental features on live deployments.");
+				}
 
 				if (feature == ExperimentalFeature::ABIEncoderV2)
 				{

libsolidity/analysis/SyntaxChecker.h

@@ -45,9 +45,12 @@ class SyntaxChecker: private ASTConstVisitor
 {
 public:
 	/// @param _errorReporter provides the error logging functionality.
-	SyntaxChecker(langutil::ErrorReporter& _errorReporter, bool _useYulOptimizer):
+	/// @param _useYulOptimizer indicates whether Yul optimizer is enabled.
+	/// @param _experimental indicates whether the experimental mode is enabled.
+	SyntaxChecker(langutil::ErrorReporter& _errorReporter, bool _useYulOptimizer, bool _experimental):
 		m_errorReporter(_errorReporter),
-		m_useYulOptimizer(_useYulOptimizer)
+		m_useYulOptimizer(_useYulOptimizer),
+		m_experimental(_experimental)
 	{}
 
 	bool checkSyntax(ASTNode const& _astRoot);
@@ -112,6 +115,9 @@ class SyntaxChecker: private ASTConstVisitor
 	/// Flag that indicates whether we are inside an unchecked block.
 	bool m_uncheckedArithmetic = false;
 
+	/// Flag that indicates whether the experimental mode is enabled.
+	bool m_experimental = false;
+
 	int m_inLoopDepth = 0;
 	std::optional<ContractKind> m_currentContractKind;
 

libsolidity/interface/CompilerStack.cpp

@@ -229,6 +229,12 @@ void CompilerStack::setViaIR(bool _viaIR)
 	m_viaIR = _viaIR;
 }
 
+void CompilerStack::setExperimental(bool _experimental)
+{
+	solAssert(m_stackState < ParsedAndImported, "Must set experimental before parsing.");
+	m_experimental = _experimental;
+}
+
 void CompilerStack::setEVMVersion(langutil::EVMVersion _version)
 {
 	solAssert(m_stackState < ParsedAndImported, "Must set EVM version before parsing.");
@@ -450,6 +456,19 @@ void CompilerStack::importASTs(std::map<std::string, Json> const& _sources)
 	storeContractDefinitions();
 }
 
+namespace
+{
+
+bool onlySafeExperimentalFeaturesActivated(std::set<ExperimentalFeature> const& _features)
+{
+	for (auto const feature: _features)
+		if (!ExperimentalFeatureWithoutWarning.contains(feature))
+			return false;
+	return true;
+}
+
+}
+
 bool CompilerStack::analyze()
 {
 	solAssert(m_stackState == ParsedAndImported, "Must call analyze only after parsing was successful.");
@@ -467,7 +486,7 @@ bool CompilerStack::analyze()
 	{
 		bool experimentalSolidity = isExperimentalSolidity();
 
-		SyntaxChecker syntaxChecker(m_errorReporter, m_optimiserSettings.runYulOptimiser);
+		SyntaxChecker syntaxChecker(m_errorReporter, m_optimiserSettings.runYulOptimiser, m_experimental);
 		for (Source const* source: m_sourceOrder)
 			if (source->ast && !syntaxChecker.checkSyntax(*source->ast))
 				noErrors = false;
@@ -527,6 +546,10 @@ bool CompilerStack::analyze()
 	if (!noErrors)
 		return false;
 
+	for (Source const* source: m_sourceOrder)
+		if (source->ast && !m_experimental)
+			solAssert(onlySafeExperimentalFeaturesActivated(source->ast->annotation().experimentalFeatures));
+
 	m_stackState = AnalysisSuccessful;
 	return true;
 }
@@ -1446,17 +1469,6 @@ void CompilerStack::annotateInternalFunctionIDs()
 	}
 }
 
-namespace
-{
-bool onlySafeExperimentalFeaturesActivated(std::set<ExperimentalFeature> const& features)
-{
-	for (auto const feature: features)
-		if (!ExperimentalFeatureWithoutWarning.count(feature))
-			return false;
-	return true;
-}
-}
-
 void CompilerStack::assembleYul(
 	ContractDefinition const& _contract,
 	std::shared_ptr<evmasm::Assembly> _assembly,
@@ -1816,6 +1828,8 @@ std::string CompilerStack::createMetadata(Contract const& _contract, bool _forIR
 	if (_forIR)
 		meta["settings"]["viaIR"] = _forIR;
 	meta["settings"]["evmVersion"] = m_evmVersion.name();
+	if (m_experimental)
+		meta["settings"]["experimental"] = m_experimental;
 	if (m_eofVersion.has_value())
 		meta["settings"]["eofVersion"] = *m_eofVersion;
 	meta["settings"]["compilationTarget"][_contract.contract->sourceUnitName()] =
@@ -1927,10 +1941,14 @@ bytes CompilerStack::createCBORMetadata(Contract const& _contract, bool _forIR)
 	if (m_metadataFormat == MetadataFormat::NoMetadata)
 		return bytes{};
 
-	bool const experimentalMode = !onlySafeExperimentalFeaturesActivated(
+	bool const usesExperimentalSyntax = !_contract.contract->sourceUnit().annotation().experimentalFeatures.empty();
+	bool const onlySafeExperimentalFeatures = onlySafeExperimentalFeaturesActivated(
 		_contract.contract->sourceUnit().annotation().experimentalFeatures
 	);
 
+	if (m_eofVersion.has_value() || (usesExperimentalSyntax && !onlySafeExperimentalFeatures))
+		solAssert(m_experimental, "Experimental mode not enabled");
+
 	std::string meta = (_forIR == m_viaIR ? metadata(_contract) : createMetadata(_contract, _forIR));
 
 	MetadataCBOREncoder encoder;
@@ -1942,7 +1960,7 @@ bytes CompilerStack::createCBORMetadata(Contract const& _contract, bool _forIR)
 	else
 		solAssert(m_metadataHash == MetadataHash::None, "Invalid metadata hash");
 
-	if (experimentalMode || m_eofVersion.has_value())
+	if (m_experimental)
 		encoder.pushBool("experimental", true);
 	if (m_metadataFormat == MetadataFormat::WithReleaseVersionTag)
 		encoder.pushBytes("solc", VersionCompactBytes);

libsolidity/interface/CompilerStack.h

@@ -223,6 +223,9 @@ class CompilerStack: public langutil::CharStreamProvider, public evmasm::Abstrac
 	/// Must be set before parsing.
 	void setViaIR(bool _viaIR);
 
+	/// Sets the experimental toggle to allow usage of experimental features.
+	void setExperimental(bool _experimental);
+
 	/// Set the EVM version used before running compile.
 	/// When called without an argument it will revert to the default version.
 	/// Must be set before parsing.
@@ -607,6 +610,7 @@ class CompilerStack: public langutil::CharStreamProvider, public evmasm::Abstrac
 	RevertStrings m_revertStrings = RevertStrings::Default;
 	State m_stopAfter = State::CompilationSuccessful;
 	bool m_viaIR = false;
+	bool m_experimental = false;
 	langutil::EVMVersion m_evmVersion;
 	std::optional<uint8_t> m_eofVersion;
 	ModelCheckerSettings m_modelCheckerSettings;