Deduce templates using a symboldatabase loop

Author: Your Name

Makefile

@@ -669,7 +669,7 @@ $(libcppdir)/summaries.o: lib/summaries.cpp lib/analyzerinfo.h lib/checkers.h li
 $(libcppdir)/suppressions.o: lib/suppressions.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/checkers.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/filesettings.h lib/library.h lib/mathlib.h lib/path.h lib/pathmatch.h lib/platform.h lib/settings.h lib/smallvector.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/vfvalue.h lib/xml.h
 	$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/suppressions.cpp
 
-$(libcppdir)/templatesimplifier.o: lib/templatesimplifier.cpp lib/checkers.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/standards.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
+$(libcppdir)/templatesimplifier.o: lib/templatesimplifier.cpp lib/astutils.h lib/checkers.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
 	$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/templatesimplifier.cpp
 
 $(libcppdir)/timer.o: lib/timer.cpp lib/config.h lib/timer.h

lib/symboldatabase.cpp

@@ -1947,6 +1947,7 @@ SymbolDatabase::~SymbolDatabase()
         tok->variable(nullptr);
         tok->enumerator(nullptr);
         tok->setValueType(nullptr);
+        tok->exprId(0);
     }
 }
 

lib/templatesimplifier.cpp

@@ -317,6 +317,31 @@ class CPPCHECKLIB TemplateSimplifier {
      */
     void simplifyTemplates(std::time_t maxtime);
 
+    /**
+     * Simplify templates again, using type information (AST, SymbolDatabase, ValueType)
+     * to deduce the template arguments of function template calls such as "f(x)",
+     * "f(x+y)" or "f(x.g())" from the types of the argument expressions.
+     * @param maxtime time when the simplification should be stopped
+     * @return true if the token list was changed
+     */
+    bool simplifyTemplatesUsingTypeInformation(std::time_t maxtime);
+
+    /**
+     * @return true when there are function template calls where the template arguments
+     * could not be deduced yet but deduction may succeed once type information is
+     * available (see simplifyTemplatesUsingTypeInformation).
+     */
+    bool hasPendingTypeDeductions() const {
+        return mPendingTypeDeductions;
+    }
+
+    /**
+     * Remove the instantiated function template declarations whose removal was deferred
+     * while type deductions were pending.
+     * @return true if any tokens were removed
+     */
+    bool removeDeferredTemplateDeclarations();
+
     /**
      * Simplify constant calculations such as "1+2" => "3"
      * @param tok start token
@@ -353,6 +378,29 @@ class CPPCHECKLIB TemplateSimplifier {
      */
     void addInstantiation(Token *token, const std::string &scope);
 
+    /**
+     * Deduce the template arguments of a function template call from the function
+     * arguments and insert them after the name token: "f ( 1 )" => "f < int > ( 1 )".
+     * Literal arguments are always handled; arbitrary argument expressions are handled
+     * when type information is available (mUseTypeInformation). Sets
+     * mPendingTypeDeductions when a deduction could succeed later with type information.
+     * @param tok name token of the function call
+     * @param qualification qualification of the call ("A :: B" for "A :: B :: f ( 1 )")
+     * @param scopeName name of the scope the call is in
+     * @param functionNameMap map with all function template declarations
+     */
+    void deduceFunctionTemplateArguments(Token* tok,
+                                         std::string& qualification,
+                                         const std::string& scopeName,
+                                         const std::multimap<std::string, const TokenAndName*>& functionNameMap);
+
+    /**
+     * Remember that the given instantiated function template declaration should not be
+     * removed from the token list yet because pending type deductions may instantiate
+     * it again. It is removed later by removeDeferredTemplateDeclarations().
+     */
+    void deferRemoval(Token* declTok);
+
     /**
      * Get template instantiations
      */
@@ -509,6 +557,17 @@ class CPPCHECKLIB TemplateSimplifier {
     const Settings &mSettings;
     ErrorLogger &mErrorLogger;
     bool mChanged{};
+    /** true when type information (AST, SymbolDatabase, ValueType) is available for deduction */
+    bool mUseTypeInformation{};
+    /** true when there are calls where deduction may succeed once type information is available */
+    bool mPendingTypeDeductions{};
+    /** true when the current simplifyTemplatesUsingTypeInformation() call deduced something */
+    bool mTypeDeductionsMade{};
+    /** names of all expanded instantiations; persists between simplifyTemplates() calls so a
+     * later deduced instantiation reuses the existing expansion instead of duplicating it */
+    std::set<std::string> mExpandedTemplateNames;
+    /** instantiated function template declarations whose removal has been deferred */
+    std::vector<Token*> mDeferredRemovals;
 
     std::list<TokenAndName> mTemplateDeclarations;
     std::list<TokenAndName> mTemplateForwardDeclarations;

lib/templatesimplifier.h

@@ -1652,6 +1652,14 @@ class CPPCHECKLIB Token {
         mImpl->mValues = nullptr;
     }
 
+    /** Clear the AST links (operands, parent and the cached top token) so the AST can be created again */
+    void clearAst() {
+        mImpl->mAstOperand1 = nullptr;
+        mImpl->mAstOperand2 = nullptr;
+        mImpl->mAstParent = nullptr;
+        mImpl->mAstTop = nullptr;
+    }
+
     // cppcheck-suppress unusedFunction - used in tests only
     std::string astString(const char *sep = "") const {
         std::string ret;

lib/token.h

@@ -3546,6 +3546,12 @@ bool Tokenizer::simplifyTokens1(const std::string &configuration, int fileIndex)
         mSymbolDatabase->setValueTypeInTokenList(true);
     });
 
+    if (isCPP() && mTemplateSimplifier->hasPendingTypeDeductions()) {
+        Timer::run("Tokenizer::simplifyTokens1::simplifyTemplatesUsingTypeInformation", mTimerResults, [&]() {
+            simplifyTemplatesUsingTypeInformation();
+        });
+    }
+
     if (!mSettings.buildDir.empty())
         Summaries::create(*this, configuration, fileIndex);
 
@@ -4264,6 +4270,52 @@ void Tokenizer::simplifyTemplates()
     mTemplateSimplifier->simplifyTemplates(
         maxTime);
 }
+
+void Tokenizer::simplifyTemplatesUsingTypeInformation()
+{
+    if (isC())
+        return;
+
+    const std::time_t maxTime = mSettings.templateMaxTime > 0 ? std::time(nullptr) + mSettings.templateMaxTime : 0;
+    bool finalized = false;
+    constexpr int maxRounds = 5;
+    for (int round = 0; round < maxRounds; ++round) {
+        if (Settings::terminated())
+            return;
+        if (!mTemplateSimplifier->simplifyTemplatesUsingTypeInformation(maxTime))
+            break;
+        // when no deductions are pending anymore the deferred template declarations can
+        // be removed now, so that the rebuilt token information is already final
+        if (!mTemplateSimplifier->hasPendingTypeDeductions()) {
+            mTemplateSimplifier->removeDeferredTemplateDeclarations();
+            finalized = true;
+        }
+        rebuildTokenDataAfterTemplateSimplification();
+        if (finalized)
+            return;
+    }
+    if (!finalized && mTemplateSimplifier->removeDeferredTemplateDeclarations())
+        rebuildTokenDataAfterTemplateSimplification();
+}
+
+void Tokenizer::rebuildTokenDataAfterTemplateSimplification()
+{
+    // update the supporting token information for the added and removed tokens, in the
+    // same order the information was created initially
+    createLinks();
+    setVarId();
+    createLinks2();
+    Token::assignProgressValues(list.front());
+    list.front()->assignIndexes();
+    list.clearAst();
+    list.createAst();
+    list.validateAst(mSettings.debugnormal);
+    delete mSymbolDatabase;
+    mSymbolDatabase = nullptr;
+    createSymbolDatabase();
+    mSymbolDatabase->setValueTypeInTokenList(false);
+    mSymbolDatabase->setValueTypeInTokenList(true);
+}
 //---------------------------------------------------------------------------
 
 

lib/tokenize.cpp

@@ -297,6 +297,21 @@ class CPPCHECKLIB Tokenizer {
      */
     void simplifyTemplates();
 
+    /**
+     * Simplify templates again, this time using type information (AST, SymbolDatabase,
+     * ValueType) to deduce the template arguments of function template calls from the
+     * argument expressions. Runs in a loop: after each round of new instantiations the
+     * supporting token information (varid, links, AST, SymbolDatabase, ValueTypes) is
+     * updated so the new tokens can be used by the next round.
+     */
+    void simplifyTemplatesUsingTypeInformation();
+
+    /**
+     * Update varid, links, AST, SymbolDatabase and ValueTypes after the template
+     * simplifier changed the token list.
+     */
+    void rebuildTokenDataAfterTemplateSimplification();
+
     void simplifyDoublePlusAndDoubleMinus();
 
     void simplifyRedundantConsecutiveBraces();

lib/tokenize.h

@@ -1896,6 +1896,12 @@ static Token * createAstAtToken(Token *tok)
     return tok;
 }
 
+void TokenList::clearAst() const
+{
+    for (Token *tok = mTokensFrontBack->front; tok; tok = tok->next())
+        tok->clearAst();
+}
+
 void TokenList::createAst() const
 {
     for (Token *tok = mTokensFrontBack->front; tok; tok = tok ? tok->next() : nullptr) {

lib/tokenlist.cpp

@@ -172,6 +172,12 @@ class CPPCHECKLIB TokenList {
      */
     void createAst() const;
 
+    /**
+     * Remove the abstract syntax tree from all tokens so createAst() can be called again,
+     * for instance after the token list has been modified.
+     */
+    void clearAst() const;
+
     /**
      * Check abstract syntax tree.
      * @throws InternalError thrown if validation fails

lib/tokenlist.h

@@ -339,7 +339,7 @@ $(libcppdir)/summaries.o: ../lib/summaries.cpp ../lib/analyzerinfo.h ../lib/chec
 $(libcppdir)/suppressions.o: ../lib/suppressions.cpp ../externals/tinyxml2/tinyxml2.h ../lib/addoninfo.h ../lib/checkers.h ../lib/config.h ../lib/errorlogger.h ../lib/errortypes.h ../lib/filesettings.h ../lib/library.h ../lib/mathlib.h ../lib/path.h ../lib/pathmatch.h ../lib/platform.h ../lib/settings.h ../lib/smallvector.h ../lib/standards.h ../lib/suppressions.h ../lib/templatesimplifier.h ../lib/token.h ../lib/tokenlist.h ../lib/utils.h ../lib/vfvalue.h ../lib/xml.h
 	$(CXX) ${LIB_FUZZING_ENGINE} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/suppressions.cpp
 
-$(libcppdir)/templatesimplifier.o: ../lib/templatesimplifier.cpp ../lib/checkers.h ../lib/config.h ../lib/errorlogger.h ../lib/errortypes.h ../lib/library.h ../lib/mathlib.h ../lib/platform.h ../lib/settings.h ../lib/smallvector.h ../lib/standards.h ../lib/templatesimplifier.h ../lib/token.h ../lib/tokenize.h ../lib/tokenlist.h ../lib/utils.h ../lib/vfvalue.h
+$(libcppdir)/templatesimplifier.o: ../lib/templatesimplifier.cpp ../lib/astutils.h ../lib/checkers.h ../lib/config.h ../lib/errorlogger.h ../lib/errortypes.h ../lib/library.h ../lib/mathlib.h ../lib/platform.h ../lib/settings.h ../lib/smallvector.h ../lib/sourcelocation.h ../lib/standards.h ../lib/symboldatabase.h ../lib/templatesimplifier.h ../lib/token.h ../lib/tokenize.h ../lib/tokenlist.h ../lib/utils.h ../lib/vfvalue.h
 	$(CXX) ${LIB_FUZZING_ENGINE} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/templatesimplifier.cpp
 
 $(libcppdir)/timer.o: ../lib/timer.cpp ../lib/config.h ../lib/timer.h