diff --git a/lib/clangimport.cpp b/lib/clangimport.cpp index c5962268998..5cf6df26cac 100644 --- a/lib/clangimport.cpp +++ b/lib/clangimport.cpp @@ -677,8 +677,7 @@ void clangimport::AstNode::setValueType(Token *tok) // TODO continue; - TokenList decl(nullptr); - decl.setLang(tok->isCpp() ? Standards::Language::CPP : Standards::Language::C); + TokenList decl(nullptr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C); addTypeTokens(decl, type, tok->scope()); if (!decl.front()) break; diff --git a/lib/cppcheck.cpp b/lib/cppcheck.cpp index 5dd0576db71..e079cf0354d 100644 --- a/lib/cppcheck.cpp +++ b/lib/cppcheck.cpp @@ -715,7 +715,7 @@ unsigned int CppCheck::checkClang(const FileWithDetails &file) } try { - TokenList tokenlist{&mSettings}; + TokenList tokenlist{&mSettings, file.lang()}; tokenlist.appendFileIfNew(file.spath()); Tokenizer tokenizer(std::move(tokenlist), mSettings, mErrorLogger); std::istringstream ast(output2); @@ -910,9 +910,8 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string if (mUnusedFunctionsCheck && (mSettings.useSingleJob() || analyzerInformation)) { std::size_t hash = 0; - TokenList tokenlist{&mSettings}; - // enforce the language since markup files are special and do not adhere to the enforced language - tokenlist.setLang(Standards::Language::C, true); + // markup files are special and do not adhere to the enforced language + TokenList tokenlist{&mSettings, Standards::Language::C}; if (fileStream) { std::vector files; simplecpp::TokenList tokens(*fileStream, files, file.spath()); @@ -1056,10 +1055,9 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string if (startsWith(dir.str,"#define ") || startsWith(dir.str,"#include ")) code += "#line " + std::to_string(dir.linenr) + " \"" + dir.file + "\"\n" + dir.str + '\n'; } - TokenList tokenlist(&mSettings); + TokenList tokenlist(&mSettings, file.lang()); std::istringstream istr2(code); - // TODO: asserts when file has unknown extension - tokenlist.createTokens(istr2, Path::identify(*files.begin(), false)); // TODO: check result? + tokenlist.createTokens(istr2); // TODO: check result? executeRules("define", tokenlist); } #endif @@ -1135,7 +1133,7 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string } try { - TokenList tokenlist{&mSettings}; + TokenList tokenlist{&mSettings, file.lang()}; // Create tokens, skip rest of iteration if failed Timer::run("Tokenizer::createTokens", mSettings.showtime, &s_timerResults, [&]() { diff --git a/lib/importproject.cpp b/lib/importproject.cpp index 25b33697698..fdcf5441db9 100644 --- a/lib/importproject.cpp +++ b/lib/importproject.cpp @@ -577,9 +577,9 @@ namespace { // TODO: improve evaluation const Settings s; - TokenList tokenlist(&s); + TokenList tokenlist(&s, Standards::Language::C); std::istringstream istr(c); - tokenlist.createTokens(istr, Standards::Language::C); // TODO: check result + tokenlist.createTokens(istr); // TODO: check result // TODO: put in a helper // generate links { diff --git a/lib/library.cpp b/lib/library.cpp index 6b2fa6497a8..d802bff118d 100644 --- a/lib/library.cpp +++ b/lib/library.cpp @@ -175,10 +175,10 @@ static std::vector getnames(const char *names) return ret; } -static void gettokenlistfromvalid(const std::string& valid, bool cpp, TokenList& tokenList) +static void gettokenlistfromvalid(const std::string& valid, TokenList& tokenList) { std::istringstream istr(valid + ','); - tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C); // TODO: check result? + tokenList.createTokens(istr); // TODO: check result? for (Token *tok = tokenList.front(); tok; tok = tok->next()) { if (Token::Match(tok,"- %num%")) { tok->str("-" + tok->strAt(1)); @@ -1062,8 +1062,8 @@ bool Library::isIntArgValid(const Token *ftok, int argnr, const MathLib::bigint return true; if (ac->valid.find('.') != std::string::npos) return isFloatArgValid(ftok, argnr, static_cast(argvalue)); - TokenList tokenList(nullptr); - gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList); + TokenList tokenList(nullptr, ftok->isCpp() ? Standards::Language::CPP : Standards::Language::C); + gettokenlistfromvalid(ac->valid, tokenList); for (const Token *tok = tokenList.front(); tok; tok = tok->next()) { if (tok->isNumber() && argvalue == MathLib::toBigNumber(tok)) return true; @@ -1082,8 +1082,8 @@ bool Library::isFloatArgValid(const Token *ftok, int argnr, double argvalue) con const ArgumentChecks *ac = getarg(ftok, argnr); if (!ac || ac->valid.empty()) return true; - TokenList tokenList(nullptr); - gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList); + TokenList tokenList(nullptr, ftok->isCpp() ? Standards::Language::CPP : Standards::Language::C); + gettokenlistfromvalid(ac->valid, tokenList); for (const Token *tok = tokenList.front(); tok; tok = tok->next()) { if (Token::Match(tok, "%num% : %num%") && argvalue >= MathLib::toDoubleNumber(tok) && argvalue <= MathLib::toDoubleNumber(tok->tokAt(2))) return true; diff --git a/lib/programmemory.cpp b/lib/programmemory.cpp index de6c9733062..8bc0b3a20ab 100644 --- a/lib/programmemory.cpp +++ b/lib/programmemory.cpp @@ -1750,11 +1750,11 @@ static std::shared_ptr createTokenFromExpression(const std::string& retur bool cpp, std::unordered_map& lookupVarId) { - std::shared_ptr tokenList = std::make_shared(&settings); + std::shared_ptr tokenList = std::make_shared(&settings, cpp ? Standards::Language::CPP : Standards::Language::C); { const std::string code = "return " + returnValue + ";"; std::istringstream istr(code); - if (!tokenList->createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C)) + if (!tokenList->createTokens(istr)) return nullptr; } diff --git a/lib/symboldatabase.cpp b/lib/symboldatabase.cpp index cda94aaadb7..c7ebc6eb286 100644 --- a/lib/symboldatabase.cpp +++ b/lib/symboldatabase.cpp @@ -1808,8 +1808,7 @@ void SymbolDatabase::setArrayDimensionsUsingValueFlow() // In template arguments, there might not be AST // Determine size by using the "raw tokens" - TokenList tokenList(&mSettings); - tokenList.setLang(dimension.tok->isCpp() ? Standards::Language::CPP : Standards::Language::C); + TokenList tokenList(&mSettings, dimension.tok->isCpp() ? Standards::Language::CPP : Standards::Language::C); tokenList.addtoken(";", 0, 0, 0, false); bool fail = false; for (const Token *tok = dimension.tok; tok && !Token::Match(tok, "[,>]"); tok = tok->next()) { @@ -7293,7 +7292,7 @@ static const Token* parsedecl(const Token* type, else if (Token::simpleMatch(type, "volatile")) valuetype->volatileness |= (1 << (valuetype->pointer - pointer0)); else if (settings.clang && type->str().size() > 2 && type->str().find("::") < type->str().find('<')) { - TokenList typeTokens(&settings); + TokenList typeTokens(&settings, type->isCpp() ? Standards::Language::CPP : Standards::Language::C); std::string::size_type pos1 = 0; do { const std::string::size_type pos2 = type->str().find("::", pos1); @@ -7716,9 +7715,9 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to const std::string& typestr(mSettings.library.returnValueType(tok->previous())); if (!typestr.empty()) { ValueType valuetype; - TokenList tokenList(&mSettings); + TokenList tokenList(&mSettings, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C); std::istringstream istr(typestr+";"); - tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C); // TODO: check result? + tokenList.createTokens(istr); // TODO: check result? tokenList.simplifyStdType(); if (parsedecl(tokenList.front(), &valuetype, mDefaultSignedness, mSettings)) { valuetype.originalTypeName = typestr; @@ -7806,9 +7805,9 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to } continue; } - TokenList tokenList(&mSettings); + TokenList tokenList(&mSettings, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C); std::istringstream istr(typestr+";"); - if (tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C)) { + if (tokenList.createTokens(istr)) { ValueType vt; tokenList.simplifyPlatformTypes(); tokenList.simplifyStdType(); diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp index 38c1d207565..111ceb27d9d 100644 --- a/lib/tokenize.cpp +++ b/lib/tokenize.cpp @@ -10870,7 +10870,7 @@ bool Tokenizer::isPacked(const Token * bodyStart) const void Tokenizer::getErrorMessages(ErrorLogger& errorLogger, const Settings& settings) { - TokenList tokenlist{&settings}; + TokenList tokenlist{&settings, Standards::Language::C}; Tokenizer tokenizer(std::move(tokenlist), settings, errorLogger); tokenizer.invalidConstFunctionTypeError(nullptr); // checkLibraryNoReturn diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp index 8fa2e164c45..588d0c2c8cf 100644 --- a/lib/tokenlist.cpp +++ b/lib/tokenlist.cpp @@ -60,13 +60,12 @@ static constexpr int AST_MAX_DEPTH = 150; -TokenList::TokenList(const Settings* settings) +TokenList::TokenList(const Settings* settings, Standards::Language lang) : mTokensFrontBack(new TokensFrontBack) , mSettings(settings) { - if (mSettings && (mSettings->enforcedLang != Standards::Language::None)) { - mLang = mSettings->enforcedLang; - } + assert(lang != Standards::Language::None); + mLang = lang; } TokenList::~TokenList() @@ -346,15 +345,8 @@ void TokenList::insertTokens(Token *dest, const Token *src, nonneg int n) //--------------------------------------------------------------------------- -bool TokenList::createTokens(std::istream &code, Standards::Language lang) +bool TokenList::createTokens(std::istream &code) { - ASSERT_LANG(lang != Standards::Language::None); - if (mLang == Standards::Language::None) { - mLang = lang; - } else { - ASSERT_LANG(lang == mLang); - } - return createTokensInternal(code, mFiles.empty() ? "" : *mFiles.cbegin()); } @@ -2288,17 +2280,6 @@ bool TokenList::isCPP() const return mLang == Standards::Language::CPP; } -void TokenList::setLang(Standards::Language lang, bool force) -{ - ASSERT_LANG(lang != Standards::Language::None); - if (!force) - { - ASSERT_LANG(mLang == Standards::Language::None); - } - - mLang = lang; -} - const Token * TokenList::isFunctionHead(const Token *tok, const std::string &endsWith) { if (!tok) diff --git a/lib/tokenlist.h b/lib/tokenlist.h index 7f01ca20a75..113a753c9a6 100644 --- a/lib/tokenlist.h +++ b/lib/tokenlist.h @@ -51,7 +51,7 @@ struct TokensFrontBack { class CPPCHECKLIB TokenList { public: // TODO: pass settings as reference - explicit TokenList(const Settings* settings); + explicit TokenList(const Settings* settings, Standards::Language lang); ~TokenList(); TokenList(const TokenList &) = delete; @@ -68,9 +68,6 @@ class CPPCHECKLIB TokenList { /** @return true if the code is C++ */ bool isCPP() const; - // TODO: get rid of this - void setLang(Standards::Language lang, bool force = false); - /** * Delete all tokens in given token list * @param tok token list to delete @@ -103,9 +100,8 @@ class CPPCHECKLIB TokenList { * - UTF in the code are not handled. * - comments are not handled. * @param code input stream for code - * @param lang the language of the code */ - bool createTokens(std::istream &code, Standards::Language lang); + bool createTokens(std::istream &code); void createTokens(simplecpp::TokenList&& tokenList); diff --git a/lib/valueflow.cpp b/lib/valueflow.cpp index 3ba67c5d081..edd86f681ab 100644 --- a/lib/valueflow.cpp +++ b/lib/valueflow.cpp @@ -1943,9 +1943,9 @@ static bool isNotEqual(std::pair x, std::pair x, const std::string& y, bool cpp) { - TokenList tokenList(nullptr); + TokenList tokenList(nullptr, cpp ? Standards::Language::CPP : Standards::Language::C); std::istringstream istr(y); - tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C); // TODO: check result? + tokenList.createTokens(istr); // TODO: check result? return isNotEqual(x, std::make_pair(tokenList.front(), tokenList.back())); } static bool isNotEqual(std::pair x, const ValueType* y, bool cpp) @@ -7051,9 +7051,9 @@ static bool getMinMaxValues(const std::string& typestr, MathLib::bigint& minvalue, MathLib::bigint& maxvalue) { - TokenList typeTokens(&settings); + TokenList typeTokens(&settings, cpp ? Standards::Language::CPP : Standards::Language::C); std::istringstream istr(typestr + ";"); - if (!typeTokens.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C)) + if (!typeTokens.createTokens(istr)) return false; typeTokens.simplifyPlatformTypes(); typeTokens.simplifyStdType(); diff --git a/test/helpers.h b/test/helpers.h index 00ef9262626..2f1f49b4f84 100644 --- a/test/helpers.h +++ b/test/helpers.h @@ -45,21 +45,16 @@ namespace tinyxml2 { class SimpleTokenizer : public Tokenizer { public: explicit SimpleTokenizer(ErrorLogger& errorlogger, bool cpp = true) - : Tokenizer{TokenList{&s_settings}, s_settings, errorlogger} - { - list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true); - } + : Tokenizer{TokenList{&s_settings, cpp ? Standards::Language::CPP : Standards::Language::C}, s_settings, errorlogger} + {} SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, bool cpp = true) - : Tokenizer{TokenList{&settings}, settings, errorlogger} - { - list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true); - } + : Tokenizer{TokenList{&settings, cpp ? Standards::Language::CPP : Standards::Language::C}, settings, errorlogger} + {} SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, const std::string& filename) - : Tokenizer{TokenList{&settings}, settings, errorlogger} + : Tokenizer{TokenList{&settings, Path::identify(filename, false)}, settings, errorlogger} { - list.setLang(Path::identify(filename, false)); list.appendFileIfNew(filename); } @@ -89,7 +84,7 @@ class SimpleTokenizer : public Tokenizer { if (list.front()) throw std::runtime_error("token list is not empty"); list.appendFileIfNew(filename); - if (!list.createTokens(istr, Path::identify(filename, false))) + if (!list.createTokens(istr)) return false; return simplifyTokens1(""); @@ -104,9 +99,10 @@ class SimpleTokenList public: template explicit SimpleTokenList(const char (&code)[size], Standards::Language lang = Standards::Language::CPP) + : list{&settings, lang} { std::istringstream iss(code); - if (!list.createTokens(iss, lang)) + if (!list.createTokens(iss)) throw std::runtime_error("creating tokens failed"); } @@ -120,7 +116,7 @@ class SimpleTokenList private: const Settings settings; - TokenList list{&settings}; + TokenList list; }; @@ -238,14 +234,14 @@ class SimpleTokenizer2 : public Tokenizer { public: template SimpleTokenizer2(const Settings &settings, ErrorLogger &errorlogger, const char (&code)[size], const std::string& file0) - : Tokenizer{TokenList{&settings}, settings, errorlogger} + : Tokenizer{TokenList{&settings, Path::identify(file0, false)}, settings, errorlogger} { preprocess(code, mFiles, file0, *this, errorlogger); } // TODO: get rid of this SimpleTokenizer2(const Settings &settings, ErrorLogger &errorlogger, const char code[], const std::string& file0) - : Tokenizer{TokenList{&settings}, settings, errorlogger} + : Tokenizer{TokenList{&settings, Path::identify(file0, false)}, settings, errorlogger} { preprocess(code, mFiles, file0, *this, errorlogger); } @@ -263,7 +259,7 @@ struct TokenListHelper if (tokenlist.front()) throw std::runtime_error("token list is not empty"); tokenlist.appendFileIfNew(file); - return tokenlist.createTokens(istr, Path::identify(file, false)); + return tokenlist.createTokens(istr); } }; diff --git a/test/testclangimport.cpp b/test/testclangimport.cpp index c2434367cc2..1498b029605 100644 --- a/test/testclangimport.cpp +++ b/test/testclangimport.cpp @@ -140,7 +140,7 @@ class TestClangImport : public TestFixture { std::string parse(const char clang[]) { const Settings settings = settingsBuilder().clang().build(); - TokenList tokenlist{&settings}; + TokenList tokenlist{&settings, Standards::Language::CPP}; Tokenizer tokenizer(std::move(tokenlist), settings, *this); std::istringstream istr(clang); clangimport::parseClangAstDump(tokenizer, istr); @@ -1061,7 +1061,7 @@ class TestClangImport : public TestFixture { #define GET_SYMBOL_DB(AST) \ const Settings settings = settingsBuilder().clang().platform(Platform::Type::Unix64).build(); \ - TokenList tokenlist{&settings}; \ + TokenList tokenlist{&settings, Standards::Language::CPP}; \ Tokenizer tokenizer(std::move(tokenlist), settings, *this); \ { \ std::istringstream istr(AST); \ diff --git a/test/testlibrary.cpp b/test/testlibrary.cpp index b5d0866a386..984ebde24ed 100644 --- a/test/testlibrary.cpp +++ b/test/testlibrary.cpp @@ -151,9 +151,9 @@ class TestLibrary : public TestFixture { " \n" ""; - TokenList tokenList(&settingsDefault); + TokenList tokenList(&settingsDefault, Standards::Language::CPP); std::istringstream istr("foo();"); // <- too few arguments, not library function - ASSERT(tokenList.createTokens(istr, Standards::Language::CPP)); + ASSERT(tokenList.createTokens(istr)); Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous()); tokenList.createAst(); @@ -175,18 +175,18 @@ class TestLibrary : public TestFixture { ASSERT(LibraryHelper::loadxmldata(library, xmldata, sizeof(xmldata))); { - TokenList tokenList(&settingsDefault); + TokenList tokenList(&settingsDefault, Standards::Language::CPP); std::istringstream istr("foo();"); // <- too few arguments, not library function - ASSERT(tokenList.createTokens(istr, Standards::Language::CPP)); + ASSERT(tokenList.createTokens(istr)); Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous()); tokenList.createAst(); ASSERT(library.isNotLibraryFunction(tokenList.front())); } { - TokenList tokenList(&settingsDefault); + TokenList tokenList(&settingsDefault, Standards::Language::CPP); std::istringstream istr("foo(a);"); // <- library function - ASSERT(tokenList.createTokens(istr, Standards::Language::CPP)); + ASSERT(tokenList.createTokens(istr)); Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous()); tokenList.createAst(); @@ -195,9 +195,9 @@ class TestLibrary : public TestFixture { ASSERT(func); } { - TokenList tokenList(&settingsDefault); + TokenList tokenList(&settingsDefault, Standards::Language::CPP); std::istringstream istr("foo(a, b);"); // <- library function - ASSERT(tokenList.createTokens(istr, Standards::Language::CPP)); + ASSERT(tokenList.createTokens(istr)); Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous()); tokenList.createAst(); @@ -206,9 +206,9 @@ class TestLibrary : public TestFixture { ASSERT(func); } { - TokenList tokenList(&settingsDefault); + TokenList tokenList(&settingsDefault, Standards::Language::CPP); std::istringstream istr("foo(a, b, c);"); // <- too much arguments, not library function - ASSERT(tokenList.createTokens(istr, Standards::Language::CPP)); + ASSERT(tokenList.createTokens(istr)); Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous()); tokenList.createAst(); diff --git a/test/testmathlib.cpp b/test/testmathlib.cpp index 31a370b83f6..a3311a56669 100644 --- a/test/testmathlib.cpp +++ b/test/testmathlib.cpp @@ -410,7 +410,7 @@ class TestMathLib : public TestFixture { ASSERT_THROW_INTERNAL_EQUALS(MathLib::toBigNumber("1 invalid"), INTERNAL, "Internal Error. MathLib::toBigNumber: input was not completely consumed: 1 invalid"); { - TokenList list{&settingsDefault}; + TokenList list{&settingsDefault, Standards::Language::C}; list.appendFileIfNew("test.c"); auto tokensFrontBack = std::make_shared(); auto *tok = new Token(list, std::move(tokensFrontBack)); @@ -586,7 +586,7 @@ class TestMathLib : public TestFixture { ASSERT_THROW_INTERNAL_EQUALS(MathLib::toBigUNumber("1 invalid"), INTERNAL, "Internal Error. MathLib::toBigUNumber: input was not completely consumed: 1 invalid"); { - TokenList list{&settingsDefault}; + TokenList list{&settingsDefault, Standards::Language::C}; list.appendFileIfNew("test.c"); auto tokensFrontBack = std::make_shared(); auto *tok = new Token(list, std::move(tokensFrontBack)); @@ -716,7 +716,7 @@ class TestMathLib : public TestFixture { //ASSERT_THROW_INTERNAL_EQUALS(MathLib::toDoubleNumber("1.0LL"), INTERNAL, "Internal Error. MathLib::toDoubleNumber: input was not completely consumed: 1.0LL"); { - TokenList list{&settingsDefault}; + TokenList list{&settingsDefault, Standards::Language::C}; list.appendFileIfNew("test.c"); auto tokensFrontBack = std::make_shared(); auto *tok = new Token(list, std::move(tokensFrontBack)); diff --git a/test/testpreprocessor.cpp b/test/testpreprocessor.cpp index 73dbff0c4aa..04fd7923468 100644 --- a/test/testpreprocessor.cpp +++ b/test/testpreprocessor.cpp @@ -2581,7 +2581,7 @@ class TestPreprocessor : public TestFixture { { dui.std = "c89"; std::vector files; - TokenList tokenlist{&settingsDefault}; + TokenList tokenlist{&settingsDefault, Standards::Language::CPP}; preprocess(code, files, "test.cpp", tokenlist, dui); ASSERT(tokenlist.front()); } @@ -2589,7 +2589,7 @@ class TestPreprocessor : public TestFixture { { dui.std = "gnu23"; std::vector files; - TokenList tokenlist{&settingsDefault}; + TokenList tokenlist{&settingsDefault, Standards::Language::CPP}; preprocess(code, files, "test.cpp", tokenlist, dui); ASSERT(tokenlist.front()); } @@ -2597,7 +2597,7 @@ class TestPreprocessor : public TestFixture { { dui.std = "c++98"; std::vector files; - TokenList tokenlist{&settingsDefault}; + TokenList tokenlist{&settingsDefault, Standards::Language::CPP}; preprocess(code, files, "test.cpp", tokenlist, dui); ASSERT(tokenlist.front()); } @@ -2605,7 +2605,7 @@ class TestPreprocessor : public TestFixture { { dui.std = "gnu++26"; std::vector files; - TokenList tokenlist{&settingsDefault}; + TokenList tokenlist{&settingsDefault, Standards::Language::CPP}; preprocess(code, files, "test.cpp", tokenlist, dui); ASSERT(tokenlist.front()); } @@ -2613,7 +2613,7 @@ class TestPreprocessor : public TestFixture { { dui.std = "gnu77"; std::vector files; - TokenList tokenlist{&settingsDefault}; + TokenList tokenlist{&settingsDefault, Standards::Language::CPP}; preprocess(code, files, "test.cpp", tokenlist, dui); ASSERT(!tokenlist.front()); // nothing is tokenized when an unknown standard is provided } diff --git a/test/testsimplifytemplate.cpp b/test/testsimplifytemplate.cpp index 1fd45ce4239..f3ec0219bf6 100644 --- a/test/testsimplifytemplate.cpp +++ b/test/testsimplifytemplate.cpp @@ -5428,10 +5428,10 @@ class TestSimplifyTemplate : public TestFixture { } unsigned int templateParameters(const char code[]) { - TokenList tokenlist{&settings}; + TokenList tokenlist{&settings, Standards::Language::CPP}; std::istringstream istr(code); tokenlist.appendFileIfNew("test.cpp"); - if (!tokenlist.createTokens(istr, Path::identify("test.cpp", false))) + if (!tokenlist.createTokens(istr)) return false; Tokenizer tokenizer(std::move(tokenlist), settings, *this); tokenizer.createLinks(); @@ -5497,11 +5497,11 @@ class TestSimplifyTemplate : public TestFixture { // Helper function to unit test TemplateSimplifier::getTemplateNamePosition int templateNamePositionHelper(const char code[], unsigned offset = 0) { - TokenList tokenlist{&settings}; + TokenList tokenlist{&settings, Standards::Language::CPP}; std::istringstream istr(code); tokenlist.appendFileIfNew("test.cpp"); - if (!tokenlist.createTokens(istr, Path::identify("test.cpp", false))) + if (!tokenlist.createTokens(istr)) return false; Tokenizer tokenizer(std::move(tokenlist), settings, *this); tokenizer.createLinks(); @@ -5570,7 +5570,7 @@ class TestSimplifyTemplate : public TestFixture { // Helper function to unit test TemplateSimplifier::findTemplateDeclarationEnd bool findTemplateDeclarationEndHelper(const char code[], const char pattern[], unsigned offset = 0) { - TokenList tokenlist{&settings}; + TokenList tokenlist{&settings, Standards::Language::CPP}; std::istringstream istr(code); if (!TokenListHelper::createTokens(tokenlist, istr, "test.cpp")) return false; @@ -5600,7 +5600,7 @@ class TestSimplifyTemplate : public TestFixture { // Helper function to unit test TemplateSimplifier::getTemplateParametersInDeclaration bool getTemplateParametersInDeclarationHelper(const char code[], const std::vector & params) { - TokenList tokenlist{&settings}; + TokenList tokenlist{&settings, Standards::Language::CPP}; std::istringstream istr(code); if (!TokenListHelper::createTokens(tokenlist, istr, "test.cpp")) diff --git a/test/testsimplifytypedef.cpp b/test/testsimplifytypedef.cpp index 5f79d19e946..26648f99ac7 100644 --- a/test/testsimplifytypedef.cpp +++ b/test/testsimplifytypedef.cpp @@ -273,9 +273,9 @@ class TestSimplifyTypedef : public TestFixture { } std::string simplifyTypedef(const char code[]) { - TokenList tokenlist{&settings1}; + TokenList tokenlist{&settings1, Standards::Language::CPP}; std::istringstream istr(code); - if (!tokenlist.createTokens(istr, Standards::Language::CPP)) + if (!tokenlist.createTokens(istr)) return ""; Tokenizer tokenizer(std::move(tokenlist), settings1, *this); tokenizer.createLinks(); @@ -307,7 +307,7 @@ class TestSimplifyTypedef : public TestFixture { std::string simplifyTypedefC(const char code[]) { - TokenList tokenlist{&settings1}; + TokenList tokenlist{&settings1, Standards::Language::C}; std::istringstream istr(code); if (!TokenListHelper::createTokens(tokenlist, istr, "file.c")) @@ -324,7 +324,7 @@ class TestSimplifyTypedef : public TestFixture { } std::string dumpTypedefInfo(const char code[]) { - TokenList tokenlist{&settings1}; + TokenList tokenlist{&settings1, Standards::Language::C}; std::istringstream istr(code); if (!TokenListHelper::createTokens(tokenlist, istr, "file.c")) return {}; @@ -4453,7 +4453,7 @@ class TestSimplifyTypedef : public TestFixture { "uint8_t t;" "void test(rFunctionPointer_fp functionPointer);"; - TokenList tokenlist{&settings1}; + TokenList tokenlist{&settings1, Standards::Language::C}; std::istringstream istr(code); ASSERT(TokenListHelper::createTokens(tokenlist, istr, "file.c")); Tokenizer tokenizer(std::move(tokenlist), settings1, *this); @@ -4496,7 +4496,7 @@ class TestSimplifyTypedef : public TestFixture { " MY_INT x = 0;\n" "}"; - TokenList tokenlist{&settings1}; + TokenList tokenlist{&settings1, Standards::Language::C}; std::istringstream istr(code); ASSERT(TokenListHelper::createTokens(tokenlist, istr, "file.c")); Tokenizer tokenizer(std::move(tokenlist), settings1, *this); @@ -4515,7 +4515,7 @@ class TestSimplifyTypedef : public TestFixture { " F x = 0;\n" "}"; - TokenList tokenlist{&settings1}; + TokenList tokenlist{&settings1, Standards::Language::C}; std::istringstream istr(code); ASSERT(TokenListHelper::createTokens(tokenlist, istr, "file.c")); Tokenizer tokenizer(std::move(tokenlist), settings1, *this); diff --git a/test/testtoken.cpp b/test/testtoken.cpp index d7c27168cb6..045b0cbc59b 100644 --- a/test/testtoken.cpp +++ b/test/testtoken.cpp @@ -33,12 +33,10 @@ class TestToken : public TestFixture { public: - TestToken() : TestFixture("TestToken") { - list.setLang(Standards::Language::C); - } + TestToken() : TestFixture("TestToken") {} private: - /*const*/ TokenList list{&settingsDefault}; + const TokenList list{&settingsDefault, Standards::Language::C}; std::vector arithmeticalOps; std::vector logicalOps; @@ -1372,24 +1370,21 @@ class TestToken : public TestFixture { { { const Settings s = settingsBuilder().c(Standards::cstd_t::C89).build(); - TokenList list_c{&s}; - list_c.setLang(Standards::Language::C); + TokenList list_c{&s, Standards::Language::C}; auto tokensFrontBack = std::make_shared(); Token tok(list_c, std::move(tokensFrontBack)); tok.str("alignas"); // not a C89 keyword assert_tok(&tok, Token::Type::eName); } { - TokenList list_c{&settingsDefault}; - list_c.setLang(Standards::Language::C); + TokenList list_c{&settingsDefault, Standards::Language::C}; auto tokensFrontBack = std::make_shared(); Token tok(list_c, std::move(tokensFrontBack)); tok.str("alignas"); // a C23 keyword assert_tok(&tok, Token::Type::eKeyword); } { - TokenList list_c{&settingsDefault}; - list_c.setLang(Standards::Language::C); + TokenList list_c{&settingsDefault, Standards::Language::C}; auto tokensFrontBack = std::make_shared(); Token tok(list_c, std::move(tokensFrontBack)); tok.str("and_eq"); // a C++ keyword @@ -1401,24 +1396,21 @@ class TestToken : public TestFixture { { { const Settings s = settingsBuilder().cpp(Standards::cppstd_t::CPP03).build(); - TokenList list_cpp{&s}; - list_cpp.setLang(Standards::Language::CPP); + TokenList list_cpp{&s, Standards::Language::CPP}; auto tokensFrontBack = std::make_shared(); Token tok(list_cpp, std::move(tokensFrontBack)); tok.str("consteval"); // not a C++03 keyword assert_tok(&tok, Token::Type::eName); } { - TokenList list_cpp{&settingsDefault}; - list_cpp.setLang(Standards::Language::CPP); + TokenList list_cpp{&settingsDefault, Standards::Language::CPP}; auto tokensFrontBack = std::make_shared(); Token tok(list_cpp, std::move(tokensFrontBack)); tok.str("consteval"); // a C++20 keyword assert_tok(&tok, Token::Type::eKeyword); } { - TokenList list_cpp{&settingsDefault}; - list_cpp.setLang(Standards::Language::CPP); + TokenList list_cpp{&settingsDefault, Standards::Language::CPP}; auto tokensFrontBack = std::make_shared(); Token tok(list_cpp, std::move(tokensFrontBack)); tok.str("typeof_unqual"); // a C keyword @@ -1477,16 +1469,14 @@ class TestToken : public TestFixture { void update_property_info_etype_c() const { { - TokenList list_c{&settingsDefault}; - list_c.setLang(Standards::Language::C); + TokenList list_c{&settingsDefault, Standards::Language::C}; auto tokensFrontBack = std::make_shared(); Token tok(list_c, std::move(tokensFrontBack)); tok.str("char"); // not treated as keyword in TokenList::isKeyword() assert_tok(&tok, Token::Type::eType, /*l=*/ false, /*std=*/ true); } { - TokenList list_c{&settingsDefault}; - list_c.setLang(Standards::Language::C); + TokenList list_c{&settingsDefault, Standards::Language::C}; auto tokensFrontBack = std::make_shared(); Token tok(list_c, std::move(tokensFrontBack)); tok.str("size_t"); // not treated as keyword in TokenList::isKeyword() @@ -1497,16 +1487,14 @@ class TestToken : public TestFixture { void update_property_info_etype_cpp() const { { - TokenList list_cpp{&settingsDefault}; - list_cpp.setLang(Standards::Language::CPP); + TokenList list_cpp{&settingsDefault, Standards::Language::CPP}; auto tokensFrontBack = std::make_shared(); Token tok(list_cpp, std::move(tokensFrontBack)); tok.str("bool"); // not treated as keyword in TokenList::isKeyword() assert_tok(&tok, Token::Type::eType, /*l=*/ false, /*std=*/ true); } { - TokenList list_cpp{&settingsDefault}; - list_cpp.setLang(Standards::Language::CPP); + TokenList list_cpp{&settingsDefault, Standards::Language::CPP}; auto tokensFrontBack = std::make_shared(); Token tok(list_cpp, std::move(tokensFrontBack)); tok.str("size_t"); @@ -1526,8 +1514,7 @@ class TestToken : public TestFixture { void varid_reset() const { - TokenList list_c{&settingsDefault}; - list_c.setLang(Standards::Language::C); + TokenList list_c{&settingsDefault, Standards::Language::C}; auto tokensFrontBack = std::make_shared(); Token tok(list_c, std::move(tokensFrontBack)); tok.str("int"); // not treated as keyword in TokenList::isKeyword() diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index 2870feb4cae..866f7a4087f 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -557,7 +557,7 @@ class TestTokenizer : public TestFixture { Preprocessor preprocessor(settings, *this, Path::identify(tokens1.getFiles()[0], false)); std::list directives = preprocessor.createDirectives(tokens1); - TokenList tokenlist{&settings}; + TokenList tokenlist{&settings, Path::identify(filename, false)}; Tokenizer tokenizer(std::move(tokenlist), settings, *this); tokenizer.setDirectives(std::move(directives)); @@ -865,11 +865,11 @@ class TestTokenizer : public TestFixture { ASSERT_THROW_INTERNAL(tokenizeAndStringify(";template class X { };",false,Platform::Type::Native,false), SYNTAX); ASSERT_THROW_INTERNAL(tokenizeAndStringify("int X() {};",false,Platform::Type::Native,false), SYNTAX); { - TokenList tokenlist{&settings1}; + TokenList tokenlist{&settings1, Standards::Language::C}; // headers are treated as C files const char code[] = "void foo(int i) { reinterpret_cast(i) };"; std::istringstream istr(code); tokenlist.appendFileIfNew("test.h"); - ASSERT(tokenlist.createTokens(istr, Path::identify("test.h", false))); + ASSERT(tokenlist.createTokens(istr)); Tokenizer tokenizer(std::move(tokenlist), settings1, *this); ASSERT_THROW_INTERNAL(tokenizer.simplifyTokens1(""), SYNTAX); } @@ -3688,7 +3688,7 @@ class TestTokenizer : public TestFixture { } void simplifyString() { - TokenList tokenlist{&settings0}; + TokenList tokenlist{&settings0, Standards::Language::CPP}; Tokenizer tokenizer(std::move(tokenlist), settings0, *this); ASSERT_EQUALS("\"abc\"", tokenizer.simplifyString("\"abc\"")); ASSERT_EQUALS("\"\n\"", tokenizer.simplifyString("\"\\xa\"")); @@ -6137,10 +6137,10 @@ class TestTokenizer : public TestFixture { std::string testAst(const char code[], AstStyle style = AstStyle::Simple) { // tokenize given code.. - TokenList tokenlist{&settings0}; + TokenList tokenlist{&settings0, Standards::Language::CPP}; std::istringstream istr(code); tokenlist.appendFileIfNew("test.cpp"); - if (!tokenlist.createTokens(istr,Path::identify("test.cpp", false))) + if (!tokenlist.createTokens(istr)) return "ERROR"; Tokenizer tokenizer(std::move(tokenlist), settings0, *this); diff --git a/test/testtokenlist.cpp b/test/testtokenlist.cpp index 8c86bb3db31..e7fe97474fd 100644 --- a/test/testtokenlist.cpp +++ b/test/testtokenlist.cpp @@ -55,7 +55,7 @@ class TestTokenList : public TestFixture { // inspired by #5895 void testaddtoken1() const { const std::string code = "0x89504e470d0a1a0a"; - TokenList tokenlist(&settings); + TokenList tokenlist(&settings, Standards::Language::CPP); tokenlist.addtoken(code, 1, 1, false); ASSERT_EQUALS("0x89504e470d0a1a0a", tokenlist.front()->str()); } @@ -64,7 +64,7 @@ class TestTokenList : public TestFixture { const std::string code = "0xF0000000"; /*const*/ Settings settings1 = settings; settings1.platform.int_bit = 32; - TokenList tokenlist(&settings1); + TokenList tokenlist(&settings1, Standards::Language::CPP); tokenlist.addtoken(code, 1, 1, false); ASSERT_EQUALS("0xF0000000", tokenlist.front()->str()); } @@ -128,10 +128,10 @@ class TestTokenList : public TestFixture { { const char code2[] = "_Generic"; // C11 keyword const Settings s = settingsBuilder().c(Standards::C89).build(); - TokenList tokenlist(&s); + TokenList tokenlist(&s, Standards::Language::C); std::istringstream istr(code2); tokenlist.appendFileIfNew("a.c"); - ASSERT(tokenlist.createTokens(istr, Path::identify("a.c", false))); + ASSERT(tokenlist.createTokens(istr)); ASSERT_EQUALS(false, tokenlist.front()->isKeyword()); } @@ -150,10 +150,10 @@ class TestTokenList : public TestFixture { { const char code2[] = "noexcept"; // C++11 keyword const Settings s = settingsBuilder().cpp(Standards::CPP03).build(); - TokenList tokenlist(&s); + TokenList tokenlist(&s, Standards::Language::CPP); std::istringstream istr(code2); tokenlist.appendFileIfNew("a.cpp"); - ASSERT(tokenlist.createTokens(istr, Path::identify("a.cpp", false))); + ASSERT(tokenlist.createTokens(istr)); ASSERT_EQUALS(false, tokenlist.front()->isKeyword()); } } @@ -167,16 +167,16 @@ class TestTokenList : public TestFixture { simplecpp::TokenList tokens1(istr, files, "poll.h", nullptr); Preprocessor preprocessor(settingsDefault, *this, Path::identify(tokens1.getFiles()[0], false)); simplecpp::TokenList tokensP = preprocessor.preprocess(tokens1, "", files, true); - TokenList tokenlist(&settingsDefault); + TokenList tokenlist(&settingsDefault, Standards::Language::C); // headers are treated as C files tokenlist.createTokens(std::move(tokensP)); // do not assert } void ast1() const { const std::string s = "('Release|x64' == 'Release|x64');"; - TokenList tokenlist(&settings); + TokenList tokenlist(&settings, Standards::Language::C); std::istringstream istr(s); - ASSERT(tokenlist.createTokens(istr, Standards::Language::C)); + ASSERT(tokenlist.createTokens(istr)); // TODO: put this logic in TokenList // generate links {