testrunner: avoid dome duplicated tokenizing

Author: firewave

test/helpers.h

@@ -112,6 +112,8 @@ class SimpleTokenizer : public Tokenizer {
                   const std::string& filename,
                   const std::string &configuration = "")
     {
+        if (list.front())
+            throw std::runtime_error("token list is not empty");
         list.appendFileIfNew(filename);
         if (!list.createTokens(istr, Path::identify(filename, false)))
             return false;
@@ -284,6 +286,8 @@ struct TokenListHelper
 {
     static bool createTokens(TokenList& tokenlist, std::istream& istr, const std::string& file)
     {
+        if (tokenlist.front())
+            throw std::runtime_error("token list is not empty");
         tokenlist.appendFileIfNew(file);
         return tokenlist.createTokens(istr, Path::identify(file, false));
     }

test/testsimplifyusing.cpp

@@ -25,7 +25,6 @@
 #include "utils.h"
 
 #include <cstddef>
-#include <sstream>
 #include <string>
 
 class TestSimplifyUsing : public TestFixture {
@@ -113,8 +112,6 @@ class TestSimplifyUsing : public TestFixture {
         if (options.preprocess) {
             SimpleTokenizer2 tokenizer(settings, *this, code, "test.cpp");
 
-            std::istringstream istr(code);
-            ASSERT_LOC(TokenListHelper::createTokens(tokenizer.list, istr, "test.cpp"), file, line); // TODO: this creates the tokens a second time
             ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);
             return tokenizer.tokens()->stringifyList(nullptr);
         }
@@ -1586,7 +1583,7 @@ class TestSimplifyUsing : public TestFixture {
                             "STAMP(B, A);\n"
                             "STAMP(C, B);\n";
         (void)tok(code, dinit(TokOptions, $.preprocess = true));
-        ASSERT(startsWith(errout_str(), "[test.cpp:6]: (debug) Failed to parse 'using C = S < S < S < int"));
+        TODO_ASSERT(startsWith(errout_str(), "[test.cpp:6]: (debug) Failed to parse 'using C = S < S < S < int"));
     }
 
     void scopeInfo1() {