@@ -61,30 +61,36 @@ class SimpleTokenizer : public Tokenizer {
6161 template <size_t size>
6262 bool tokenize (const char (&code)[size])
6363 {
64- std::istringstream istr (code);
65- return tokenize (istr, std::string (list.isCPP () ? " test.cpp" : " test.c" ));
64+ return tokenize (code, size-1 , std::string (list.isCPP () ? " test.cpp" : " test.c" ));
6665 }
6766
6867 bool tokenize (const std::string& code)
6968 {
70- std::istringstream istr (code);
71- return tokenize (istr, std::string (list.isCPP () ? " test.cpp" : " test.c" ));
69+ return tokenize (code.data (), code.size (), std::string (list.isCPP () ? " test.cpp" : " test.c" ));
7270 }
7371
7472private:
7573 /* *
7674 * Tokenize code
77- * @param istr The code as stream
75+ * @param code The code
7876 * @param filename Indicates if the code is C++
7977 * @return false if source code contains syntax errors
8078 */
81- bool tokenize (std::istream& istr,
79+ template <size_t size>
80+ bool tokenize (const char (&code)[size],
81+ const std::string& filename)
82+ {
83+ return tokenize (code, size-1 , filename);
84+ }
85+
86+ bool tokenize (const char * code,
87+ std::size_t size,
8288 const std::string& filename)
8389 {
8490 if (list.front ())
8591 throw std::runtime_error (" token list is not empty" );
8692 list.appendFileIfNew (filename);
87- if (!list.createTokens (istr ))
93+ if (!list.createTokens (code, size ))
8894 return false ;
8995
9096 return simplifyTokens1 (" " );
0 commit comments