New lexer dev

.gitmodules

@@ -13,3 +13,6 @@
 	path = external/throwing_ptr
 	url = https://github.com/rockdreamer/throwing_ptr
 	ignore = dirty
+[submodule "lexertl17"]
+	path = external/lexertl17
+	url = https://github.com/BenHanson/lexertl17.git

lib/CMakeLists.txt

@@ -11,6 +11,22 @@ else ()
     message(STATUS "libpl static library is being created")
 endif ()
 
+add_subdirectory(lexer_gen)
+
+set(PL_STATICLEXERPATH "${CMAKE_CURRENT_SOURCE_DIR}/include/pl/core/new_lexer_static.hpp")
+
+add_custom_command(
+    OUTPUT ${PL_STATICLEXERPATH}
+    COMMAND $<TARGET_FILE:lexer_gen> ${PL_STATICLEXERPATH}
+    DEPENDS lexer_gen
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMENT "Generating lexertl17 static lexer"
+)
+
+add_custom_target(run_lexer_gen
+    DEPENDS ${PL_STATICLEXERPATH}
+)
+
 add_library(libpl ${LIBRARY_TYPE}
         source/pl/helpers/utils.cpp
 
@@ -53,6 +69,11 @@ add_library(libpl ${LIBRARY_TYPE}
         source/pl/core/token.cpp
         source/pl/core/evaluator.cpp
         source/pl/core/lexer.cpp
+
+        $<$<NOT:$<CONFIG:Debug>>:${PL_STATICLEXERPATH}>
+        $<$<CONFIG:Debug>:source/pl/core/new_lexer_sm.cpp>
+        source/pl/core/new_lexer.cpp
+
         source/pl/core/parser.cpp
         source/pl/core/preprocessor.cpp
         source/pl/core/validator.cpp
@@ -102,6 +123,7 @@ endif ()
 
 target_include_directories(libpl PUBLIC include ../external/throwing_ptr/include)
 target_include_directories(libpl_includes INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include ../external/throwing_ptr/include)
+target_include_directories(libpl PRIVATE include ../external/lexertl17/include)
 target_link_libraries(libpl PRIVATE ${FMT_LIBRARIES})
 target_link_libraries(libpl PUBLIC wolv::types wolv::io wolv::utils wolv::hash wolv::containers)
 

lib/include/pl/core/errors/error.hpp

@@ -130,33 +130,27 @@ namespace pl::core::err {
         std::vector<Location> m_trace;
     };
 
-    class ErrorCollector {
+    class ErrorCollectorExplicitLocation
+    {
     public:
-
-        virtual ~ErrorCollector() = default;
-
-        virtual Location location() = 0;
+        virtual ~ErrorCollectorExplicitLocation() = default;
 
         template <typename... Args>
-        void error(const fmt::format_string<Args...>& fmt, Args&&... args) {
-            this->m_errors.emplace_back(fmt::format(fmt, std::forward<Args>(args)...), location());
+        void error(const Location &location, const fmt::format_string<Args...> &fmt, Args&&... args) {
+            this->m_errors.emplace_back(fmt::format(fmt, std::forward<Args>(args)...), location);
         }
 
-        void error(const std::string &message) {
-            this->m_errors.emplace_back(message, location());
-        }
-
-        void errorDesc(const std::string &message, const std::string &description) {
-            this->m_errors.emplace_back(message, description, location());
+        void errorDesc(const Location &location, const std::string &message, const std::string &description) {
+            this->m_errors.emplace_back(message, description, location);
         }
 
         template<typename... Args>
-        void errorDesc(const fmt::format_string<Args...>& message, const std::string &description, Args&&... args) {
-            this->m_errors.emplace_back(fmt::format(message, std::forward<Args>(args)...), description, location());
+        void errorDesc(const Location &location, const fmt::format_string<Args...>& message, const std::string &description, Args&&... args) {
+            this->m_errors.emplace_back(fmt::format(message, std::forward<Args>(args)...), description, location);
         }
 
-        void error(CompileError& error) {
-            error.getTrace().push_back(location());
+        void error(const Location &location, CompileError& error) {
+            error.getTrace().push_back(location);
             this->m_errors.push_back(std::move(error));
         }
 
@@ -188,8 +182,39 @@ namespace pl::core::err {
         void clear() {
             this->m_errors.clear();
         }
+
     private:
         std::vector<CompileError> m_errors;
     };
 
+    class ErrorCollector : public ErrorCollectorExplicitLocation {
+    public:
+
+        virtual ~ErrorCollector() = default;
+
+        virtual Location location() = 0;
+
+        template <typename... Args>
+        void error(const fmt::format_string<Args...> &fmt, Args&&... args) {
+            this->ErrorCollectorExplicitLocation::error(location(), fmt, std::forward<Args>(args)...);
+        }
+
+        void error(const std::string &message) {
+            this->errorAt(location(), message);
+        }
+
+        void errorDesc(const std::string &message, const std::string &description) {
+            this->ErrorCollectorExplicitLocation::errorDesc(location(), message, description);
+        }
+
+        template<typename... Args>
+        void errorDesc(const fmt::format_string<Args...>& message, const std::string &description, Args&&... args) {
+            this->ErrorCollectorExplicitLocation::errorDesc(location(), message, description, std::forward<Args>(args)...);
+        }
+
+        void error(CompileError& error) {
+            this->ErrorCollectorExplicitLocation::error(location(), error);
+        }
+    };
+
 }

lib/include/pl/core/new_lexer.hpp

@@ -0,0 +1,38 @@
+// "new_lexer.hpp"
+#pragma once
+
+#include <cstddef>
+#include <optional>
+#include <vector>
+#include <pl/core/token.hpp>
+#include <pl/core/errors/error.hpp>
+#include <pl/core/errors/result.hpp>
+// Debugging
+#include <string>
+
+namespace pl::core {
+
+    class New_Lexer : err::ErrorCollectorExplicitLocation {
+    public:
+        New_Lexer();
+
+        void reset();
+
+        hlp::CompileResult<std::vector<Token>> lex(const api::Source *source);
+        size_t getLongestLineLength() const { return m_longestLineLength; }
+
+    private:
+        std::optional<Token::Literal> parseInteger(std::string_view literal, const auto &location);
+        std::optional<double> parseFloatingPoint(std::string_view literal, const char suffix, const auto &location);
+        std::optional<char> parseCharacter(const char* &pchar, const char* e, const auto &location);
+        std::optional<Token> parseStringLiteral(std::string_view literal, const auto &location);
+
+        std::vector<Token> m_tokens;
+        std::size_t m_longestLineLength = 0;
+    };
+
+    // Debugging
+    void compareCompileResults(const hlp::CompileResult<std::vector<Token>> &o, const hlp::CompileResult<std::vector<Token>> &n);
+    void saveCompileResults(const std::string &fn, const hlp::CompileResult<std::vector<Token>> &res);
+
+} // namespace pl::core

lib/include/pl/core/new_lexer_sm.hpp

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <lexertl/state_machine.hpp>
+
+namespace pl::core {
+
+    namespace {
+
+        namespace LexerToken {
+
+            enum {
+                EndOfFile, NewLine, KWNamedOpTypeConstIdent, SingleLineComment,
+                MultiLineCommentOpen,  MultiLineCommentClose, String, Separator,
+                Directive, DirectiveType, DirectiveParam, Operator, Char,
+                Integer, FPNumber
+            };
+
+        }
+    }
+
+    void newLexerBuild(lexertl::state_machine &sm);
+
+}

lib/include/pl/core/token.hpp

@@ -28,7 +28,7 @@ namespace pl::core {
 
         Token() = default;
 
-        enum class Type : u64 {
+        enum class Type /*: u64*/ { // Why so big?
             Keyword,
             ValueType,
             Operator,
@@ -280,6 +280,16 @@ namespace pl::core {
         bool operator==(const ValueTypes &other) const;
         bool operator!=(const ValueTypes &other) const;
 
+        // Debugging
+        friend bool operator==(const Token &l, const Token &r) {
+            return l.type==r.type && l.value==r.value;
+        }
+
+        friend bool operator!=(const Token &l, const Token &r) {
+            return l.type!=r.type || l.value!=r.value;
+        }
+        //
+
         Type type = {};
         ValueTypes value = {};
         Location location = {};

lib/include/pl/pattern_language.hpp

@@ -29,6 +29,7 @@ namespace pl {
     namespace core {
         class Preprocessor;
         class Lexer;
+        class New_Lexer;
         class Parser;
         class Validator;
         class Evaluator;
@@ -62,6 +63,7 @@ namespace pl {
         struct Internals {
             std::unique_ptr<core::Preprocessor> preprocessor;
             std::unique_ptr<core::Lexer>        lexer;
+            std::unique_ptr<core::New_Lexer>    new_lexer;
             std::unique_ptr<core::Parser>       parser;
             std::unique_ptr<core::Validator>    validator;
             std::unique_ptr<core::Evaluator>    evaluator;

lib/lexer_gen/CMakeLists.txt

@@ -0,0 +1,12 @@
+set(SOURCES
+    ../source/pl/core/new_lexer_sm.cpp
+    main.cpp
+)
+
+add_executable(lexer_gen ${SOURCES})
+
+target_include_directories(lexer_gen PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../include
+)
+
+target_include_directories(lexer_gen PRIVATE include ../../external/lexertl17/include)

lib/lexer_gen/main.cpp

@@ -0,0 +1,29 @@
+/*
+Build a "static" lexer in release builds.
+
+Static in the sense that the state machine is built in a pre-build
+step to optimize application start-up time. 
+*/
+#include <pl/core/new_lexer_sm.hpp>
+
+#include <lexertl/state_machine.hpp>
+#include <lexertl/generate_cpp.hpp>
+
+#include <fstream>
+#include <iostream>
+
+int main(int argc, char *argv[])
+{
+    if (argc!=2)
+        return 1;
+
+    lexertl::state_machine sm;
+
+    pl::core::newLexerBuild(sm);
+    sm.minimise();
+
+    std::ofstream ofs(argv[1]);
+    lexertl::table_based_cpp::generate("lookup", sm, false, ofs);
+
+    return 0;
+}