Merge branch 'BadHorizontalScrolling' of https://github.com/paxcut/PatternLanguage into HEAD

Author: paxcut

fuzz/README.md

@@ -29,6 +29,19 @@ afl-fuzz -i inputs -o output -x ./dict/hexpat.dict -- ./plfuzz @@
 ```
 This will run a simple fuzzing session with the inputs in the `inputs` folder and outputting to the `output` folder.
 
+### Minimization
+
+To minimize the found crashes, you can simpy use the `afl-tmin` tool that comes with AFL++.
+Here is an example of how to minimize a crash:
+```bash
+afl-tmin -i output/crashes/<crash_file> -o output/minimized/<crash_file> -- ./plfuzz @@
+```
+
+We also provide a small script to minimize all crashes in the `output/crashes` folder:
+```bash
+python3 afl-pytmin.py output/crashes output/minimized ./plfuzz
+```
+
 ### Debugging
 During the session, if the fuzzer finds crashes or halts, it will output the crashing input to the
 `output/crashes` or `output/hangs` folder.   

fuzz/afl-pytmin.py

@@ -0,0 +1,94 @@
+#!/env/python
+# Adapted from https://github.com/ilsani/afl-pytmin/tree/master
+# ALL RIGHTS RESERVED TO THE ORIGINAL AUTHOR
+#
+# python afl-pytmin.py <input-dir> <output-dir> <harness-bin>
+#
+
+import sys
+import os
+import subprocess
+
+from queue import Queue
+from threading import Thread
+
+AFL_TMIN_CMD = "afl-tmin"
+
+def tmin(workerid, queue, output_dirname, harness_bin):
+
+    while True:
+
+        data = queue.get()
+        filename = data["filename"]
+        counter = data["counter"]
+
+        output_path = "%s/%d" %(output_dirname, counter)
+
+        args = [
+            AFL_TMIN_CMD,
+            "-i",
+            filename,
+            "-o",
+            output_path,
+            "-t",
+            "3000",
+            "-m",
+            "4096",
+            "--",
+            harness_bin,
+            "@@"
+        ]
+
+        print("[i] Processing %s file ... " %(filename))
+
+        with open(os.devnull, 'w') as devnull:
+            p = subprocess.Popen(args, stdout=devnull, stderr=devnull, shell=False)
+            # p.communicate()
+            p.wait()
+
+        # p.stdout.close()
+
+        queue.task_done()
+
+def read_filename(input_dirname):
+
+    for f in os.listdir(input_dirname):
+        yield ("%s/%s" %(input_dirname, f)).strip()
+
+def main():
+
+    try:
+
+        n_cores = 5
+        input_dirname = sys.argv[1]
+        output_dirname = sys.argv[2]
+        harness_bin = sys.argv[3]
+
+        if not os.path.exists(output_dirname):
+            os.makedirs(output_dirname)
+
+        queue = Queue()
+
+        # Set up some threads
+        for i in range(n_cores):
+            worker = Thread(target=tmin, args=(i, queue, output_dirname, harness_bin, ))
+            worker.setDaemon(True)
+            worker.start()
+
+        # Fill the queue
+        counter = 0
+        for filename in read_filename(input_dirname):
+
+            data = { "filename": filename,
+                     "counter": counter }
+
+            queue.put(data)
+            counter = counter + 1
+
+        queue.join()
+
+    except Exception as e:
+        print("[!] Error: %s" %(str(e)))
+
+if __name__ == "__main__":
+    main()

lib/include/pl/core/lexer.hpp

@@ -49,7 +49,7 @@ namespace pl::core {
         std::optional<u128> parseInteger(std::string_view literal);
 
         Token makeToken(const Token& token, size_t length = 1);
-        static Token makeTokenAt(const Token& token, Location& location, size_t length = 1);
+        Token makeTokenAt(const Token& token, Location& location, size_t length = 1);
         void addToken(const Token& token);
         bool hasTheLineEnded(const char &ch) {
             if(ch == '\n') {

lib/include/pl/core/preprocessor.hpp

@@ -36,7 +36,8 @@ namespace pl::core {
         void addStatementHandler(const Token::Keyword &statementType, const api::StatementHandler &handler);
         void removePragmaHandler(const std::string &pragmaType);
         void removeDirectiveHandler(const Token::Directive &directiveType);
-
+        size_t getLongestLineLength() const { return m_longestLineLength; }
+        void setLongestLineLength(size_t length) { m_longestLineLength = length; }
         void validateOutput();
 
         [[nodiscard]] const std::vector<ExcludedLocation>& getExcludedLocations() const {
@@ -140,7 +141,12 @@ namespace pl::core {
         std::unordered_map<Token::Directive, api::DirectiveHandler> m_directiveHandlers;
         std::unordered_map<Token::Keyword, api::StatementHandler> m_statementHandlers;
 
-        std::unordered_map<std::string, std::vector<Token>> m_defines;
+        struct Define {
+            Token nameToken;
+            std::vector<Token> values;
+        };
+
+        std::unordered_map<std::string, Define> m_defines;
         std::unordered_map<std::string, std::vector<std::pair<std::string, u32>>> m_pragmas;
         std::vector<ExcludedLocation> m_excludedLocations;
 
@@ -157,7 +163,7 @@ namespace pl::core {
         std::vector<Token> m_result;
         std::vector<Token> m_output;
         std::vector<std::string> m_namespaces;
-
+        size_t m_longestLineLength = 0;
         api::Source* m_source = nullptr;
 
         bool m_onlyIncludeOnce = false;

lib/source/pl/core/ast/ast_node_attribute.cpp

@@ -335,7 +335,7 @@ namespace pl::core::ast {
             auto actualSize = pattern->getSize();
             if (actualSize < requestedSize) {
                 pattern->setSize(requestedSize);
-                evaluator->setReadOffset(u64(evaluator->getReadOffset() + (requestedSize - actualSize)));
+                evaluator->setReadOffset(pattern->getOffset() + requestedSize);
             }
             else if (actualSize > requestedSize)
                 err::E0004.throwError("Type size larger than expected", fmt::format("Pattern of type {} is larger than expected. Expected size {}, got {}", pattern->getTypeName(), requestedSize, actualSize), node->getLocation());
@@ -384,6 +384,8 @@ namespace pl::core::ast {
                         .bitOffset = bitfieldPattern->getBitOffset()
                     });
                 }
+            } else if (attributable->hasAttribute("fixed_size", true)) {
+                // read offset might be modified by fixed_size's padding. keep it as is.
             } else {
                 evaluator->setBitwiseReadOffset(endOffset);
             }

lib/source/pl/core/lexer.cpp

@@ -458,11 +458,13 @@ namespace pl::core {
     Token Lexer::makeToken(const Token &token, const size_t length) {
         auto location = this->location();
         location.length = length;
+        m_longestLineLength = std::max(m_longestLineLength, location.column + location.length);
         return { token.type, token.value, location };
     }
 
     Token Lexer::makeTokenAt(const Token &token, Location& location, const size_t length) {
         location.length = length;
+        m_longestLineLength = std::max(m_longestLineLength, location.column + location.length);
         return { token.type, token.value, location };
     }
 

lib/source/pl/core/preprocessor.cpp

@@ -178,17 +178,34 @@ namespace pl::core {
         }
 
         if (m_defines.contains(name)) {
-            bool isValueSame = m_defines[name] == values;
+            bool isValueSame = m_defines[name].values == values;
             if (!isValueSame) {
-                errorAt(values[0].location, "Previous definition occurs at line '{}'.", m_defines[name][0].location.line);
-                errorAt(m_defines[name][0].location, "Macro '{}' is redefined in line '{}'.", name, values[0].location.line);
-                m_defines[name].clear();
-                m_defines[name] = values;
+                auto& define = m_defines[name];
+                Location defineLocation{};
+
+                if (define.values.empty()) {
+                    defineLocation = define.nameToken.location;
+                } else {
+                    defineLocation = define.values[0].location;
+                }
+
+                Location ourLocation{};
+                if (values.empty()) {
+                    ourLocation = token.location;
+                } else {
+                    ourLocation = values[0].location;
+                }
+
+                errorAt(ourLocation, "Previous definition occurs at line '{}'.", defineLocation.line);
+                errorAt(defineLocation, "Macro '{}' is redefined in line '{}'.", name, defineLocation.line);
+
+                m_defines[name] = { token, values };
+
                 removeKey(token);
                 m_keys.emplace_back(token);
             }
         } else {
-            m_defines[name] = values;
+            m_defines[name] = { token, values };
             m_keys.emplace_back(token);
         }
     }
@@ -340,7 +357,7 @@ namespace pl::core {
         if (auto *tokenLiteral = std::get_if<Token::Literal>(&m_token->value); m_token->type == Token::Type::String && tokenLiteral != nullptr) {
             path = tokenLiteral->toString(false);
 
-        } else if (auto *identifier = std::get_if<Token::Identifier>(&m_token->value); m_token->type == Token::Type::Identifier) {
+        } else if (auto *identifier = std::get_if<Token::Identifier>(&m_token->value); m_token->type == Token::Type::Identifier && identifier != nullptr) {
             saveImport.push_back(*m_token);
             path = identifier->get();
             m_token++;
@@ -476,7 +493,7 @@ namespace pl::core {
                         Token::Identifier *tokenIdentifier = std::get_if<Token::Identifier>(&m_token->value);
                         if (tokenIdentifier != nullptr)
                             tokenIdentifier->setType(Token::Identifier::IdentifierType::Macro);
-                        for (const auto &newToken: m_defines[keyIdentifier->get()])
+                        for (const auto &newToken: m_defines[keyIdentifier->get()].values)
                             resultValues.push_back(newToken);
                     } else
                         resultValues.push_back(value);
@@ -584,7 +601,7 @@ namespace pl::core {
                 this->error(item);
             return { m_output, collectErrors() };
         }
-
+        setLongestLineLength(lexer->getLongestLineLength());
         m_token = m_result.begin();
         m_initialized = true;
         while (!eof())
@@ -614,7 +631,9 @@ namespace pl::core {
     }
 
     void Preprocessor::addDefine(const std::string &name, const std::string &value) {
-        m_defines[name] = {Token { Token::Type::String, value, {nullptr, 0, 0, 0 } } } ;
+        auto nameToken = Token { Token::Type::Identifier, name, Location::Empty() };
+        auto valueToken = Token { Token::Type::String, value, Location::Empty() };
+        m_defines[name] = { nameToken, { valueToken }};
     }
 
     void Preprocessor::addPragmaHandler(const std::string &pragmaType, const api::PragmaHandler &handler) {

tests/include/test_patterns/test_pattern_attributes.hpp

@@ -41,14 +41,31 @@ namespace pl::test {
                     return 1337;
                 };
 
+                struct FixedSizeTest1 {
+                    u8 x;
+                } [[fixed_size(4)]];
+
+                struct FixedSizeTest2 {
+                    u32 pos = $;
+                    u32 elem [[fixed_size(7)]];
+                    std::assert($ == pos + 7, "Fixed size variable attribute padding not working");
+
+                    pos = $;
+                    FixedSizeTest1 fixedSizeTest1;
+                    std::assert($ == pos + 4, "Fixed size pattern attribute padding not working");
+                };
+
                 FormatTransformTest formatTransformTest @ 0x00;
                 SealedTest sealedTest @ 0x10;
                 HiddenTest hiddenTest @ 0x20;
                 ColorTest colorTest @ 0x30;
                 NoUniqueAddressTest noUniqueAddressTest @ 0x40;
+                FixedSizeTest1 fixedSizeTest1 @ 0x50;
+                FixedSizeTest2 fixedSizeTest2 @ 0x60;
 
                 std::assert(formatTransformTest == 1337, "Transform attribute not working");
                 std::assert(sizeof(noUniqueAddressTest) == sizeof(u32), "No Unique Address attribute not working");
+                std::assert(sizeof(fixedSizeTest1) == 4, "Fixed size attribute not working");
             )test";
         }