fix(parser): disallow requests without CR/LF linefeeds

in strict mode by default. Keep a unstrict mode, for easier CLI parsing
on linux.
prevents HTTP Request Smuggling via LF-only line endings.

RFC 7230 requires CRLF.

Author: spaceone

httoop/header/headers.py

@@ -11,11 +11,15 @@
 
 class Headers(CaseInsensitiveDict, Semantic):
 
-    __slots__ = ()
+    __slots__ = ('line_end',)
 
     # disallowed bytes for HTTP header field names
     HEADER_RE = re.compile(rb'[\x00-\x1F\x7F()<>@,;:\\\\\"/\[\]?={} \t\x80-\xFF]')
 
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.line_end = b'\r\n'
+
     @staticmethod
     def formatvalue(value: Any) -> bytes:
         if isinstance(value, str):
@@ -122,7 +126,7 @@ def parse(self, data: bytes) -> None:
         without trailing "\r\n"
         :type  data: bytes
         """
-        lines = data.split(b'\r\n')
+        lines = data.split(self.line_end)
 
         while lines:
             curr = lines.pop(0)

httoop/parser.py

@@ -31,9 +31,10 @@ class StateMachine:
 
     Message = Message  # subclass provides the type
 
-    def __init__(self) -> None:
+    def __init__(self, strict: bool = True) -> None:
         self.buffer = bytearray()
         self.message = None
+        self.strict = strict
 
     def _reset_state(self) -> None:
         self.message = self.Message()
@@ -126,7 +127,9 @@ def parse_startline(self) -> bool | None:
         if CRLF not in self.buffer:
             if LF not in self.buffer:
                 return NOT_RECEIVED_YET
-            self.line_end = LF
+            if self.strict:
+                raise BAD_REQUEST(_('HTTP requires CR/LF line feeds.'))
+            self.line_end = self.message.headers.line_end = LF
 
         requestline, self.buffer = self.buffer.split(self.line_end, 1)
 

httoop/server/__init__.py

@@ -20,8 +20,8 @@ class ServerStateMachine(StateMachine):
     Message = Request
     HTTP2 = None
 
-    def __init__(self, scheme: str, host: str, port: int) -> None:
-        super().__init__()
+    def __init__(self, scheme: str, host: str, port: int, strict: bool = True) -> None:
+        super().__init__(strict=strict)
         self.MAX_URI_LENGTH = float('inf')  # 8000
         self._default_scheme = scheme
         self._default_host = host

tests/api/test_statemachine.py

@@ -209,3 +209,15 @@ def test_parse_cl_te_combination(statemachine):
     with pytest.raises(BAD_REQUEST) as exc:
         statemachine.parse(b'POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nContent-Length: 0\r\nAccept: */*\r\nUser-Agent: httoop/0.0\r\nHost: localhost\r\nContent-Type: text/plain; charset="UTF-8"\r\n\r\n')
     assert 'Invalid Content-Length and Transfer-Encoding combination' in str(exc.value)
+
+
+def test_parse_lf_messages(statemachine):
+    with pytest.raises(BAD_REQUEST) as exc:
+        statemachine.parse(b'GET / HTTP/1.1\nAccept: */*\nUser-Agent: httoop/0.0\nHost: localhost\nContent-Type: text/plain; charset="UTF-8"\n\n')
+    assert 'HTTP requires CR/LF line feeds' in str(exc.value)
+
+
+def test_parse_lf_messages_unstrict(statemachine):
+    statemachine.strict = False
+    assert statemachine.parse(b'GET / HTTP/1.1\nAccept: */*\nUser-Agent: httoop/0.0\nHost: localhost\nContent-Type: text/plain; charset="UTF-8"\n\n')[0]
+    assert statemachine.parse(b'GET / HTTP/1.1\nHost: x\n\n')[0]