Feat: Add protocol-specific event types

Author: Chocapikk

.github/workflows/action.yml

@@ -6,7 +6,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.13"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
         poetry-version: ["2.3.1"]
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     runs-on: ${{ matrix.os }}

l9format/__init__.py

@@ -1 +1 @@
-__version__ = "1.3.2"
+__version__ = "1.4.0"

l9format/l9format.py

@@ -3,8 +3,6 @@
 from serde import Model, fields
 
 
-# Import from https://github.com/rossmacarthur/serde/commit/304884bca3c80e8b9a22a054b64dd94e3324b593#diff-dd2f65f16516311f0183377201a3d0b7894438787da732fc2bbc9c28cbde9fb8
-# A helper function...
 def round_decimal(
     decimal_obj: decimal.Decimal, num_of_places: int = 6
 ) -> decimal.Decimal:
@@ -16,19 +14,18 @@ def round_decimal(
 class Decimal(fields.Instance):
     """
     A `~decimal.Decimal` field.
+
     This field serializes `~decimal.Decimal` objects as strings and
     deserializes string representations of Decimals as `~decimal.Decimal`
     objects.
-    The resolution of the decimal can be specified. When not specified, the number
-    is not rounded. When it is specified, the decimal is rounded to this number of
-    decimal places upon serialization and deserialization.
-    Note: When float type numbers are not rounded before serialization,
-    they will be serialized in exact form, which as they are floats,
-    is almost never the exact intended value,
-    e.g. 0.2 = 0.20000000000000000000023
+
+    The resolution of the decimal can be specified. When not specified,
+    the number is not rounded. When it is specified, the decimal is rounded
+    to this number of decimal places upon serialization and deserialization.
+
     Args:
-        resolution (Union[int, bool]): The number of decimal places to round to.
-        When None, rounding is disabled.
+        resolution (int | None): The number of decimal places to round to.
+            When None, rounding is disabled.
         **kwargs: keyword arguments for the `Field` constructor.
     """
 
@@ -49,44 +46,33 @@ def deserialize(self, value: object) -> decimal.Decimal:
                 return round_decimal(
                     decimal.Decimal(str(value)), num_of_places=self.resolution
                 )
-
             return decimal.Decimal(str(value))
         except decimal.DecimalException as e:
             raise ValueError(f"invalid decimal: {value}") from e
 
 
-class L9HttpEvent(Model):
-    root: fields.Str()
-    url: fields.Str()
-    status: fields.Int()
-    # FIXME: must be int64
-    length: fields.Int()
-    header: fields.Optional(fields.Dict(key=fields.Str(), value=fields.Str()))
-    title: fields.Str()
-    favicon_hash: fields.Str()
+# --- Base Models ---
 
 
-class ServiceCredentials(Model):
-    noauth: fields.Bool()
-    username: fields.Str()
-    password: fields.Str()
-    key: fields.Str()
-    # Bytes
-    raw: fields.Optional(fields.Str())
+class GeoPoint(Model):
+    lat: Decimal()
+    lon: Decimal()
 
 
-class SoftwareModule(Model):
-    name: fields.Str()
-    version: fields.Str()
-    fingerprint: fields.Str()
+class GeoLocation(Model):
+    continent_name: fields.Optional(fields.Str())
+    region_iso_code: fields.Optional(fields.Str())
+    city_name: fields.Optional(fields.Str())
+    country_iso_code: fields.Optional(fields.Str())
+    country_name: fields.Optional(fields.Str())
+    region_name: fields.Optional(fields.Str())
+    location: fields.Optional(fields.Nested(GeoPoint))
 
 
-class Software(Model):
-    name: fields.Str()
-    version: fields.Str()
-    os: fields.Str()
-    modules: fields.Optional(fields.List(fields.Nested(SoftwareModule)))
-    fingerprint: fields.Str()
+class Network(Model):
+    organization_name: fields.Str()
+    asn: fields.Int()
+    network: fields.Str()
 
 
 class Certificate(Model):
@@ -101,26 +87,26 @@ class Certificate(Model):
     valid: fields.Bool()
 
 
-class GeoPoint(Model):
-    lat: Decimal()
-    lon: Decimal()
+class SoftwareModule(Model):
+    name: fields.Str()
+    version: fields.Str()
+    fingerprint: fields.Str()
 
 
-class GeoLocation(Model):
-    continent_name: fields.Optional(fields.Str())
-    region_iso_code: fields.Optional(fields.Str())
-    city_name: fields.Optional(fields.Str())
-    country_iso_code: fields.Optional(fields.Str())
-    country_name: fields.Optional(fields.Str())
-    region_name: fields.Optional(fields.Str())
-    location: fields.Optional(fields.Nested(GeoPoint))
+class Software(Model):
+    name: fields.Str()
+    version: fields.Str()
+    os: fields.Str()
+    modules: fields.Optional(fields.List(fields.Nested(SoftwareModule)))
+    fingerprint: fields.Str()
 
 
-class L9SSHEvent(Model):
-    fingerprint: fields.Str()
-    version: fields.Int()
-    banner: fields.Str()
-    motd: fields.Str()
+class ServiceCredentials(Model):
+    noauth: fields.Bool()
+    username: fields.Str()
+    password: fields.Str()
+    key: fields.Str()
+    raw: fields.Optional(fields.Str())
 
 
 class DatasetSummary(Model):
@@ -132,11 +118,17 @@ class DatasetSummary(Model):
     ransom_notes: fields.Optional(fields.List(fields.Str()))
 
 
-class L9LeakEvent(Model):
-    stage: fields.Str()
-    type: fields.Str()
-    severity: fields.Str()
-    dataset: fields.Nested(DatasetSummary)
+# --- Service Events ---
+
+
+class L9HttpEvent(Model):
+    root: fields.Str()
+    url: fields.Str()
+    status: fields.Int()
+    length: fields.Int()
+    header: fields.Optional(fields.Dict(key=fields.Str(), value=fields.Str()))
+    title: fields.Str()
+    favicon_hash: fields.Str()
 
 
 class L9SSLEvent(Model):
@@ -153,10 +145,149 @@ class L9ServiceEvent(Model):
     software: fields.Nested(Software)
 
 
-class Network(Model):
-    organization_name: fields.Str()
-    asn: fields.Int()
-    network: fields.Str()
+class L9LeakEvent(Model):
+    stage: fields.Str()
+    type: fields.Str()
+    severity: fields.Str()
+    dataset: fields.Nested(DatasetSummary)
+
+
+# --- Protocol Events ---
+
+
+class L9SSHEvent(Model):
+    fingerprint: fields.Optional(fields.Str())
+    version: fields.Optional(fields.Int())
+    banner: fields.Optional(fields.Str())
+    motd: fields.Optional(fields.Str())
+    key_type: fields.Optional(fields.Str())
+    key: fields.Optional(fields.Str())
+    kex_algorithms: fields.Optional(fields.List(fields.Str()))
+    host_key_algorithms: fields.Optional(fields.List(fields.Str()))
+    encryption_algorithms: fields.Optional(fields.List(fields.Str()))
+    mac_algorithms: fields.Optional(fields.List(fields.Str()))
+    compression_algorithms: fields.Optional(fields.List(fields.Str()))
+    auth_methods: fields.Optional(fields.List(fields.Str()))
+
+
+class L9VNCEvent(Model):
+    version: fields.Optional(fields.Str())
+    security_types: fields.Optional(fields.List(fields.Str()))
+    noauth: fields.Optional(fields.Bool())
+
+
+class L9FTPEvent(Model):
+    banner: fields.Optional(fields.Str())
+    tls_supported: fields.Optional(fields.Bool())
+    anonymous: fields.Optional(fields.Bool())
+
+
+class L9SMTPEvent(Model):
+    banner: fields.Optional(fields.Str())
+    starttls: fields.Optional(fields.Bool())
+    extensions: fields.Optional(fields.List(fields.Str()))
+
+
+class L9TelnetEvent(Model):
+    banner: fields.Optional(fields.Str())
+    options: fields.Optional(fields.List(fields.Str()))
+    auth_required: fields.Optional(fields.Bool())
+
+
+class L9RedisEvent(Model):
+    version: fields.Optional(fields.Str())
+    mode: fields.Optional(fields.Str())
+    os: fields.Optional(fields.Str())
+    auth_required: fields.Optional(fields.Bool())
+
+
+class L9MySQLEvent(Model):
+    version: fields.Optional(fields.Str())
+    protocol_version: fields.Optional(fields.Int())
+    auth_plugin: fields.Optional(fields.Str())
+    server_status: fields.Optional(fields.Str())
+
+
+class L9PostgreSQLEvent(Model):
+    version: fields.Optional(fields.Str())
+    databases: fields.Optional(fields.List(fields.Str()))
+    ssl_enabled: fields.Optional(fields.Bool())
+    auth_method: fields.Optional(fields.Str())
+    server_encoding: fields.Optional(fields.Str())
+    client_encoding: fields.Optional(fields.Str())
+    timezone: fields.Optional(fields.Str())
+    max_connections: fields.Optional(fields.Int())
+
+
+class L9MongoDBEvent(Model):
+    version: fields.Optional(fields.Str())
+    databases: fields.Optional(fields.List(fields.Str()))
+    auth_required: fields.Optional(fields.Bool())
+    wire_version: fields.Optional(fields.Int())
+
+
+class L9MemcachedEvent(Model):
+    version: fields.Optional(fields.Str())
+    libevent: fields.Optional(fields.Str())
+    curr_items: fields.Optional(fields.Int())
+    total_items: fields.Optional(fields.Int())
+    bytes: fields.Optional(fields.Int())
+    max_bytes: fields.Optional(fields.Int())
+    cmd_get: fields.Optional(fields.Int())
+    cmd_set: fields.Optional(fields.Int())
+    get_hits: fields.Optional(fields.Int())
+    get_misses: fields.Optional(fields.Int())
+    threads: fields.Optional(fields.Int())
+
+
+class L9AMQPEvent(Model):
+    protocol_major: fields.Optional(fields.Int())
+    protocol_minor: fields.Optional(fields.Int())
+    product: fields.Optional(fields.Str())
+    version: fields.Optional(fields.Str())
+    platform: fields.Optional(fields.Str())
+
+
+class L9LDAPEvent(Model):
+    naming_contexts: fields.Optional(fields.List(fields.Str()))
+    supported_versions: fields.Optional(fields.List(fields.Str()))
+    vendor_name: fields.Optional(fields.Str())
+    vendor_version: fields.Optional(fields.Str())
+    supported_sasl: fields.Optional(fields.List(fields.Str()))
+    anonymous_bind: fields.Optional(fields.Bool())
+    can_enumerate: fields.Optional(fields.Bool())
+
+
+class L9SIPEvent(Model):
+    version: fields.Optional(fields.Str())
+    user_agent: fields.Optional(fields.Str())
+    server: fields.Optional(fields.Str())
+    allow: fields.Optional(fields.List(fields.Str()))
+    supported: fields.Optional(fields.List(fields.Str()))
+
+
+class L9RDPEvent(Model):
+    product_version: fields.Optional(fields.Str())
+    nla_required: fields.Optional(fields.Bool())
+    ssl_enabled: fields.Optional(fields.Bool())
+    hostname: fields.Optional(fields.Str())
+
+
+class L9DNSEvent(Model):
+    software: fields.Optional(fields.Str())
+    version: fields.Optional(fields.Str())
+    recursion: fields.Optional(fields.Bool())
+    dnssec: fields.Optional(fields.Bool())
+    zone_transfer: fields.Optional(fields.Bool())
+    nameservers: fields.Optional(fields.List(fields.Str()))
+
+
+class L9RTSPEvent(Model):
+    server: fields.Optional(fields.Str())
+    methods: fields.Optional(fields.List(fields.Str()))
+
+
+# --- Main Event ---
 
 
 class L9Event(Model):
@@ -176,25 +307,45 @@ class L9Event(Model):
     summary: fields.Str()
     time: fields.DateTime()
     ssl: fields.Optional(fields.Nested(L9SSLEvent))
-    ssh: fields.Nested(L9SSHEvent)
+    # Protocol-specific events
+    ssh: fields.Optional(fields.Nested(L9SSHEvent))
+    vnc: fields.Optional(fields.Nested(L9VNCEvent))
+    ftp: fields.Optional(fields.Nested(L9FTPEvent))
+    smtp: fields.Optional(fields.Nested(L9SMTPEvent))
+    telnet: fields.Optional(fields.Nested(L9TelnetEvent))
+    redis: fields.Optional(fields.Nested(L9RedisEvent))
+    mysql: fields.Optional(fields.Nested(L9MySQLEvent))
+    postgresql: fields.Optional(fields.Nested(L9PostgreSQLEvent))
+    mongodb: fields.Optional(fields.Nested(L9MongoDBEvent))
+    memcached: fields.Optional(fields.Nested(L9MemcachedEvent))
+    amqp: fields.Optional(fields.Nested(L9AMQPEvent))
+    ldap: fields.Optional(fields.Nested(L9LDAPEvent))
+    sip: fields.Optional(fields.Nested(L9SIPEvent))
+    rdp: fields.Optional(fields.Nested(L9RDPEvent))
+    dns: fields.Optional(fields.Nested(L9DNSEvent))
+    rtsp: fields.Optional(fields.Nested(L9RTSPEvent))
+    # Service events
     service: fields.Nested(L9ServiceEvent)
-    leak: fields.Optional(fields.Nested((L9LeakEvent)))
+    leak: fields.Optional(fields.Nested(L9LeakEvent))
     tags: fields.Optional(fields.List(fields.Str()))
     geoip: fields.Nested(GeoLocation)
     network: fields.Nested(Network)
 
 
+# --- Aggregation ---
+
+
 class L9Aggregation(Model):
-    summary = fields.Optional(fields.Str())
-    ip = fields.Str()
-    resource_id = fields.Str()
-    open_ports = fields.List(fields.Str())
-    leak_count = fields.Int()
-    leak_event_count = fields.Int()
-    events = fields.List(fields.Nested(L9Event))
-    plugins = fields.List(fields.Str())
-    geoip = fields.Nested(GeoLocation)
-    network = fields.Nested(Network)
-    creation_date = fields.DateTime()
-    update_date = fields.DateTime()
-    fresh = fields.Bool()
+    summary: fields.Optional(fields.Str())
+    ip: fields.Str()
+    resource_id: fields.Str()
+    open_ports: fields.List(fields.Str())
+    leak_count: fields.Int()
+    leak_event_count: fields.Int()
+    events: fields.List(fields.Nested(L9Event))
+    plugins: fields.List(fields.Str())
+    geoip: fields.Nested(GeoLocation)
+    network: fields.Nested(Network)
+    creation_date: fields.DateTime()
+    update_date: fields.DateTime()
+    fresh: fields.Bool()