fields.yml

version: 2.0

defaults:
  type: string
  required: true
  index: not_analyzed
  doc_values: true
  ignore_above: 4096

sections:
  - ["ctx", "Context"]
  - ["app-logs", "Application log message fields"]

summary: |
  This is a specification of the "app-logs beat" message type that we use to store
  "application logs" written via a proper logging infrastructure implementation.

  NOTE: For Java applications running on CF, this is our logging support library, see
  https://github.com/SAP/cf-java-logging-support

  Field names follow the convention that we're not using camel case names, but
  rather use "_" as a delimiter between parts.

  For any field that actually represents a measure (or metric for that matter)
  the name includes a descriptive suffix mnemonic that helps identifying the unit,
  e.g.:

    * "_ms" -> milliseconds
    * "_b"  -> bytes
    * "_p"  -> percent
    * "_at" -> date with time
    * "_ts" -> timestamp

ctx:
  type: group
  description: |
    This group contains context information for the given application log message
    such that analytics can slice and dice information based on these criteria.

  fields:
    - name: "written_at"
      type: date
      required: true
      description: |
        The human-readable date when the message was written.

    - name: "written_ts"
      type: long
      required: true
      example: 1456820553816849408
      description: |
        The timestamp in nano-second precision when the message was written.

    - name: "correlation_id"
      type: string
      required: true
      example: "db2d002e-2702-41ec-66f5-c002a80a3d3f"
      description: |
        A unique identifier that can be used to correlate multiple messages into a logical unit.
        
        Currently, we use a UUID for that matter, but future versions may use
        a different key for this.

        "-" indicates that no proper value could be provided.

        NOTE: As these type of messages can be written at any time and outside
        a "request context" it may happen that this correlation id does not
        match with any other. Logging implementations may provide a way to
        inherit correlation ids from request processing layers further up in
        the processing stack. If so, that id should be used.

    - name: "request_id"
      string: string
      required: false
      example: "e24a5963-95eb-4568-b1ae-81b67c41e99a"
      description: |
        A unique identifier that can be used to correlate multiple messages to a request.

    - name: "w3c_traceparent"
      type: string
      required: false
      example: "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
      description: |
        The content of the W3C traceparent header as defined in 
        https://www.w3.org/TR/trace-context/#traceparent-header.
        The traceparent allows correlation of logs to the request.

    - name: "sap_passport"
      type: string
      required: false
      example: "2a54482a0300e60000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002a54482a"
      description: |
        SAP Passport is an end to end tracing technology used in many SAP products.
        It is a binary format encoded in hex notation.
        Applications can add the whole SAP Passport in this field or give its constituents in the respective fields. 
    - name: "sap_passport_Action"
      type: string
      required: false
      description: |
        This field can contain the action of an SAP Passport. 
    - name: "sap_passport_ActionType"
      type: string
      required: false
      description: |
        This field can contain the action type of an SAP Passport. 
    - name: "sap_passport_ClientNumber"
      type: string
      required: false
      description: |
        This field can contain the client number of an SAP Passport. 
    - name: "sap_passport_ConnectionCounter"
      type: integer
      required: false
      description: |
        This field can contain the connection counter of an SAP Passport. 
    - name: "sap_passport_ConnectionId"
      type: string
      required: false
      description: |
        This field can contain the connection id type of an SAP Passport. 
    - name: "sap_passport_ComponentName"
      type: string
      required: false
      description: |
        This field can contain the component name of an SAP Passport. 
    - name: "sap_passport_ComponentType"
      type: string
      required: false
      description: |
        This field can contain the component type of an SAP Passport. 
    - name: "sap_passport_PreviousComponentName"
      type: string
      required: false
      description: |
        This field can contain the previous component name of an SAP Passport. 
    - name: "sap_passport_TraceFlags"
      type: string
      required: false
      description: |
        This field can contain the trace flags of an SAP Passport. 
    - name: "sap_passport_TransactionId"
      type: string
      required: false
      description: |
        This field can contain the transaction id of an SAP Passport. 
    - name: "sap_passport_RootContextId"
      type: string
      required: false
      description: |
        This field can contain the root context id of an SAP Passport. 
    - name: "sap_passport_UserId"
      type: string
      required: false
      description: |
        This field can contain the user id of an SAP Passport. 

    - name: "tenant_id"
      string: string
      required: false
      example: "e24a5963-95eb-4568-b1ae-81b67c41e99a"
      description: |
        A unique identifier that can be used to correlate multiple messages to a tenant.

    - name: "tenant_subdomain"
      string: string
      required: false
      example: "acme-inc"
      description: |
        A unique identifier that represents the subdomain of the current tenant specific request url, e.g. acme-inc in https://acme-inc.eu10.cloud.alm.sap/home
    
    - name: "component_id"
      type: string
      required: true
      example: "9e6f3ecf-def0-4baf-8fac-9339e61d5645"
      description: |
        A unique identifier of the software component that has written the message. 
        It may either be a "service" or an "application" that can be deployed to
        a container or VM, but it may not yet fully identify the actual "running instance" of
        that "component".

        NOTE: For applications in CF, this is the "application_id", but we may have more than one
        instance of that running.

    - name: "component_name"
      type: string
      required: false
      example: "my-fancy-component"
      description: |
        A human-friendly, not necessarily unique, name representing the software component.

    - name: "component_type"
      type: string
      required: true
      example: "application"
      description: |
        The type of the software component that has written the message.

        Either "application" or "service"

    - name: "component_instance"
      type: string
      required: true
      example: "7"
      description: |
        CF allows us to scale out horizontally by running several instances of the
        same "component" in one container, this is identifying the instance.
        Defaults to "0".

    - name: "container_id"
      type: string
      required: false
      format: ???
      example: "???"
      description: |
        The unique identifier of the container on which the software component instance is running.

        NOTE: If security/privacy is not an issue, an IP address would be ok.

        NOTE: CF sets an environment variable "INSTANCE_IP" that is currently used
        for that purpose

    - name: "organization_id"
      type: string
      required: true
      example: "280437b3-dd8b-40b1-bbab-1f05a44345f8"
      description: |
        The unique identifier of the Cloud Foundry organization to which the software component belongs.

        "-" indicates that no proper value could be provided.

    - name: "organization_name"
      type: string
      required: true
      example: "acme"
      description: |
        The human-readable name of the Cloud Foundry organization.

        NOTE: Most likely NOT unique!

        "-" indicates that no proper value could be provided.

    - name: "space_id"
      type: string
      required: true
      example: "280437b3-dd8b-40b1-bbab-1f05a44345f8"
      description: |
        The unique identifier of the Cloud Foundry space to which the software component belongs.

        "-" indicates that no proper value could be provided.

    - name: "space_name"
      type: string
      required: true
      example: "test"
      description: |
        The human-readable name of the Cloud Foundry space.

        NOTE: Most likely NOT unique!

    - name: "layer"
      type: string
      required: true
      example: "[JAVA:BusinessLogicController]"
      description: |
        The execution layer in the overall processing chain that has written the message.

    - name: "type"
      type: string
      required: true
      description: |
        The type of the message.
        To make the message self-contained this is the type tag.
        As we're talking about log messages, it's always "logs".

app-logs:
  type: group
  description: |
    The individual fields of a log message. Should be familiar to anyone who
    ever used a logging library in Java...

  fields:
    - name: "logger"
      type: string
      required: true
      example: "com.sap.demo.shine.OrderController"
      description: |
        The actual application entity that wrote the log message, e.g. the Java class name.

        NOTE: In Java, this is the name that was passed to the LoggerFactory
        when instantiating a Logger.

    - name: "thread"
      type: string
      required: false
      example: "http-nio-4655"
      description: |
        The execution thread in which the log message has been written.

        "-" indicates that no proper value could be provided.

    - name: "level"
      type: string
      required: true
      example: "INFO"
      description: |
        The log "level" indicating the severity of the log message.

    - name: "msg"
      type: string
      required: true
      example: "This is a log messasge"
      description: |
        The original log message that has been written by the application.

    - name: "exception_type"
      type: string
      required: false
      example: "java.lang.NullPointerException"
      description: |
        The Java class name of the logged exception when available.

    - name: "exception_message"
      type: string
      required: false
      example: "Something went wrong"
      description: |
        The message of the logged exception when available.

    - name: "stacktrace"
      type: array
      required: false
      example: "stacktrace: [\"java.lang.NullPointerException\",\"at com.sap.demo.exception.NullPointerException\"]"
      description: |
        Stacktrace information in case the message is reporting an exception.

        NOTE: This MUST NOT contain NEWLINE or TAB characters!

    - name: "categories"
      type: string
      required: false
      description: |
        A list of names to further categorize this log message.

    - name: "#cf"
      java_alias: "CUSTOM_FIELDS"
      type: object
      required: false
      example: |
        "#cf": {
          "string": [
            {"k":"some_key", "v":"some_value", "i": 0},
            {"k":"other_key", "v":"other_value", "i": 1}
          ]
        }
      description: |
        An object containing collections of non-standard fields.
        The field "string" contains custom fields with key "k", value "v" and an index "i".
        The index can be used for field order during parsing.

        NOTE: As this is "custom" there are no predefined fields here!