Skip to content

Commit 52bacbd

Browse files
Jackie-Robsonrtrieu
Jackie-Robson
and
rtrieu
authored
Add missing VRL functions to Custom Processor doc [WEB-7892] (#37611)
* Add missing VRL functions to Custom Processor doc [WEB-7892] Adds four VRL functions that exist in Vector but were missing from the OP Custom Processor reference: pop (Array), encrypt_ip (IP), decrypt_ip (IP), and haversine (new Map category). * Update content/en/observability_pipelines/processors/custom_processor.md Co-authored-by: Rosa Trieu <107086888+rtrieu@users.noreply.github.com> * Update data/reference/functions.json Co-authored-by: Rosa Trieu <107086888+rtrieu@users.noreply.github.com> * Update data/reference/functions.json Co-authored-by: Rosa Trieu <107086888+rtrieu@users.noreply.github.com> --------- Co-authored-by: Rosa Trieu <107086888+rtrieu@users.noreply.github.com>
1 parent a2a7634 commit 52bacbd

2 files changed

Lines changed: 232 additions & 22 deletions

File tree

‎content/en/observability_pipelines/processors/custom_processor.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ Use this processor with Vector Remap Language (VRL) to modify and enrich your lo
3030
- [Convert syslog values](#convert) to read-able values.
3131
- Enrich values by using [enrichment tables](#enrichment).
3232
- [Manipulate IP values](#ip).
33+
- Calculate [geographic distances](#map) and bearing with haversine.
3334
- [Parse](#parse) values with custom rules (for example, grok, regex, and so on) and out-of-the-box functions (for example, syslog, apache, VPC flow logs, and so on). See [Writing Effective Grok Parsing Rules with Regular Expressions][3] for information.
3435
- Manipulate event [paths](#path).
3536

@@ -63,6 +64,7 @@ To set up this processor:
6364
{{< nextlink href="observability_pipelines/processors/custom_processor/#debug" >}}Debug{{< /nextlink >}}
6465
{{< nextlink href="observability_pipelines/processors/custom_processor/#enrichment" >}}Enrichment{{< /nextlink >}}
6566
{{< nextlink href="observability_pipelines/processors/custom_processor/#ip" >}}IP{{< /nextlink >}}
67+
{{< nextlink href="observability_pipelines/processors/custom_processor/#map" >}}Map{{< /nextlink >}}
6668
{{< nextlink href="observability_pipelines/processors/custom_processor/#number" >}}Number{{< /nextlink >}}
6769
{{< nextlink href="observability_pipelines/processors/custom_processor/#object" >}}Object{{< /nextlink >}}
6870
{{< nextlink href="observability_pipelines/processors/custom_processor/#parse" >}}Parse{{< /nextlink >}}

‎data/reference/functions.json‎

Lines changed: 230 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -435,8 +435,40 @@
435435
},
436436
{
437437
"title": "Chunks do not respect unicode code point boundaries",
438-
"source": "chunks(\"ab你好\", 4)",
439-
"return": "[\"ab�\",\"�好\"]"
438+
"source": "chunks(\"ab\u4f60\u597d\", 4)",
439+
"return": "[\"ab\ufffd\",\"\ufffd\u597d\"]"
440+
}
441+
],
442+
"deprecated": false,
443+
"pure": true
444+
},
445+
{
446+
"anchor": "pop",
447+
"name": "pop",
448+
"category": "Array",
449+
"description": "Removes the last item from the `value` array, returning a new array without the last element.",
450+
"notices": [],
451+
"arguments": [
452+
{
453+
"name": "value",
454+
"description": "The array to pop from.",
455+
"required": true,
456+
"type": [
457+
"array"
458+
]
459+
}
460+
],
461+
"internal_failure_reasons": [],
462+
"return": {
463+
"types": [
464+
"array"
465+
]
466+
},
467+
"examples": [
468+
{
469+
"title": "Pop an item from an array",
470+
"source": "pop([1, 2, 3])",
471+
"return": "[1,2]"
440472
}
441473
],
442474
"deprecated": false,
@@ -2493,12 +2525,12 @@
24932525
},
24942526
{
24952527
"title": "UTF-8 string with bytes segmentation",
2496-
"source": "floor(shannon_entropy(\"test123%456.فوائد.net.\"), precision: 4)",
2528+
"source": "floor(shannon_entropy(\"test123%456.\u0641\u0648\u0627\u0626\u062f.net.\"), precision: 4)",
24972529
"return": "4.0784"
24982530
},
24992531
{
25002532
"title": "UTF-8 string with grapheme segmentation",
2501-
"source": "floor(shannon_entropy(\"test123%456.فوائد.net.\", segmentation: \"grapheme\"), precision: 4)",
2533+
"source": "floor(shannon_entropy(\"test123%456.\u0641\u0648\u0627\u0626\u062f.net.\", segmentation: \"grapheme\"), precision: 4)",
25022534
"return": "3.9362"
25032535
}
25042536
],
@@ -2556,12 +2588,12 @@
25562588
"examples": [
25572589
{
25582590
"title": "Sieve with regex",
2559-
"source": "sieve(\"test123%456.فوائد.net.\", r'[a-z0-9.]')",
2591+
"source": "sieve(\"test123%456.\u0641\u0648\u0627\u0626\u062f.net.\", r'[a-z0-9.]')",
25602592
"return": "\"test123456..net.\""
25612593
},
25622594
{
25632595
"title": "Custom replacements",
2564-
"source": "sieve(\"test123%456.فوائد.net.\", r'[a-z.0-9]', replace_single: \"X\", replace_repeated: \"<REMOVED>\")",
2596+
"source": "sieve(\"test123%456.\u0641\u0648\u0627\u0626\u062f.net.\", r'[a-z.0-9]', replace_single: \"X\", replace_repeated: \"<REMOVED>\")",
25652597
"return": "\"test123X456.<REMOVED>.net.\""
25662598
}
25672599
],
@@ -3509,7 +3541,7 @@
35093541
"examples": [
35103542
{
35113543
"title": "strlen",
3512-
"source": "strlen(\"ñandú\")",
3544+
"source": "strlen(\"\u00f1and\u00fa\")",
35133545
"return": "5"
35143546
}
35153547
],
@@ -3943,17 +3975,17 @@
39433975
{
39443976
"title": "Decode EUC-KR string",
39453977
"source": "decode_charset!(decode_base64!(\"vsiz58fPvLy/5A==\"), \"euc-kr\")",
3946-
"return": "\"안녕하세요\""
3978+
"return": "\"\uc548\ub155\ud558\uc138\uc694\""
39473979
},
39483980
{
39493981
"title": "Decode EUC-JP string",
39503982
"source": "decode_charset!(decode_base64!(\"pLOk86TLpMGkzw==\"), \"euc-jp\")",
3951-
"return": "\"こんにちは\""
3983+
"return": "\"\u3053\u3093\u306b\u3061\u306f\""
39523984
},
39533985
{
39543986
"title": "Decode GB2312 string",
39553987
"source": "decode_charset!(decode_base64!(\"xOO6ww==\"), \"gb2312\")",
3956-
"return": "\"你好\""
3988+
"return": "\"\u4f60\u597d\""
39573989
}
39583990
],
39593991
"deprecated": false,
@@ -4140,7 +4172,7 @@
41404172
{
41414173
"title": "Decode a punycode encoded internationalized domain name",
41424174
"source": "decode_punycode!(\"www.xn--caf-dma.com\")",
4143-
"return": "\"www.café.com\""
4175+
"return": "\"www.caf\u00e9.com\""
41444176
},
41454177
{
41464178
"title": "Decode an ASCII only string",
@@ -4150,7 +4182,7 @@
41504182
{
41514183
"title": "Ignore validation",
41524184
"source": "decode_punycode!(\"xn--8hbb.xn--fiba.xn--8hbf.xn--eib.\", validate: false)",
4153-
"return": "\"١٠.٦٦.٣٠.٥.\""
4185+
"return": "\"\u0661\u0660.\u0666\u0666.\u0663\u0660.\u0665.\""
41544186
}
41554187
],
41564188
"deprecated": false,
@@ -4389,17 +4421,17 @@
43894421
"examples": [
43904422
{
43914423
"title": "Encode UTF8 string to EUC-KR",
4392-
"source": "encode_base64(encode_charset!(\"안녕하세요\", \"euc-kr\"))",
4424+
"source": "encode_base64(encode_charset!(\"\uc548\ub155\ud558\uc138\uc694\", \"euc-kr\"))",
43934425
"return": "\"vsiz58fPvLy/5A==\""
43944426
},
43954427
{
43964428
"title": "Encode UTF8 string to EUC-JP",
4397-
"source": "encode_base64(encode_charset!(\"こんにちは\", \"euc-jp\"))",
4429+
"source": "encode_base64(encode_charset!(\"\u3053\u3093\u306b\u3061\u306f\", \"euc-jp\"))",
43984430
"return": "\"pLOk86TLpMGkzw==\""
43994431
},
44004432
{
44014433
"title": "Encode UTF8 string to GB2312",
4402-
"source": "encode_base64(encode_charset!(\"你好\", \"gb2312\"))",
4434+
"source": "encode_base64(encode_charset!(\"\u4f60\u597d\", \"gb2312\"))",
44034435
"return": "\"xOO6ww==\""
44044436
}
44054437
],
@@ -4818,12 +4850,12 @@
48184850
"examples": [
48194851
{
48204852
"title": "Encode an internationalized domain name",
4821-
"source": "encode_punycode!(\"www.café.com\")",
4853+
"source": "encode_punycode!(\"www.caf\u00e9.com\")",
48224854
"return": "\"www.xn--caf-dma.com\""
48234855
},
48244856
{
48254857
"title": "Encode an internationalized domain name with mixed case",
4826-
"source": "encode_punycode!(\"www.CAFé.com\")",
4858+
"source": "encode_punycode!(\"www.CAF\u00e9.com\")",
48274859
"return": "\"www.xn--caf-dma.com\""
48284860
},
48294861
{
@@ -6176,6 +6208,110 @@
61766208
}
61776209
],
61786210
"IP": [
6211+
{
6212+
"anchor": "decrypt_ip",
6213+
"name": "decrypt_ip",
6214+
"category": "IP",
6215+
"description": "Decrypts an IP address encrypted with `encrypt_ip`, returning the original IP address.\n\nSupported modes:\n\n* `aes128` - Reverses AES-128 ipcrypt-deterministic encryption. Key must be exactly 16 bytes.\n* `pfx` - Reverses prefix-preserving ipcrypt-pfx encryption. Key must be exactly 32 bytes.\n\nThe key and mode must match those used during encryption.",
6216+
"notices": [],
6217+
"arguments": [
6218+
{
6219+
"name": "ip",
6220+
"description": "The encrypted IP address to decrypt (IPv4 or IPv6).",
6221+
"required": true,
6222+
"type": [
6223+
"string"
6224+
]
6225+
},
6226+
{
6227+
"name": "key",
6228+
"description": "The decryption key as raw bytes. Must match the key used for encryption: 16 bytes for `aes128`, 32 bytes for `pfx`.",
6229+
"required": true,
6230+
"type": [
6231+
"string"
6232+
]
6233+
},
6234+
{
6235+
"name": "mode",
6236+
"description": "The decryption mode: `aes128` or `pfx`. Must match the mode used for encryption.",
6237+
"required": true,
6238+
"type": [
6239+
"string"
6240+
]
6241+
}
6242+
],
6243+
"internal_failure_reasons": [
6244+
"`ip` is not a valid IP address.",
6245+
"`mode` is not a supported mode (`aes128` or `pfx`).",
6246+
"`key` length does not match the mode requirements (16 bytes for `aes128`, 32 bytes for `pfx`)."
6247+
],
6248+
"return": {
6249+
"types": [
6250+
"string"
6251+
]
6252+
},
6253+
"examples": [
6254+
{
6255+
"title": "Decrypt an encrypted IPv4 address",
6256+
"source": "decrypt_ip!(\"72b9:a747:f2e9:72af:76ca:5866:6dcf:c3b0\", key: \"sixteen byte key\", mode: \"aes128\")",
6257+
"return": "\"192.168.1.1\""
6258+
}
6259+
],
6260+
"deprecated": false,
6261+
"pure": true
6262+
},
6263+
{
6264+
"anchor": "encrypt_ip",
6265+
"name": "encrypt_ip",
6266+
"category": "IP",
6267+
"description": "Encrypts an IP address, transforming it into a different valid IP address of the same version.\n\nSupported modes:\n\n* `aes128` - Scrambles the IP using AES-128 encryption (ipcrypt-deterministic). Accepts IPv4 or IPv6; key must be exactly 16 bytes.\n* `pfx` - Prefix-preserving encryption (ipcrypt-pfx), that maintains network hierarchy so addresses in the same subnet encrypt to addresses sharing the same prefix. Key must be exactly 32 bytes.\n\nEncryption is deterministic: the same input IP, key, and mode always produce the same output.",
6268+
"notices": [],
6269+
"arguments": [
6270+
{
6271+
"name": "ip",
6272+
"description": "The IP address to encrypt (IPv4 or IPv6).",
6273+
"required": true,
6274+
"type": [
6275+
"string"
6276+
]
6277+
},
6278+
{
6279+
"name": "key",
6280+
"description": "The encryption key as raw bytes. Must be 16 bytes for `aes128` mode or 32 bytes for `pfx` mode.",
6281+
"required": true,
6282+
"type": [
6283+
"string"
6284+
]
6285+
},
6286+
{
6287+
"name": "mode",
6288+
"description": "The encryption mode: `aes128` or `pfx`.",
6289+
"required": true,
6290+
"type": [
6291+
"string"
6292+
]
6293+
}
6294+
],
6295+
"internal_failure_reasons": [
6296+
"`ip` is not a valid IP address.",
6297+
"`mode` is not a supported mode (`aes128` or `pfx`).",
6298+
"`key` length does not match the mode requirements (16 bytes for `aes128`, 32 bytes for `pfx`)."
6299+
],
6300+
"return": {
6301+
"types": [
6302+
"string"
6303+
]
6304+
},
6305+
"examples": [
6306+
{
6307+
"title": "Encrypt an IPv4 address",
6308+
"source": "encrypt_ip!(\"192.168.1.1\", key: \"sixteen byte key\", mode: \"aes128\")",
6309+
"return": "\"72b9:a747:f2e9:72af:76ca:5866:6dcf:c3b0\""
6310+
}
6311+
],
6312+
"deprecated": false,
6313+
"pure": true
6314+
},
61796315
{
61806316
"anchor": "ip_aton",
61816317
"name": "ip_aton",
@@ -7397,7 +7533,7 @@
73977533
"enum": {
73987534
"ns": "Nanoseconds (1 billion nanoseconds in a second)",
73997535
"us": "Microseconds (1 million microseconds in a second)",
7400-
"µs": "Microseconds (1 million microseconds in a second)",
7536+
"\u00b5s": "Microseconds (1 million microseconds in a second)",
74017537
"ms": "Milliseconds (1 thousand microseconds in a second)",
74027538
"cs": "Centiseconds (100 centiseconds in a second)",
74037539
"ds": "Deciseconds (10 deciseconds in a second)",
@@ -7775,7 +7911,7 @@
77757911
},
77767912
{
77777913
"name": "lossy",
7778-
"description": "Whether to parse the JSON in a lossy manner. Replaces invalid UTF-8 characters\nwith the Unicode character `�` (U+FFFD) if set to true, otherwise returns an error\nif there are any invalid UTF-8 characters present.",
7914+
"description": "Whether to parse the JSON in a lossy manner. Replaces invalid UTF-8 characters\nwith the Unicode character `\ufffd` (U+FFFD) if set to true, otherwise returns an error\nif there are any invalid UTF-8 characters present.",
77797915
"required": false,
77807916
"default": true,
77817917
"type": [
@@ -8512,12 +8648,12 @@
85128648
},
85138649
{
85148650
"title": "Parse URL with internationalized domain name",
8515-
"source": "parse_url!(\"https://www.café.com\")",
8651+
"source": "parse_url!(\"https://www.caf\u00e9.com\")",
85168652
"return": "{\"scheme\":\"https\",\"username\":\"\",\"password\":\"\",\"host\":\"www.xn--caf-dma.com\",\"port\":null,\"path\":\"/\",\"query\":{},\"fragment\":null}"
85178653
},
85188654
{
85198655
"title": "Parse URL with mixed case internationalized domain name",
8520-
"source": "parse_url!(\"https://www.CAFé.com\")",
8656+
"source": "parse_url!(\"https://www.CAF\u00e9.com\")",
85218657
"return": "{\"scheme\":\"https\",\"username\":\"\",\"password\":\"\",\"host\":\"www.xn--caf-dma.com\",\"port\":null,\"path\":\"/\",\"query\":{},\"fragment\":null}"
85228658
}
85238659
],
@@ -9235,5 +9371,77 @@
92359371
"deprecated": false,
92369372
"pure": true
92379373
}
9374+
],
9375+
"Map": [
9376+
{
9377+
"anchor": "haversine",
9378+
"name": "haversine",
9379+
"category": "Map",
9380+
"description": "Calculates the [haversine](https://en.wikipedia.org/wiki/Haversine_formula) great-circle distance and bearing between two geographic coordinates.",
9381+
"notices": [],
9382+
"arguments": [
9383+
{
9384+
"name": "latitude1",
9385+
"description": "Latitude of the first point in decimal degrees.",
9386+
"required": true,
9387+
"type": [
9388+
"float"
9389+
]
9390+
},
9391+
{
9392+
"name": "longitude1",
9393+
"description": "Longitude of the first point in decimal degrees.",
9394+
"required": true,
9395+
"type": [
9396+
"float"
9397+
]
9398+
},
9399+
{
9400+
"name": "latitude2",
9401+
"description": "Latitude of the second point in decimal degrees.",
9402+
"required": true,
9403+
"type": [
9404+
"float"
9405+
]
9406+
},
9407+
{
9408+
"name": "longitude2",
9409+
"description": "Longitude of the second point in decimal degrees.",
9410+
"required": true,
9411+
"type": [
9412+
"float"
9413+
]
9414+
},
9415+
{
9416+
"name": "measurement_unit",
9417+
"description": "Unit for the returned distance: `kilometers` or `miles`.",
9418+
"required": false,
9419+
"type": [
9420+
"string"
9421+
],
9422+
"default": "kilometers"
9423+
}
9424+
],
9425+
"internal_failure_reasons": [],
9426+
"return": {
9427+
"types": [
9428+
"object"
9429+
]
9430+
},
9431+
"examples": [
9432+
{
9433+
"title": "Calculate distance and bearing in kilometers",
9434+
"source": "haversine(0.0, 0.0, 10.0, 10.0)",
9435+
"return": "{\"bearing\":44.561,\"distance\":1568.5227233}"
9436+
},
9437+
{
9438+
"title": "Calculate distance in miles",
9439+
"source": "haversine(0.0, 0.0, 10.0, 10.0, measurement_unit: \"miles\")",
9440+
"return": "{\"bearing\":44.561,\"distance\":974.6348468}"
9441+
}
9442+
],
9443+
"deprecated": false,
9444+
"pure": true
9445+
}
92389446
]
9239-
}
9447+
}

0 commit comments

Comments
 (0)