journal: Improve error handling, document filtering (#441)

* juornal: Improve error handling, document filtering

* Update docs

* Use single-quote in eslintrc

Author: ad-m

.eslintrc.js

@@ -1,5 +1,6 @@
 'use strict';
 
 module.exports = {
-    extends: '@hyperone'
+    extends: '@hyperone',
+    parser: 'babel-eslint',
 };

bin/journal/examples.md

@@ -10,34 +10,6 @@
 {{command_name}} credential password add --log my-server-log --name syslog --password my-strong-secret
 ```
 
-# Configure rsyslog
-
-Open or create a new configuration file for rsyslog:
-
-```
-sudo nano /etc/rsyslog.d/50-hyperone.conf
-```
-
-Paste in this configuration:
-
-```
-$template HyperOneFormat,"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [{log_id}:{secret}@HyperOne tag=\"Rsyslog\"]%msg%\n"
-
-*.* @@{log_id}.logarchive.{region}.hyperone.cloud:6514; HyperOneFormat
-```
-
-Replace the following values in the example:
-
-* ```{log_id}``` - ID of log. To identify available logs use ```{{scope}} log list```.
-* ```{secret}``` - The password added to given log. See example above how to create a write-only password.
-* ```{region}``` - Region where resource exists eg. ```pl-waw-1```
-
-Remember to restart rsyslog:
-
-```
-sudo service rsyslog restart
-```
-
 # Display today's log entries
 
 ```bash

bin/journal/filters.js

@@ -1,22 +1,20 @@
 'use strict';
 const Cli = require('lib/cli');
-
+const { get } = require('../../lib/transform');
 
 const handlers = {
-    '^([a-zA-Z]+) *= *(.+?)$' : (key, val, data) => data[key] == val,
-    '^([a-zA-Z]+) *> *(.+?)$' : (key, val, data) => data[key] > val,
-    '^([a-zA-Z]+) *< *(.+?)$' : (key, val, data) => data[key] < val,
-    '^([a-zA-Z]+) *~ *(.+?)$' : (key, val, data) => new RegExp(val).test(data[key]),
+    '^([a-zA-Z0-9\.]+)\s*=~\s*(.+?)$': (key, val, data) => new RegExp(val).test(get(data, key)),
+    '^([a-zA-Z0-9\.]+)\s*=\s*(.+?)$': (key, val, data) => get(data, key) == val,
+    '^([a-zA-Z0-9\.]+)\s*>\s*(.+?)$': (key, val, data) => get(data, key) > val,
+    '^([a-zA-Z0-9\.]+)\s*<\s*(.+?)$': (key, val, data) => get(data, key) < val,
 };
 
-
 const match = (filter, data) => {
-    for (const pattern in handlers) {
+    for (const [pattern, handler] of Object.entries(handlers)) {
         const res = new RegExp(pattern).exec(filter);
         if (!res) {
             continue;
         }
-        const handler = handlers[pattern];
         return handler(res[1], res[2], data);
     }
     throw Cli.error.cancelled(`Unable to parse query '${filter}'`);

bin/journal/format.js

@@ -32,17 +32,16 @@ const outputFormat = {
         header: () => {
         },
         row: (fields, row) => {
-            const values = fields.map(field => row[field]);
+            const values = fields.map(field => get(row, field));
             return values.map(value => {
                 if (typeof value === 'number' || typeof value === 'boolean') {
-                    return value;
+                    return `${value}`;
                 }
                 if (typeof value === 'string') {
                     return value.match(/\t/) ? `"${value}"` : value;
                 }
                 return '-';
             }).join('\t');
-
         },
     },
     csv: {
@@ -60,7 +59,7 @@ const outputOptions = {
     },
     fields: {
         type: (val) => val.split(','),
-        description: 'Fields displayed. Not apply to "jsonl" output.',
+        description: 'Fields displayed. Apply only to "tsv" & "csv" output.',
         defaultValue: default_fields.join(','),
     },
     filter: {

bin/journal/stream/examples.md

@@ -7,5 +7,17 @@
 # View live-stream of log entries for the Nginx application
 
 ```bash
-{{command_name}} --log my-server-log --follow --filter appName~nginx
+{{command_name}} --log my-server-log --follow --filter appName=~nginx
 ```
+
+# Filtering
+
+Parameter ```--filter``` accept following query format:
+
+* `{fieldName}=~{value}` - filter by regexp pattern. Example ```--filter 'message=~.*isAuthenticated: true.*'```
+* `{fieldName}={value}` - filter by exact match field to value. Example: ```--filter 'tag.containerId=1234'```
+* `{fieldName}={value}` - filter by values higher than ```{value}``
+* `{fieldName}={value}` - filter by values lower than ```{value}``
+
+```{fieldName}``` is name of property of log entry. It can contains any character in range of ```A-Za-z.``` .
+The ```.``` sign is specially treated and is used to refer to the nested properties of the object.

bin/journal/stream/index.js

@@ -79,19 +79,14 @@ module.exports = resource => {
             const url = `${resource.url(args)}/${args[resource.name]}/log?${qs.stringify(query)}`;
             let count = 0;
             const stream = await args.helpers.api.stream(url);
-            return new Promise((resolve, reject) => stream
-                .on('data', jsonl => {
-                    if (formatter.print_jsonl(jsonl)) {
-                        count += 1;
-                    }
-                    if (args.head && count >= args.head) {
-                        stream.end();
-                    }
-                })
-                .on('error', reject)
-                .on('end', resolve)
-                .resume()
-            );
+            for await (const jsonl of stream) {
+                if (formatter.print_jsonl(jsonl)) {
+                    count += 1;
+                }
+                if (args.head && count >= args.head) {
+                    stream.end();
+                }
+            }
         },
     });
 

bin/journal/tests.js

@@ -1,5 +1,5 @@
 'use strict';
-const fs = require('fs');
+const fs = require('fs').promises;
 const ava = require('ava');
 
 require('../../scope/h1');
@@ -27,8 +27,9 @@ ava.serial('journal logger & stream', async t => {
     try {
         await tests.run(`journal credential password add --journal ${journal.id} --name my-token --password ${token}`);
         await tests.run(`journal logger --journal ${journal.id} --token ${token} --log-file ${log_file}`);
+        await tests.delay(3 * 1000);
         await tests.run(`journal stream --head 1 --journal ${journal.id} --jsonl-file ${output_file}`);
-        const log_content = fs.readFileSync(output_file, 'utf-8');
+        const log_content = await fs.readFile(output_file, 'utf-8');
         t.true(!!log_content);
         t.true(JSON.parse(log_content).message === content);
     } finally {
@@ -48,8 +49,9 @@ ava.serial('journal logger & stream with tags', async t => {
     try {
         await tests.run(`journal logger --journal ${journal.id} --token ${token} --log-file ${log_file_with_tag} --tag host=123`);
         await tests.run(`journal logger --journal ${journal.id} --token ${token} --log-file ${log_file_without_tag}`);
+        await tests.delay(3 * 1000);
         await tests.run(`journal stream --journal ${journal.id} --jsonl-file ${output_file} --tag host=123`);
-        const log_content = fs.readFileSync(output_file, 'utf-8');
+        const log_content = await fs.readFile(output_file, 'utf-8');
         t.true(!!log_content);
         const logs = log_content.split('\n').filter(x => !!x).map(x => JSON.parse(x));
         t.true(logs.some(x => x.message === content_with_tag));

docs/journal.md

@@ -53,34 +53,6 @@ h1 journal create --name my-server-log
 h1 journal credential password add --log my-server-log --name syslog --password my-strong-secret
 ```
 
-#### Configure rsyslog
-
-Open or create a new configuration file for rsyslog:
-
-```
-sudo nano /etc/rsyslog.d/50-hyperone.conf
-```
-
-Paste in this configuration:
-
-```
-$template HyperOneFormat,"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [{log_id}:{secret}@HyperOne tag=\"Rsyslog\"]%msg%\n"
-
-*.* @@{log_id}.logarchive.{region}.hyperone.cloud:6514; HyperOneFormat
-```
-
-Replace the following values in the example:
-
-* ```{log_id}``` - ID of log. To identify available logs use ```h1 log list```.
-* ```{secret}``` - The password added to given log. See example above how to create a write-only password.
-* ```{region}``` - Region where resource exists eg. ```pl-waw-1```
-
-Remember to restart rsyslog:
-
-```
-sudo service rsyslog restart
-```
-
 #### Display today's log entries
 
 ```bash
@@ -294,9 +266,21 @@ h1 journal stream --log my-server-log
 #### View live-stream of log entries for the Nginx application
 
 ```bash
-h1 journal stream --log my-server-log --follow --filter appName~nginx
+h1 journal stream --log my-server-log --follow --filter appName=~nginx
 ```
 
+#### Filtering
+
+Parameter ```--filter``` accept following query format:
+
+* `{fieldName}=~{value}` - filter by regexp pattern. Example ```--filter 'message=~.*isAuthenticated: true.*'```
+* `{fieldName}={value}` - filter by exact match field to value. Example: ```--filter 'tag.containerId=1234'```
+* `{fieldName}={value}` - filter by values higher than ```{value}``
+* `{fieldName}={value}` - filter by values lower than ```{value}``
+
+```{fieldName}``` is name of property of log entry. It can contains any character in range of ```A-Za-z.``` .
+The ```.``` sign is specially treated and is used to refer to the nested properties of the object.
+
 ### Required arguments
 
 | Name | Default | Description |

package-lock.json

@@ -66,6 +66,7 @@
     "@hyperone/eslint-config": "^1.0.4",
     "ajv": "^6.10.2",
     "ava": "^2.4.0",
+    "babel-eslint": "^10.0.3",
     "dns-socket": "^4.2.0",
     "eslint": "^6.4.0",
     "fast-glob": "^3.0.4",