API v4.2.1- Added NGINX javascript support (#38)

* 20240214-01 Commit - Initial njs support

* 20240215-01 Commit - Initial njs support

* 20240216-01 Commit - njs support

Author: fabriziofiorucci

USAGE-v4.2.md

@@ -52,6 +52,57 @@ Locations `.declaration.http.servers[].locations[].uri` match modifiers in `.dec
 - *iregex* - case insensitive regex matching
 - *best* - case sensitive regex matching that halts any other location matching once a match is made
 
+### Javascript profiles ###
+
+NGINX Javascript profiles are defined in `.declaration.http.njs[]`:
+
+- `name` - the NJS profile name
+- `file.content` - the base64-encoded njs source code or the `http(s)://` URL of the file
+- `file.authentication.server[0].profile` - authentication profile name if `file.content` is a URL and the request must be authenticated
+
+### Javascript hooks ###
+
+NGINX Javascript hooks can be used in:
+
+- `.declaration.http.njs`
+  - Supported hooks:
+    - `js_preload_object'
+    - 'js_set`
+- `.declaration.http.server[].njs`
+  - Supported hooks:
+    - `js_preload_object'
+    - 'js_set`
+- `.declaration.http.server[].location[].njs`
+  - Supported hooks:
+    - `js_body_filter'
+    - 'js_content'
+    - 'js_header_filter'
+    - 'js_periodic'
+    - 'js_preload_object'
+    - 'js_set`
+
+Hooks invocation is:
+
+```
+"njs": [
+  {
+    "hook": {
+      "name": "<HOOK_NAME>",
+      "parameters": [
+        {
+          "name": "<HOOK_PARAMETER_NAME>",
+          "value": "<HOOK_PARAMETER_VALUE>"
+        }
+      ]
+    },
+    "profile": "<NGINX_JAVASCRIPT_PROFILE>",
+    "function": "<JAVASCRIPT_FUNCTION_NAME>"
+  }
+]
+```
+
+For detailed examples see the [Postman collection](/contrib/postman)
+
 ### API Gateway ###
 
 Swagger files and OpenAPI schemas can be used to automatically configure NGINX as an API Gateway. Developer portal creation is supported through [Redocly](https://redocly.com/)
@@ -88,11 +139,7 @@ is:
             "username": "{{nim_username}}",
             "password": "{{nim_password}}",
             "instancegroup": "{{nim_instancegroup}}",
-            "synctime": 0,
-            "modules": [
-                "ngx_http_js_module",
-                "ngx_stream_js_module"
-            ]
+            "synctime": 0
         }
     },
     "declaration": {

contrib/postman/NGINX Declarative API.postman_collection.json

@@ -5832,6 +5832,55 @@
 									"response": []
 								}
 							]
+						},
+						{
+							"name": "NGINX Javascript",
+							"item": [
+								{
+									"name": "NGINX Javascript test",
+									"event": [
+										{
+											"listen": "test",
+											"script": {
+												"exec": [
+													"var respData = JSON.parse(responseBody);",
+													"",
+													"tests[\"configUid is: \" +respData.configUid] = respData.configUid;",
+													"",
+													"pm.collectionVariables.set('configUid',respData.configUid);"
+												],
+												"type": "text/javascript"
+											}
+										}
+									],
+									"request": {
+										"method": "POST",
+										"header": [],
+										"body": {
+											"mode": "raw",
+											"raw": "{\n    \"output\": {\n        \"type\": \"nms\",\n        \"nms\": {\n            \"url\": \"{{nim_host}}\",\n            \"username\": \"{{nim_username}}\",\n            \"password\": \"{{nim_password}}\",\n            \"instancegroup\": \"{{nim_instancegroup}}\",\n            \"synctime\": 0,\n            \"modules\": [\n                \"ngx_http_js_module\",\n                \"ngx_stream_js_module\"\n            ]\n        }\n    },\n    \"declaration\": {\n        \"http\": {\n            \"servers\": [\n                {\n                    \"name\": \"Example HTTP server with Javascript\",\n                    \"resolver\": \"8.8.8.8\",\n                    \"names\": [\n                        \"njs-test.vm-test.ie.ff.lan\"\n                    ],\n                    \"listen\": {\n                        \"address\": \"0.0.0.0:80\"\n                    },\n                    \"log\": {\n                        \"access\": \"/var/log/nginx/njs-test.nginx.lab_access_log\",\n                        \"error\": \"/var/log/nginx/njs-test.nginx.lab_error_log\"\n                    },\n                    \"njs\": [\n                        {\n                            \"hook\": {\n                                \"type\": \"js_set\",\n                                \"js_set\": {\n                                    \"variable\": \"$serverVarSetByNjs\"\n                                }\n                            },\n                            \"profile\": \"njs_set_variable\",\n                            \"function\": \"njsSetVariable\"\n                        }\n                    ],\n                    \"headers\": {\n                        \"to_server\": {\n                            \"set\": [\n                                {\n                                    \"name\": \"X-Injected-Client-IP\",\n                                    \"value\": \"$remote_addr\"\n                                },\n                                {\n                                    \"name\": \"Host\",\n                                    \"value\": \"echo.free.beeceptor.com\"\n                                }\n                            ]\n                        }\n                    },\n                    \"locations\": [\n                        {\n                            \"uri\": \"/echo\",\n                            \"urimatch\": \"prefix\",\n                            \"upstream\": \"http://test_upstream\",\n                            \"headers\": {\n                                \"to_server\": {\n                                    \"set\": [\n                                        {\n                                            \"name\": \"X-Injected-Client-IP\",\n                                            \"value\": \"$remote_addr\"\n                                        },\n                                        {\n                                            \"name\": \"X-HTTP-Var-Set-By-Njs\",\n                                            \"value\": \"$httpVarSetByNjs\"\n                                        },\n                                        {\n                                            \"name\": \"X-Server-Var-Set-By-Njs\",\n                                            \"value\": \"$serverVarSetByNjs\"\n                                        },\n                                        {\n                                            \"name\": \"Host\",\n                                            \"value\": \"echo.free.beeceptor.com\"\n                                        }\n                                    ]\n                                }\n                            }\n                        },\n                        {\n                            \"uri\": \"/generatecontent\",\n                            \"urimatch\": \"prefix\",\n                            \"upstream\": \"http://test_upstream\",\n                            \"njs\": [\n                                {\n                                    \"hook\": {\n                                        \"type\": \"js_content\"\n                                    },\n                                    \"profile\": \"njs_set_content\",\n                                    \"function\": \"njsSetContent\"\n                                }\n                            ],\n                            \"headers\": {\n                                \"to_server\": {\n                                    \"set\": [\n                                        {\n                                            \"name\": \"X-Injected-Client-IP\",\n                                            \"value\": \"$remote_addr\"\n                                        },\n                                        {\n                                            \"name\": \"X-HTTP-Var-Set-By-Njs\",\n                                            \"value\": \"$httpVarSetByNjs\"\n                                        },\n                                        {\n                                            \"name\": \"X-Server-Var-Set-By-Njs\",\n                                            \"value\": \"$serverVarSetByNjs\"\n                                        },\n                                        {\n                                            \"name\": \"Host\",\n                                            \"value\": \"echo.free.beeceptor.com\"\n                                        }\n                                    ]\n                                }\n                            }\n                        }\n                    ]\n                }\n            ],\n            \"upstreams\": [\n                {\n                    \"name\": \"test_upstream\",\n                    \"origin\": [\n                        {\n                            \"server\": \"echo.free.beeceptor.com\"\n                        }\n                    ]\n                }\n            ],\n            \"njs\": [\n                {\n                    \"hook\": {\n                        \"type\": \"js_set\",\n                        \"js_set\": {\n                            \"variable\": \"$httpVarSetByNjs\"\n                        }\n                    },\n                    \"profile\": \"njs_set_variable\",\n                    \"function\": \"njsSetVariable\"\n                }\n            ],\n            \"njs_profiles\": [\n                {\n                    \"name\": \"njs_set_variable\",\n                    \"file\": {\n                        \"content\": \"ZnVuY3Rpb24gbmpzU2V0VmFyaWFibGUocikgewogICAgcmV0dXJuICJWYXJpYWJsZV9zZXRfYnlfamF2YXNjcmlwdCAtIFVSSSAiK3IudXJpOwp9CgpleHBvcnQgZGVmYXVsdCB7bmpzU2V0VmFyaWFibGV9Cgo=\"\n                    }\n                },\n                {\n                    \"name\": \"njs_set_content\",\n                    \"file\": {\n                        \"content\": \"ZnVuY3Rpb24gbmpzU2V0Q29udGVudChyKSB7CiAgci5yZXR1cm4oMjAwLCAiSGVsbG8gd29ybGQhXG4iKTsKfQoKZXhwb3J0IGRlZmF1bHQge25qc1NldENvbnRlbnR9Cg==\"\n                    }\n                }\n            ]\n        }\n    }\n}",
+											"options": {
+												"raw": {
+													"language": "json"
+												}
+											}
+										},
+										"url": {
+											"raw": "http://{{ncg_host}}:{{ncg_port}}/{{ngc_api_version}}/config",
+											"protocol": "http",
+											"host": [
+												"{{ncg_host}}"
+											],
+											"port": "{{ncg_port}}",
+											"path": [
+												"{{ngc_api_version}}",
+												"config"
+											]
+										}
+									},
+									"response": []
+								}
+							]
 						}
 					]
 				}

etc/config.toml

@@ -43,6 +43,7 @@ certs_dir = '/etc/nginx/ssl'
 devportal_dir = '/etc/nginx/devportal'
 auth_client_dir = '/etc/nginx/auth/client'
 auth_server_dir = '/etc/nginx/auth/server'
+njs_dir = '/etc/nginx/njs'
 
 
 # Time to wait to get status after committing a staged config

src/V4_2_CreateConfig.py

@@ -198,6 +198,36 @@ def createconfig(declaration: ConfigDeclaration, apiversion: str, runfromautosyn
                             all_auth_server_profiles.append(auth_profile['name'])
                             auxFiles['files'].append(authProfileConfigFile)
 
+        # NGINX Javascript profiles
+        all_njs_profiles = []
+        d_njs_files = v4_2.MiscUtils.getDictKey(d, 'declaration.http.njs_profiles')
+        if d_njs_files is not None:
+            for i in range(len(d_njs_files)):
+                njs_file = d_njs_files[i]
+                njs_filename = njs_file['name'].replace(' ','_')
+
+                status, content = v4_2.GitOps.getObjectFromRepo(object=njs_file['file'],
+                                                                authProfiles=d['declaration']['http'][
+                                                                    'authentication'])
+
+                if status != 200:
+                    return {"status_code": 422, "message": {"status_code": status, "message": content}}
+
+                njsAuxFile = {'contents': content['content'],
+                              'name': NcgConfig.config['nms']['njs_dir'] + '/' + njs_filename + '.js'}
+                auxFiles['files'].append(njsAuxFile)
+                all_njs_profiles.append(njs_filename)
+
+        # HTTP level Javascript hooks
+        d_http_njs_hooks = v4_2.MiscUtils.getDictKey(d, 'declaration.http.njs')
+        if d_http_njs_hooks is not None:
+            for i in range(len(d_http_njs_hooks)):
+                if d_http_njs_hooks[i]['profile'] not in all_njs_profiles:
+                    return {"status_code": 422,
+                            "message": {"status_code": status, "message":
+                                {"code": status,
+                                 "content": f"invalid njs profile [{d_http_njs_hooks[i]['profile']}] in HTTP declaration, must be one of {all_njs_profiles}"}}}
+
         # Parse HTTP servers
         d_servers = v4_2.MiscUtils.getDictKey(d, 'declaration.http.servers')
         if d_servers is not None:
@@ -206,6 +236,15 @@ def createconfig(declaration: ConfigDeclaration, apiversion: str, runfromautosyn
             for server in d_servers:
                 serverSnippet = ''
 
+                # Server level Javascript hooks
+                if server['njs']:
+                    for i in range(len(server['njs'])):
+                        if server['njs'][i]['profile'] not in all_njs_profiles:
+                            return {"status_code": 422,
+                                    "message": {"status_code": status, "message":
+                                        {"code": status,
+                                         "content": f"invalid njs profile [{server['njs'][i]['profile']}] in server [{server['name']}], must be one of {all_njs_profiles}"}}}
+
                 if server['snippet']:
                     status, serverSnippet = v4_2.GitOps.getObjectFromRepo(object = server['snippet'], authProfiles = d['declaration']['http']['authentication'], base64Encode = False)
 
@@ -215,6 +254,16 @@ def createconfig(declaration: ConfigDeclaration, apiversion: str, runfromautosyn
                     serverSnippet = serverSnippet['content']
 
                 for loc in server['locations']:
+
+                    # Location level Javascript hooks
+                    if loc['njs']:
+                        for i in range(len(loc['njs'])):
+                            if loc['njs'][i]['profile'] not in all_njs_profiles:
+                                return {"status_code": 422,
+                                        "message": {"status_code": status, "message":
+                                            {"code": status,
+                                             "content": f"invalid njs profile [{loc['njs'][i]['profile']}] in location [{loc['uri']}], must be one of {all_njs_profiles}"}}}
+
                     if loc['snippet']:
                         status, snippet = v4_2.GitOps.getObjectFromRepo(object = loc['snippet'], authProfiles = d['declaration']['http']['authentication'])
 

src/V4_2_NginxConfigDeclaration.py

@@ -347,6 +347,7 @@ class Location(BaseModel, extra="forbid"):
     snippet: Optional[ObjectFromSourceOfTruth] = {}
     authentication: Optional[LocationAuth] = {}
     headers: Optional[LocationHeaders]= {}
+    njs: Optional[List[NjsHookLocation]] = []
 
     @model_validator(mode='after')
     def check_type(self) -> 'Location':
@@ -369,6 +370,70 @@ class ObjectFromSourceOfTruth(BaseModel, extra="forbid"):
     authentication: Optional[List[LocationAuthServer]] = []
 
 
+class NjsHook_js_body_filter(BaseModel, extra="forbid"):
+    buffer_type: Optional[str] = ""
+
+
+class NjsHook_js_periodic(BaseModel, extra="forbid"):
+    interval: Optional[str] = ""
+    jitter: Optional[int] = 0
+    worker_affinity: Optional[str] = ""
+
+
+class NjsHook_js_preload_object(BaseModel, extra="forbid"):
+    file: str
+
+
+class NjsHook_js_set(BaseModel, extra="forbid"):
+    variable: str
+
+
+class NjsHookHttpServerDetails(BaseModel, extra="forbid"):
+    type: str
+    js_preload_object: Optional[NjsHook_js_preload_object] = {}
+    js_set: Optional[NjsHook_js_set] = {}
+
+    @model_validator(mode='after')
+    def check_type(self) -> 'NjsHookHttpServerDetails':
+        _type = self.type
+
+        valid = ['js_preload_object', 'js_set']
+        if _type not in valid:
+            raise ValueError(f"Invalid hook [{_type}] must be one of {str(valid)}")
+
+        return self
+
+
+class NjsHookLocationDetails(BaseModel, extra="forbid"):
+    type: str
+    js_preload_object: Optional[NjsHook_js_preload_object] = {}
+    js_set: Optional[NjsHook_js_set] = {}
+    js_body_filter: Optional[NjsHook_js_body_filter] = {}
+    js_periodic: Optional[NjsHook_js_periodic] = {}
+
+    @model_validator(mode='after')
+    def check_type(self) -> 'NjsHookLocationDetails':
+        _type = self.type
+
+        valid = ['js_body_filter', 'js_content', 'js_header_filter', 'js_periodic', 'js_preload_object', 'js_set']
+        if _type not in valid:
+            raise ValueError(f"Invalid hook [{_type}] must be one of {str(valid)}")
+
+        return self
+
+class NjsHookHttpServer(BaseModel, extra="forbid"):
+    hook: NjsHookHttpServerDetails
+    profile: str
+    function: str
+
+
+
+class NjsHookLocation(BaseModel, extra="forbid"):
+    hook: NjsHookLocationDetails
+    profile: str
+    function: str
+
+
 class Server(BaseModel, extra="forbid"):
     name: str
     names: Optional[List[str]] = []
@@ -379,6 +444,7 @@ class Server(BaseModel, extra="forbid"):
     app_protect: Optional[AppProtect] = {}
     snippet: Optional[ObjectFromSourceOfTruth] = {}
     headers: Optional[LocationHeaders] = {}
+    njs: Optional[List[NjsHookHttpServer]] = []
 
 
 class L4Server(BaseModel, extra="forbid"):
@@ -520,6 +586,11 @@ class Authentication(BaseModel, extra="forbid"):
     server: Optional[List[Authentication_Server]] = []
 
 
+class NjsFile(BaseModel, extra="forbid"):
+    name: str
+    file: ObjectFromSourceOfTruth
+
+
 class Http(BaseModel, extra="forbid"):
     servers: Optional[List[Server]] = []
     upstreams: Optional[List[Upstream]] = []
@@ -529,6 +600,8 @@ class Http(BaseModel, extra="forbid"):
     maps: Optional[List[Map]] = []
     snippet: Optional[ObjectFromSourceOfTruth] = {}
     authentication: Optional[Authentication] = {}
+    njs: Optional[List[NjsHookHttpServer]] = []
+    njs_profiles: Optional[List[NjsFile]] = []
 
 
 class Declaration(BaseModel, extra="forbid"):