Merge branch 'master' of github.com:StackStorm/st2 into optimize_escaped_dict_fields

Author: Kami

CHANGELOG.rst

@@ -187,6 +187,18 @@ Changed
 
 * Updated cryptography dependency to version 3.3.2 to avoid CVE-2020-36242 (security) #5151
 
+* Update most of the code in the StackStorm API and services layer to utilize ``orjson`` library
+  for serializing and de-serializing json.
+
+  That should result in better json serialization and deserialization performance.
+
+  The change should be fully backward compatible, only difference is that API JSON responses now
+  won't be indented using 4 spaces by default (indenting adds unnecessary overhead and if needed,
+  the response can be pretty formatted on the client side using ``jq`` or similar). (improvement)
+  #5153
+
+  Contributed by @Kami
+
 Fixed
 ~~~~~
 

Makefile

@@ -530,7 +530,10 @@ micro-benchmarks: requirements .micro-benchmarks
 	. $(VIRTUALENV_DIR)/bin/activate; pytest --benchmark-only --benchmark-name=short --benchmark-columns=min,max,mean,stddev,median,ops,rounds --benchmark-group-by=group,param:fixture_file -s -v st2common/benchmarks/micro/test_mongo_field_types.py -k "test_save_large_execution"
 	. $(VIRTUALENV_DIR)/bin/activate; pytest --benchmark-only --benchmark-name=short --benchmark-columns=min,max,mean,stddev,median,ops,rounds --benchmark-group-by=group,param:fixture_file -s -v st2common/benchmarks/micro/test_mongo_field_types.py -k "test_read_large_execution"
 	. $(VIRTUALENV_DIR)/bin/activate; pytest --benchmark-only --benchmark-name=short --benchmark-columns=min,max,mean,stddev,median,ops,rounds --benchmark-group-by=group,param:fixture_file -s -v st2common/benchmarks/micro/test_mongo_field_types.py -k "test_save_multiple_fields"
-
+	. $(VIRTUALENV_DIR)/bin/activate; pytest --benchmark-only --benchmark-name=short --benchmark-columns=min,max,mean,stddev,median,ops,rounds --benchmark-group-by=group,param:dict_keys_count_and_depth -s -v st2common/benchmarks/micro/test_fast_deepcopy.py -k "test_fast_deepcopy_with_dict_values"
+	. $(VIRTUALENV_DIR)/bin/activate; pytest --benchmark-only --benchmark-name=short --benchmark-columns=min,max,mean,stddev,median,ops,rounds --benchmark-group-by=group,param:fixture_file -s -v st2common/benchmarks/micro/test_fast_deepcopy.py -k "test_fast_deepcopy_with_json_fixture_file"
+	. $(VIRTUALENV_DIR)/bin/activate; pytest --benchmark-only --benchmark-name=short --benchmark-columns=min,max,mean,stddev,median,ops,rounds --benchmark-group-by=group,param:fixture_file,param:indent_sort_keys_tuple -s -v st2common/benchmarks/micro/test_json_serialization_and_deserialization.py  -k "test_json_dumps"
+	. $(VIRTUALENV_DIR)/bin/activate; pytest --benchmark-only --benchmark-name=short --benchmark-columns=min,max,mean,stddev,median,ops,rounds --benchmark-group-by=group,param:fixture_file -s -v st2common/benchmarks/micro/test_json_serialization_and_deserialization.py  -k "test_json_loads"
 
 .PHONY: .cleanmongodb
 .cleanmongodb:

contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py

@@ -48,7 +48,7 @@
 from st2common.util import jinja as jinja_utils
 from st2common.util import param as param_utils
 from st2common.util.config_loader import get_config
-from st2common.util.ujson import fast_deepcopy
+from st2common.util.deep_copy import fast_deepcopy_dict
 
 __all__ = ["ActionChainRunner", "ChainHolder", "get_runner", "get_metadata"]
 
@@ -85,7 +85,7 @@ def init_vars(self, action_parameters, action_context=None):
             )
 
     def restore_vars(self, ctx_vars):
-        self.vars.update(fast_deepcopy(ctx_vars))
+        self.vars.update(fast_deepcopy_dict(ctx_vars))
 
     def validate(self):
         """

contrib/runners/local_runner/tests/integration/test_localrunner.py

@@ -453,7 +453,7 @@ def test_script_with_parameters_parameter_serialization(self):
         self.assertIn("PARAM_FLOAT=2.55", result["stdout"])
         self.assertIn("PARAM_BOOLEAN=1", result["stdout"])
         self.assertIn("PARAM_LIST=a,b,c", result["stdout"])
-        self.assertIn('PARAM_OBJECT={"foo": "bar"}', result["stdout"])
+        self.assertIn('PARAM_OBJECT={"foo":"bar"}', result["stdout"])
 
         action_parameters = {
             "param_string": "test string",
@@ -515,12 +515,12 @@ def test_script_with_parameters_parameter_serialization(self):
         self.assertIn("PARAM_FLOAT=2.55", result["stdout"])
         self.assertIn("PARAM_BOOLEAN=1", result["stdout"])
         self.assertIn("PARAM_LIST=a,b,c", result["stdout"])
-        self.assertIn('PARAM_OBJECT={"foo": "bar"}', result["stdout"])
+        self.assertIn('PARAM_OBJECT={"foo":"bar"}', result["stdout"])
 
         output_dbs = ActionExecutionOutput.query(output_type="stdout")
         self.assertEqual(len(output_dbs), 6)
         self.assertEqual(output_dbs[0].data, "PARAM_STRING=test string\n")
-        self.assertEqual(output_dbs[5].data, 'PARAM_OBJECT={"foo": "bar"}\n')
+        self.assertEqual(output_dbs[5].data, 'PARAM_OBJECT={"foo":"bar"}\n')
 
         output_dbs = ActionExecutionOutput.query(output_type="stderr")
         self.assertEqual(len(output_dbs), 0)

contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py

@@ -35,7 +35,7 @@
 from st2common.services import action as ac_svc
 from st2common.services import workflows as wf_svc
 from st2common.util import api as api_util
-from st2common.util import ujson
+from st2common.util import deep_copy
 
 __all__ = ["OrquestaRunner", "get_runner", "get_metadata"]
 
@@ -59,7 +59,7 @@ def _get_notify_config(self):
         )
 
     def _construct_context(self, wf_ex):
-        ctx = ujson.fast_deepcopy(self.context)
+        ctx = deep_copy.fast_deepcopy_dict(self.context)
         ctx["workflow_execution"] = str(wf_ex.id)
 
         return ctx

contrib/runners/python_runner/python_runner/python_action_wrapper.py

@@ -28,6 +28,12 @@
 
 import distutils.sysconfig
 
+# NOTE: We intentionally use orjson directly here instead of json_encode - orjson.dumps relies
+# on config option which we don't parse for the action wrapper since it speeds things down - action
+# wrapper should rely on as little imports as possible to make Python runner executions fast -
+# that's very important.
+import orjson
+
 # Note: This work-around is required to fix the issue with other Python modules which live
 # inside this directory polluting and masking sys.path for Python runner actions.
 # Since this module is ran as a Python script inside a subprocess, directory where the script
@@ -46,7 +52,6 @@
     sys.path.insert(0, distutils.sysconfig.get_python_lib())
 
 import sys
-import json
 import argparse
 
 import six
@@ -233,19 +238,25 @@ def run(self):
             # Special case if result object is not JSON serializable - aka user wanted to return a
             # non-simple type (e.g. class instance or other non-JSON serializable type)
             try:
-                json.dumps(action_output["result"])
-            except TypeError:
+                orjson.dumps(action_output["result"])
+            except (TypeError, orjson.JSONDecodeError):
                 action_output["result"] = str(action_output["result"])
 
         try:
-            print_output = json.dumps(action_output)
+            print_output = orjson.dumps(action_output)
         except Exception:
-            print_output = str(action_output)
+            print_output = str(action_output).encode("utf-8")
+
+        # Data is bytes so we use sys.stdout.buffer which works with bytes and not sys.stdout
+        # which works with strings / unicodes.
+        # This way it also works correctly with unicode sequences.
+        # Technically we could also write to sys.stdout, but this would require additional
+        # conversion back and forth
 
         # Print output to stdout so the parent can capture it
-        sys.stdout.write(ACTION_OUTPUT_RESULT_DELIMITER)
-        sys.stdout.write(print_output + "\n")
-        sys.stdout.write(ACTION_OUTPUT_RESULT_DELIMITER)
+        sys.stdout.buffer.write(ACTION_OUTPUT_RESULT_DELIMITER.encode("utf-8"))
+        sys.stdout.buffer.write(print_output + b"\n")
+        sys.stdout.buffer.write(ACTION_OUTPUT_RESULT_DELIMITER.encode("utf-8"))
         sys.stdout.flush()
 
     def _get_action_instance(self):
@@ -317,9 +328,9 @@ def _get_action_instance(self):
     )
     args = parser.parse_args()
 
-    config = json.loads(args.config) if args.config else {}
+    config = orjson.loads(args.config) if args.config else {}
     user = args.user
-    parent_args = json.loads(args.parent_args) if args.parent_args else []
+    parent_args = orjson.loads(args.parent_args) if args.parent_args else []
     log_level = args.log_level
 
     if not isinstance(config, dict):
@@ -330,7 +341,7 @@ def _get_action_instance(self):
     if args.parameters:
         LOG.debug("Getting parameters from argument")
         args_parameters = args.parameters
-        args_parameters = json.loads(args_parameters) if args_parameters else {}
+        args_parameters = orjson.loads(args_parameters) if args_parameters else {}
         parameters.update(args_parameters)
 
     if args.stdin_parameters:
@@ -349,7 +360,7 @@ def _get_action_instance(self):
         stdin_data = sys.stdin.readline().strip()
 
         try:
-            stdin_parameters = json.loads(stdin_data)
+            stdin_parameters = orjson.loads(stdin_data)
             stdin_parameters = stdin_parameters.get("parameters", {})
         except Exception as e:
             msg = (

contrib/runners/python_runner/python_runner/python_runner.py

@@ -18,7 +18,6 @@
 import os
 import re
 import sys
-import json
 import uuid
 import functools
 from subprocess import list2cmdline
@@ -54,6 +53,8 @@
 from st2common.util.shell import quote_unix
 from st2common.services.action import store_execution_output_data
 from st2common.runners.utils import make_read_and_store_stream_func
+from st2common.util.jsonify import json_decode
+from st2common.util.jsonify import json_encode
 
 from python_runner import python_action_wrapper
 
@@ -134,7 +135,7 @@ def run(self, action_parameters):
         LOG.debug("Getting user.")
         user = self.get_user()
         LOG.debug("Serializing parameters.")
-        serialized_parameters = json.dumps(
+        serialized_parameters = json_encode(
             action_parameters if action_parameters else {}
         )
         LOG.debug("Getting virtualenv_path.")
@@ -166,9 +167,9 @@ def run(self, action_parameters):
         LOG.debug("Setting args.")
 
         if self._use_parent_args:
-            parent_args = json.dumps(sys.argv[1:])
+            parent_args = json_encode(sys.argv[1:])
         else:
-            parent_args = json.dumps([])
+            parent_args = json_encode([])
 
         args = [
             python_path,
@@ -197,7 +198,7 @@ def run(self, action_parameters):
             args.append("--parameters=%s" % (serialized_parameters))
 
         if self._config:
-            args.append("--config=%s" % (json.dumps(self._config)))
+            args.append("--config=%s" % (json_encode(self._config)))
 
         if self._log_level != PYTHON_RUNNER_DEFAULT_LOG_LEVEL:
             # We only pass --log-level parameter if non default log level value is specified
@@ -362,7 +363,7 @@ def _get_output_values(self, exit_code, stdout, stderr, timed_out):
         # Parse the serialized action result object (if available)
         if action_result:
             try:
-                action_result = json.loads(action_result)
+                action_result = json_decode(action_result)
             except Exception as e:
                 # Failed to de-serialize the result, probably it contains non-simple type or similar
                 LOG.warning(

fixed-requirements.txt

@@ -63,5 +63,5 @@ python-dateutil==2.8.0
 python-statsd==2.1.0
 prometheus_client==0.1.1
 ujson==1.35
-orjson==3.4.8
+orjson==3.5.0
 zipp>=0.5,<=1.0.0

requirements.txt

@@ -36,7 +36,7 @@ networkx==1.11
 nose
 nose-parallel==0.3.1
 nose-timer==0.7.5
-orjson==3.4.8
+orjson==3.5.0
 oslo.config<1.13,>=1.12.1
 oslo.utils<5.0,>=4.0.0
 paramiko==2.7.1

st2api/st2api/controllers/v1/inquiries.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 
 import copy
-import json
 
 import six
 from oslo_config import cfg
@@ -32,6 +31,7 @@
 from st2common.rbac.backends import get_rbac_backend
 from st2common import router as api_router
 from st2common.services import inquiry as inquiry_service
+from st2common.util.jsonify import json_decode
 
 
 __all__ = ["InquiriesController"]
@@ -89,7 +89,7 @@ def get_all(
         # a list of dicts, and then individually convert these to InquiryResponseAPI instances
         inquiries = [
             inqy_api_models.InquiryResponseAPI.from_model(raw_inquiry, skip_db=True)
-            for raw_inquiry in json.loads(raw_inquiries.body)
+            for raw_inquiry in json_decode(raw_inquiries.body)
         ]
 
         # Repackage into Response with correct headers