Merge pull request #4662 from bruntib/async_store_3

Async store 3

Author: bruntib

codechecker_common/compatibility/multiprocessing.py

@@ -14,14 +14,17 @@
 # pylint: disable=unused-import
 if sys.platform in ["darwin", "win32"]:
     from multiprocess import \
-        Pool, Process, \
+        Pipe, Pool, Process, \
         Queue, \
         Value, \
         cpu_count
+    from multiprocess.managers import SyncManager
 else:
     from concurrent.futures import ProcessPoolExecutor as Pool
     from multiprocessing import \
+        Pipe, \
         Process, \
         Queue, \
         Value, \
         cpu_count
+    from multiprocessing.managers import SyncManager

codechecker_common/util.py

@@ -18,7 +18,7 @@
 import os
 import pathlib
 import random
-from typing import List, TextIO
+from typing import List, TextIO, Union
 
 import portalocker
 
@@ -59,7 +59,10 @@ def chunks(iterator, n):
         yield itertools.chain([first], rest_of_chunk)
 
 
-def load_json(path: str, default=None, lock=False, display_warning=True):
+def load_json(path: Union[str, pathlib.Path],
+              default=None,
+              lock=False,
+              display_warning=True):
     """
     Load the contents of the given file as a JSON and return it's value,
     or default if the file can't be loaded.
@@ -218,3 +221,16 @@ def generate_random_token(num_bytes: int = 32) -> str:
          for _ in range(0, -(num_bytes // -64))])
     idx = random.randrange(0, len(hash_value) - num_bytes + 1)
     return hash_value[idx:(idx + num_bytes)]
+
+
+def format_size(num: float, suffix: str = 'B') -> str:
+    """
+    Pretty print storage units.
+    Source: http://stackoverflow.com/questions/1094841/
+        reusable-library-to-get-human-readable-version-of-file-size
+    """
+    for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi', 'Ri']:
+        if abs(num) < 1024.0:
+            return f"{num:3.1f} {unit}{suffix}"
+        num /= 1024.0
+    return f"{num:.1f} Qi{suffix}"

docs/web/user_guide.md

@@ -473,6 +473,19 @@ optional arguments:
                         is given, the longest match will be removed. You may
                         also use Unix shell-like wildcards (e.g.
                         '/*/jsmith/').
+  --detach              Runs `store` in fire-and-forget mode: exit immediately
+                        once the server accepted the analysis reports for
+                        storing, without waiting for the server-side data
+                        processing to conclude. Doing this is generally not
+                        recommended, as the client will never be notified of
+                        potential processing failures, and there is no easy way
+                        to wait for the successfully stored results to become
+                        available server-side for potential further processing
+                        (e.g., `CodeChecker cmd diff`). However, using
+                        '--detach' can significantly speed up large-scale
+                        monitoring analyses where access to the results by a
+                        tool is not a goal, such as in the case of non-gating
+                        CI systems.
   --config CONFIG_FILE  Allow the configuration from an explicit configuration
                         file. The values configured in the config file will
                         overwrite the values set in the command line.

web/api/codechecker_api_shared.thrift

@@ -14,14 +14,35 @@ enum Ternary {
 }
 
 enum ErrorCode {
-  DATABASE,
-  IOERROR,
-  GENERAL,
-  AUTH_DENIED,   // Authentication denied. We do not allow access to the service.
-  UNAUTHORIZED,  // Authorization denied. User does not have right to perform an action.
-  API_MISMATCH,  // The client attempted to query an API version that is not supported by the server.
-  SOURCE_FILE,   // The client sent a source code which contains errors (e.g., source code comment errors).
-  REPORT_FORMAT, // The client sent a report with wrong format (e.g., report annotation has bad type in a .plist).
+  // Any other sort of error encountered during RPC execution.
+  GENERAL  = 2,
+
+  // Executing the request triggered a database-level fault, constraint violation.
+  DATABASE = 0,
+
+  // The request is malformed or an internal I/O operation failed.
+  IOERROR  = 1,
+
+  // Authentication denied. We do not allow access to the service.
+  AUTH_DENIED = 3,
+
+  // User does not have the necessary rights to perform an action.
+  UNAUTHORIZED = 4,
+
+  // The client attempted to query an API version that is not supported by the
+  // server.
+  API_MISMATCH = 5,
+
+  // REMOVED IN API v6.59 (CodeChecker v6.25.0)!
+  // Previously sent by report_server.thrift/codeCheckerDBAccess::massStoreRun()
+  // when the client uploaded a source file which contained errors, such as
+  // review status source-code-comment errors.
+  /* SOURCE_FILE = 6, */ // Never reuse the value of the enum constant!
+
+  // REMOVED IN API v6.59 (CodeChecker v6.25.0)!
+  // Previously sent by report_server.thrift/codeCheckerDBAccess::massStoreRun()
+  // when the client uploaded a report with annotations that had invalid types.
+  /* REPORT_FORMAT = 7, */ // Never reuse the value of the enum constant!
 }
 
 exception RequestFailed {