Merge pull request #4645 from barnabasdomozi/config_directory_rework

Rework config_directory

Author: bruntib

docs/web/products.md

@@ -158,7 +158,7 @@ database arguments:
   is the server which will make the database connection.
 
   --sqlite SQLITE_FILE  Path of the SQLite database file to use. All paths will
-                        be relative to the server's <CONFIG_DIRECTORY>. If an 
+                        be relative to the server's <WORKSPACE_DIRECTORY>. If an 
                         SQLite server needs to have a different directory, use
                         symlinks inside the config directory.
                         (default: <ENDOPINT>.sqlite)

docs/web/user_guide.md

@@ -164,9 +164,7 @@ optional arguments:
   -h, --help            show this help message and exit
   -w WORKSPACE, --workspace WORKSPACE
                         Directory where CodeChecker can store analysis result
-                        related data, such as the database. (Cannot be
-                        specified at the same time with '--sqlite' or
-                        '--config-directory'.) (default:
+                        related data, such as the database. (default:
                         /home/<username>/.codechecker)
   -f CONFIG_DIRECTORY, --config-directory CONFIG_DIRECTORY
                         Directory where CodeChecker server should read server-
@@ -215,7 +213,7 @@ optional arguments:
 
 configuration database arguments:
   --sqlite SQLITE_FILE  Path of the SQLite database file to use. (default:
-                        <CONFIG_DIRECTORY>/config.sqlite)
+                        <WORKSPACE_DIRECTORY>/config.sqlite)
   --postgresql          Specifies that a PostgreSQL database is to be used
                         instead of SQLite. See the "PostgreSQL arguments"
                         section on how to configure the database connection.

web/client/codechecker_client/cli/cmd.py

@@ -779,7 +779,7 @@ def __register_add(parser):
                              required=False,
                              help="Path of the SQLite database file to use. "
                                   "All paths will be relative to the server's "
-                                  "<CONFIG_DIRECTORY>. If an SQLite server "
+                                  "<WORKSPACE_DIRECTORY>. If an SQLite server "
                                   "needs to have a different directory, use "
                                   "symlinks inside the config directory.")
 

web/server/codechecker_server/api/product_server.py

@@ -294,7 +294,7 @@ def getProductConfiguration(self, product_id):
                 # Strip the config directory from the path, to allow for
                 # easier editing
                 config_dir = \
-                    os.path.normpath(self.__server.config_directory) + "/"
+                    os.path.normpath(self.__server.workspace_directory) + "/"
                 if args['sqlite'].startswith(config_dir):
                     db_name = args['sqlite'][len(config_dir):]
                 else:
@@ -428,8 +428,8 @@ def addProduct(self, product):
                     codechecker_api_shared.ttypes.ErrorCode.DATABASE,
                     "SQLite database must be given by relative path!")
 
-            dbc.database = path_for_fake_root(os.path.join("/", dbc.database),
-                                              self.__server.config_directory)
+            dbc.database = path_for_fake_root(os.path.join(
+                "/", dbc.database), self.__server.workspace_directory)
 
         # Check if the database is already in use by another product.
         db_in_use = self.__server.is_database_used(product)
@@ -610,7 +610,7 @@ def editProduct(self, product_id, new_config):
                         "SQLite database must be given by relative path!")
                 dbc.database = path_for_fake_root(
                     os.path.join("/", dbc.database),
-                    self.__server.config_directory)
+                    self.__server.workspace_directory)
 
             # Some values come encoded as Base64, decode these.
             displayed_name = convert.from_b64(new_config.displayedName_b64) \

web/server/codechecker_server/cli/server.py

@@ -85,14 +85,11 @@ def add_arguments_to_parser(parser):
                         default=default_workspace,
                         required=False,
                         help="Directory where CodeChecker can store analysis "
-                             "result related data, such as the database. "
-                             "(Cannot be specified at the same time with "
-                             "'--sqlite' or '--config-directory'.)")
+                             "result related data, such as the database.")
 
     parser.add_argument('-f', '--config-directory',
                         type=str,
                         dest="config_directory",
-                        default=default_workspace,
                         required=False,
                         help="Directory where CodeChecker server should read "
                              "server-specific configuration (such as "
@@ -169,9 +166,6 @@ def add_arguments_to_parser(parser):
                          type=str,
                          dest="sqlite",
                          metavar='SQLITE_FILE',
-                         default=os.path.join(
-                             '<CONFIG_DIRECTORY>',
-                             "config.sqlite"),
                          required=False,
                          help="Path of the SQLite database file to use.")
 
@@ -373,54 +367,24 @@ def arg_match(options):
             # is intended later on.
             delattr(args, 'not_host_only')
 
-        # --workspace and --sqlite cannot be specified either, as
-        # both point to a database location.
-        options = ['--sqlite', '--workspace']
-        options_short = ['--sqlite', '-w']
-        if set(arg_match(options)) == set(options) or \
-                set(arg_match(options_short)) == set(options_short):
-            parser.error("argument --sqlite: not allowed with "
-                         "argument --workspace")
-
-        # --workspace and --config-directory also aren't allowed together now,
-        # the latter one is expected to replace the earlier.
-        options = ['--config-directory', '--workspace']
-        options_short = ['--config-directory', '-w']
-        if set(arg_match(options)) == set(options) or \
-                set(arg_match(options_short)) == set(options_short):
-            parser.error("argument --config-directory: not allowed with "
-                         "argument --workspace")
-
-        # If workspace is specified, sqlite is workspace/config.sqlite
-        # and config_directory is the workspace directory.
-        if arg_match(['--workspace', '-w']):
+        # If config_directory is not specified, it will be the same
+        # as the workspace directory.
+        if not args.config_directory:
             args.config_directory = args.workspace
+
+        # If sqlite path is not specified, it will be set to
+        # workspace/config.sqlite
+        if not args.sqlite:
             args.sqlite = os.path.join(args.workspace,
                                        'config.sqlite')
-            setattr(args, 'dbdatadir', os.path.join(args.workspace,
-                                                    'pgsql_data'))
-
-        # Workspace should not exist as a Namespace key.
-        delattr(args, 'workspace')
-
-        if '<CONFIG_DIRECTORY>' in args.sqlite:
-            # Replace the placeholder variable with the actual value.
-            args.sqlite = args.sqlite.replace('<CONFIG_DIRECTORY>',
-                                              args.config_directory)
 
         # Convert relative sqlite file path to absolute.
-        if 'sqlite' in args:
-            args.sqlite = os.path.abspath(args.sqlite)
+        args.sqlite = os.path.abspath(args.sqlite)
 
         if 'postgresql' not in args:
             # Later called database modules need the argument to be actually
             # present, even though the default is suppressed in the optstring.
             setattr(args, 'postgresql', False)
-
-            # This is not needed by the database starter as we are
-            # running SQLite.
-            if 'dbdatadir' in args:
-                delattr(args, 'dbdatadir')
         else:
             # If --postgresql is given, --sqlite is useless.
             delattr(args, 'sqlite')
@@ -429,6 +393,10 @@ def arg_match(options):
         if "list" in args or "stop" in args or "stop_all" in args:
             setattr(args, "instance_manager", True)
 
+        # Log directories
+        LOG.info(f"Workspace directory: {args.workspace}")
+        LOG.info(f"Config directory: {args.config_directory}")
+
         # If everything is fine, do call the handler for the subcommand.
         return main(args)
 
@@ -808,7 +776,7 @@ def __instance_management(args):
             if 'stop' in args and \
                 not (i['port'] == args.view_port and
                      os.path.abspath(i['workspace']) ==
-                     os.path.abspath(args.config_directory)):
+                     os.path.abspath(args.workspace)):
                 continue
 
             try:
@@ -836,7 +804,7 @@ def __reload_config(args):
         if 'reload' in args and \
                 not (i['port'] == args.view_port and
                      os.path.abspath(i['workspace']) ==
-                     os.path.abspath(args.config_directory)):
+                     os.path.abspath(args.workspace)):
             continue
 
         try:
@@ -890,13 +858,6 @@ def server_init_start(args):
     if not host_check.check_zlib():
         raise ModuleNotFoundError("zlib is not available on the system!")
 
-    # WARNING
-    # In case of SQLite args.dbaddress default value is used
-    # for which the is_localhost should return true.
-    if is_localhost(args.dbaddress) and \
-            not os.path.exists(args.config_directory):
-        os.makedirs(args.config_directory)
-
     # Make sure the SQLite file can be created if it not exists.
     if 'sqlite' in args and \
             not os.path.isdir(os.path.dirname(args.sqlite)):
@@ -907,7 +868,7 @@ def server_init_start(args):
                  "option. The server will ask for users to authenticate!")
 
     context = webserver_context.get_context()
-    context.codechecker_workspace = args.config_directory
+    context.codechecker_workspace = args.workspace
     context.db_username = args.dbusername
 
     environ = env.extend(context.path_env_extra,
@@ -993,8 +954,8 @@ def server_init_start(args):
 
     # Create the main database link from the arguments passed over the
     # command line.
-    cfg_dir = os.path.abspath(args.config_directory)
-    default_product_path = os.path.join(cfg_dir, 'Default.sqlite')
+    workspace_dir = os.path.abspath(args.workspace)
+    default_product_path = os.path.join(workspace_dir, 'Default.sqlite')
     create_default_product = 'sqlite' in args and \
                              not os.path.exists(default_product_path)
 
@@ -1073,6 +1034,7 @@ def server_init_start(args):
 
     try:
         return server.start_server(args.config_directory,
+                                   args.workspace,
                                    package_data,
                                    args.view_port,
                                    cfg_sql_server,
@@ -1098,17 +1060,21 @@ def main(args):
     Setup a logger server based on the configuration and
     manage the CodeChecker server.
     """
-    workspace = (
-        args.config_directory
-        if "config_directory" in args and not hasattr(args, "instance_manager")
-        else None
-    )
 
     # Create workspace directory before logging is initialized.
-    if workspace and not os.path.exists(args.config_directory):
-        LOG.info("Creating non existing config directory: %s",
-                 args.config_directory)
-        os.makedirs(args.config_directory)
+    workspace = None
+    if not hasattr(args, "instance_manager"):
+        workspace = args.workspace
+
+        if not os.path.exists(workspace):
+            LOG.info("Creating non existing workspace directory: %s",
+                     workspace)
+            os.makedirs(workspace)
+
+        if not os.path.exists(args.config_directory):
+            LOG.info("Creating non existing config directory: %s",
+                     args.config_directory)
+            os.makedirs(args.config_directory)
 
     with logger.LogCfgServer(
         args.verbose if "verbose" in args else None, workspace=workspace

web/server/codechecker_server/server.py

@@ -802,6 +802,7 @@ def __init__(self,
                  server_address,
                  RequestHandlerClass,
                  config_directory,
+                 workspace_directory,
                  product_db_sql_server,
                  pckg_data,
                  context,
@@ -814,6 +815,7 @@ def __init__(self,
         LOG.debug("Initializing HTTP server...")
 
         self.config_directory = config_directory
+        self.workspace_directory = workspace_directory
         self.www_root = pckg_data['www_root']
         self.doc_root = pckg_data['doc_root']
         self.version = pckg_data['version']
@@ -1147,10 +1149,11 @@ def formatted_address(self) -> str:
         return f"[{str(self.address)}]:{self.port}"
 
 
-def start_server(config_directory: str, package_data, port: int,
-                 config_sql_server, listen_address: str,
-                 force_auth: bool, skip_db_cleanup: bool,
-                 context, check_env, machine_id: str) -> int:
+def start_server(config_directory: str, workspace_directory: str,
+                 package_data, port: int, config_sql_server,
+                 listen_address: str, force_auth: bool,
+                 skip_db_cleanup: bool, context, check_env,
+                 machine_id: str) -> int:
     """
     Starts the HTTP server to handle Web client and Thrift requests, execute
     background jobs.
@@ -1260,6 +1263,7 @@ def _cleanup_incomplete_tasks(action: str) -> int:
     http_server = server_clazz((listen_address, port),
                                RequestHandler,
                                config_directory,
+                               workspace_directory,
                                config_sql_server,
                                package_data,
                                context,

web/tests/Makefile

@@ -62,7 +62,7 @@ SHUTDOWN_GLOBAL_SERVERS_CMD = \
       echo "Shutting down server..."; \
       HOME="$${TEST_ROOT}" ${CODECHECKER_CMD} server -l; \
       HOME="$${TEST_ROOT}" ${CODECHECKER_CMD} server \
-        --config-directory $${TEST_ROOT} \
+        --workspace $${TEST_ROOT} \
         --port `cat "$${TEST_ROOT}/serverport"` --stop; \
       rm -f "$${TEST_ROOT}/serverport"; \
       HOME="$${TEST_ROOT}" ${CODECHECKER_CMD} server -l; \

web/tests/libtest/codechecker.py

@@ -554,10 +554,10 @@ def store(codechecker_cfg, test_project_name):
         return oserr.errno
 
 
-def serv_cmd(config_dir, port, pg_config=None, serv_args=None):
+def serv_cmd(workspace_dir, port, pg_config=None, serv_args=None):
 
     server_cmd = ['CodeChecker', 'server',
-                  '--config-directory', config_dir]
+                  '--workspace', workspace_dir]
 
     server_cmd.extend(['--host', 'localhost',
                        '--port', str(port)])
@@ -570,7 +570,8 @@ def serv_cmd(config_dir, port, pg_config=None, serv_args=None):
         server_cmd.append('--postgresql')
         server_cmd += _pg_db_config_to_cmdline_params(pg_config)
     else:
-        server_cmd += ['--sqlite', os.path.join(config_dir, 'config.sqlite')]
+        server_cmd += ['--sqlite', os.path.join(workspace_dir,
+                                                'config.sqlite')]
 
     print(' '.join(server_cmd))
 
@@ -586,11 +587,11 @@ def start_or_get_server(auth_required=False):
     server_type = 'global_auth_server' if auth_required else \
         'global_simple_server'
 
-    config_dir = os.path.join(workspace_root, server_type)
-    if not os.path.exists(config_dir):
-        os.makedirs(config_dir)
+    workspace_dir = os.path.join(workspace_root, server_type)
+    if not os.path.exists(workspace_dir):
+        os.makedirs(workspace_dir)
 
-    portfile = os.path.join(config_dir, 'serverport')
+    portfile = os.path.join(workspace_dir, 'serverport')
 
     if os.path.exists(portfile):
         print("A server appears to be already running...")
@@ -599,21 +600,21 @@ def start_or_get_server(auth_required=False):
     else:
         if auth_required:
             # Set up the root user and the authentication for the server.
-            env.enable_auth(config_dir)
+            env.enable_auth(workspace_dir)
 
         port = env.get_free_port()
-        print("Setting up CodeChecker server in " + config_dir + " :" +
+        print("Setting up CodeChecker server in " + workspace_dir + " :" +
               str(port))
 
         with open(portfile, 'w', encoding="utf-8", errors="ignore") as f:
             f.write(str(port))
 
         pg_config = env.get_postgresql_cfg()
 
-        server_cmd = serv_cmd(config_dir, port, pg_config)
+        server_cmd = serv_cmd(workspace_dir, port, pg_config)
 
         print("Starting server...")
-        server_stdout = os.path.join(config_dir,
+        server_stdout = os.path.join(workspace_dir,
                                      str(os.getpid()) + ".out")
 
         with open(server_stdout, "w",
@@ -622,7 +623,7 @@ def start_or_get_server(auth_required=False):
                 server_cmd,
                 stdout=server_out,
                 stderr=server_out,
-                env=env.test_env(config_dir),
+                env=env.test_env(workspace_dir),
                 encoding="utf-8",
                 errors="ignore")