Merge pull request easybuilders#2638 from boegel/eb_as_library

flesh out setting up of configuration into dedicated 'set_up_configuration' function

Author: bartoldeman

easybuild/main.py

@@ -39,20 +39,17 @@
 import os
 import stat
 import sys
-import tempfile
 import traceback
 
 # IMPORTANT this has to be the first easybuild import as it customises the logging
 #  expect missing log output when this not the case!
-from easybuild.tools.build_log import EasyBuildError, init_logging, print_error, print_msg, print_warning, stop_logging
+from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, stop_logging
 
-import easybuild.tools.config as config
-import easybuild.tools.options as eboptions
-from easybuild.framework.easyblock import EasyBlock, build_and_install_one, inject_checksums
+from easybuild.framework.easyblock import build_and_install_one, inject_checksums
 from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
 from easybuild.framework.easyconfig.easyconfig import verify_easyconfig_filename
 from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check
-from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, categorize_files_by_type, dep_graph
+from easybuild.framework.easyconfig.tools import categorize_files_by_type, dep_graph
 from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for
 from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available
 from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak
@@ -64,27 +61,16 @@
 from easybuild.tools.github import new_pr, merge_pr, update_pr
 from easybuild.tools.hooks import START, END, load_hooks, run_hook
 from easybuild.tools.modules import modules_tool
-from easybuild.tools.options import parse_external_modules_metadata, process_software_build_specs, use_color
-from easybuild.tools.robot import check_conflicts, det_robot_path, dry_run, resolve_dependencies, search_easyconfigs
+from easybuild.tools.options import set_up_configuration, use_color
+from easybuild.tools.robot import check_conflicts, dry_run, resolve_dependencies, search_easyconfigs
 from easybuild.tools.package.utilities import check_pkg_support
 from easybuild.tools.parallelbuild import submit_jobs
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state
-from easybuild.tools.version import this_is_easybuild
 
 _log = None
 
 
-def log_start(eb_command_line, eb_tmpdir):
-    """Log startup info."""
-    _log.info(this_is_easybuild())
-
-    # log used command line
-    _log.info("Command line: %s" % (' '.join(eb_command_line)))
-
-    _log.info("Using %s as temporary directory" % eb_tmpdir)
-
-
 def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=False):
     """Find easyconfigs by build specifications."""
     generated, ec_file = obtain_ec_for(build_specs, robot_path, None)
@@ -187,23 +173,6 @@ def run_contrib_style_checks(ecs, check_contrib, check_style):
     return check_contrib or check_style
 
 
-def check_root_usage(allow_use_as_root=False):
-    """
-    Check whether we are running as root, and act accordingly
-
-    :param allow_use_as_root: allow use of EasyBuild as root (but do print a warning when doing so)
-    """
-    if os.getuid() == 0:
-        if allow_use_as_root:
-            msg = "Using EasyBuild as root is NOT recommended, please proceed with care!\n"
-            msg += "(this is only allowed because EasyBuild was configured with "
-            msg += "--allow-use-as-root-and-accept-consequences)"
-            print_warning(msg)
-        else:
-            raise EasyBuildError("You seem to be running EasyBuild with root privileges which is not wise, "
-                                 "so let's end this here.")
-
-
 def clean_exit(logfile, tmpdir, testing, silent=False):
     """Small utility function to perform a clean exit."""
     cleanup(logfile, tmpdir, testing, silent=silent)
@@ -221,63 +190,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     # purposely session state very early, to avoid modules loaded by EasyBuild meddling in
     init_session_state = session_state()
 
-    # initialise options
-    eb_go = eboptions.parse_options(args=args)
-    options = eb_go.options
-    orig_paths = eb_go.args
-
-    # set umask (as early as possible)
-    if options.umask is not None:
-        new_umask = int(options.umask, 8)
-        old_umask = os.umask(new_umask)
+    eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing)
+    options, orig_paths = eb_go.options, eb_go.args
 
-    # set by option parsers via set_tmpdir
-    eb_tmpdir = tempfile.gettempdir()
-
-    search_query = options.search or options.search_filename or options.search_short
-
-    # initialise logging for main
     global _log
-    _log, logfile = init_logging(logfile, logtostdout=options.logtostdout,
-                                 silent=(testing or options.terse or search_query), colorize=options.color)
-
-    # disallow running EasyBuild as root (by default)
-    check_root_usage(allow_use_as_root=options.allow_use_as_root_and_accept_consequences)
-
-    # log startup info
-    eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
-    log_start(eb_cmd_line, eb_tmpdir)
-
-    if options.umask is not None:
-        _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
-
-    # process software build specifications (if any), i.e.
-    # software name/version, toolchain name/version, extra patches, ...
-    (try_to_generate, build_specs) = process_software_build_specs(options)
-
-    # determine robot path
-    # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
-    tweaked_ecs = try_to_generate and build_specs
-    tweaked_ecs_paths, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr)
-    auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query
-    robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_path, auto_robot=auto_robot)
-    _log.debug("Full robot path: %s" % robot_path)
-
-    # configure & initialize build options
-    config_options_dict = eb_go.get_options_by_section('config')
-    build_options = {
-        'build_specs': build_specs,
-        'command_line': eb_cmd_line,
-        'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata),
-        'pr_path': pr_path,
-        'robot_path': robot_path,
-        'silent': testing,
-        'try_to_generate': try_to_generate,
-        'valid_stops': [x[0] for x in EasyBlock.get_steps()],
-    }
-    # initialise the EasyBuild configuration & build options
-    config.init(options, config_options_dict)
-    config.init_build_options(build_options=build_options, cmdline_options=options)
+    (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings
 
     # load hook implementations (if any)
     hooks = load_hooks(options.hooks)

easybuild/tools/build_log.py

@@ -41,7 +41,7 @@
 from vsc.utils import fancylogger
 from vsc.utils.exceptions import LoggedException
 
-from easybuild.tools.version import VERSION
+from easybuild.tools.version import VERSION, this_is_easybuild
 
 
 # EasyBuild message prefix
@@ -204,6 +204,16 @@ def init_logging(logfile, logtostdout=False, silent=False, colorize=fancylogger.
     return log, logfile
 
 
+def log_start(log, eb_command_line, eb_tmpdir):
+    """Log startup info."""
+    log.info(this_is_easybuild())
+
+    # log used command line
+    log.info("Command line: %s", ' '.join(eb_command_line))
+
+    log.info("Using %s as temporary directory", eb_tmpdir)
+
+
 def stop_logging(logfile, logtostdout=False):
     """Stop logging."""
     if logtostdout:
@@ -212,13 +222,6 @@ def stop_logging(logfile, logtostdout=False):
         fancylogger.logToFile(logfile, enable=False)
 
 
-def get_log(name=None):
-    """
-    (NO LONGER SUPPORTED!) Generate logger object
-    """
-    log.nosupport("Use of get_log function", '2.0')
-
-
 def print_msg(msg, log=None, silent=False, prefix=True, newline=True, stderr=False):
     """
     Print a message.

easybuild/tools/config.py

@@ -439,13 +439,17 @@ def init_build_options(build_options=None, cmdline_options=None):
 
 def build_option(key, **kwargs):
     """Obtain value specified build option."""
+
     build_options = BuildOptions()
     if key in build_options:
         return build_options[key]
     elif 'default' in kwargs:
         return kwargs['default']
     else:
-        raise EasyBuildError("Undefined build option: %s", key)
+        error_msg = "Undefined build option: '%s'. " % key
+        error_msg += "Make sure you have set up the EasyBuild configuration using set_up_configuration() "
+        error_msg += "(from easybuild.tools.options) in case you're not using EasyBuild via the 'eb' CLI."
+        raise EasyBuildError(error_msg)
 
 
 def build_path():

easybuild/tools/options.py

@@ -53,17 +53,18 @@
 from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
 from easybuild.framework.easyconfig.easyconfig import HAVE_AUTOPEP8
 from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict
-from easybuild.framework.easyconfig.tools import get_paths_for
+from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, get_paths_for
 from easybuild.tools import build_log, run  # build_log should always stay there, to ensure EasyBuildLog
-from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError, raise_easybuilderror
+from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError
+from easybuild.tools.build_log import init_logging, log_start, print_warning, raise_easybuilderror
 from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE
 from easybuild.tools.config import DEFAULT_ALLOW_LOADED_MODULES, DEFAULT_FORCE_DOWNLOAD, DEFAULT_JOB_BACKEND
 from easybuild.tools.config import DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS, DEFAULT_MNS
 from easybuild.tools.config import DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES
 from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL, DEFAULT_PKG_TYPE
 from easybuild.tools.config import DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_REPOSITORY, EBROOT_ENV_VAR_ACTIONS
 from easybuild.tools.config import ERROR, IGNORE, FORCE_DOWNLOAD_CHOICES, LOADED_MODULES_ACTIONS, WARN
-from easybuild.tools.config import get_pretend_installpath, mk_full_default_path
+from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path
 from easybuild.tools.configobj import ConfigObj, ConfigObjError
 from easybuild.tools.docs import FORMAT_TXT, FORMAT_RST
 from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses
@@ -82,12 +83,14 @@
 from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes
 from easybuild.tools.modules import Lmod
 from easybuild.tools.ordereddict import OrderedDict
+from easybuild.tools.robot import det_robot_path
 from easybuild.tools.run import run_cmd
 from easybuild.tools.package.utilities import avail_package_naming_schemes
 from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler
 from easybuild.tools.repository.repository import avail_repositories
 from easybuild.tools.version import this_is_easybuild
 
+
 try:
     from humanfriendly.terminal import terminal_supports_colors
 except ImportError:
@@ -1151,6 +1154,94 @@ def parse_options(args=None, with_include=True):
     return eb_go
 
 
+def check_root_usage(allow_use_as_root=False):
+    """
+    Check whether we are running as root, and act accordingly
+
+    :param allow_use_as_root: allow use of EasyBuild as root (but do print a warning when doing so)
+    """
+    if os.getuid() == 0:
+        if allow_use_as_root:
+            msg = "Using EasyBuild as root is NOT recommended, please proceed with care!\n"
+            msg += "(this is only allowed because EasyBuild was configured with "
+            msg += "--allow-use-as-root-and-accept-consequences)"
+            print_warning(msg)
+        else:
+            raise EasyBuildError("You seem to be running EasyBuild with root privileges which is not wise, "
+                                 "so let's end this here.")
+
+
+def set_up_configuration(args=None, logfile=None, testing=False, silent=False):
+    """
+    Set up EasyBuild configuration, by parsing configuration settings & initialising build options.
+
+    :param args: command line arguments to take into account when parsing the EasyBuild configuration settings
+    :param logfile: log file to use
+    :param testing: enable testing mode
+    :param silent: stay silent (no printing)
+    """
+
+    # parse EasyBuild configuration settings
+    eb_go = parse_options(args=args)
+    options = eb_go.options
+
+    # tmpdir is set by option parser via set_tmpdir function
+    tmpdir = tempfile.gettempdir()
+
+    # set umask (as early as possible)
+    if options.umask is not None:
+        new_umask = int(options.umask, 8)
+        old_umask = os.umask(new_umask)
+
+    search_query = options.search or options.search_filename or options.search_short
+
+    # initialise logging for main
+    log, logfile = init_logging(logfile, logtostdout=options.logtostdout,
+                                silent=(testing or options.terse or search_query or silent),
+                                colorize=options.color)
+
+    # log startup info (must be done after setting up logger)
+    eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
+    log_start(log, eb_cmd_line, tmpdir)
+
+    # can't log umask setting before logger is set up...
+    if options.umask is not None:
+        log.info("umask set to '%s' (used to be '%s')", oct(new_umask), oct(old_umask))
+
+    # disallow running EasyBuild as root (by default)
+    check_root_usage(allow_use_as_root=options.allow_use_as_root_and_accept_consequences)
+
+    # process software build specifications (if any), i.e.
+    # software name/version, toolchain name/version, extra patches, ...
+    (try_to_generate, build_specs) = process_software_build_specs(options)
+
+    # determine robot path
+    # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
+    tweaked_ecs = try_to_generate and build_specs
+    tweaked_ecs_paths, pr_path = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr)
+    auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query
+    robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_path, auto_robot=auto_robot)
+    log.debug("Full robot path: %s" % robot_path)
+
+    # configure & initialize build options
+    config_options_dict = eb_go.get_options_by_section('config')
+    build_options = {
+        'build_specs': build_specs,
+        'command_line': eb_cmd_line,
+        'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata),
+        'pr_path': pr_path,
+        'robot_path': robot_path,
+        'silent': testing,
+        'try_to_generate': try_to_generate,
+        'valid_stops': [x[0] for x in EasyBlock.get_steps()],
+    }
+    # initialise the EasyBuild configuration & build options
+    init(options, config_options_dict)
+    init_build_options(build_options=build_options, cmdline_options=options)
+
+    return eb_go, (build_specs, log, logfile, robot_path, search_query, tmpdir, try_to_generate, tweaked_ecs_paths)
+
+
 def process_software_build_specs(options):
     """
     Create a dictionary with specified software build options.