diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml new file mode 100644 index 0000000..404cbae --- /dev/null +++ b/.github/workflows/python-package-conda.yml @@ -0,0 +1,34 @@ +name: Python Package using Conda + +on: [push] + +jobs: + build-linux: + runs-on: ubuntu-latest + strategy: + max-parallel: 5 + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: '3.10' + - name: Add conda to system path + run: | + # $CONDA is an environment variable pointing to the root of the miniconda directory + echo $CONDA/bin >> $GITHUB_PATH + - name: Install current library and dependencies + run: | + pip install -e . + # - name: Lint with flake8 + # run: | + # conda install flake8 + # # stop the build if there are Python syntax errors or undefined names + # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + conda install pytest + pytest diff --git a/README.md b/README.md index 743c8a0..8c841dd 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ $ pip install --editable . # OR pip install -e . **Version:**
-0.3.3-Beta +0.3.5-Beta Authors: William Gebhardt, Alexander G. Ororbia II
diff --git a/ngcsimlib/__init__.py b/ngcsimlib/__init__.py index b6f5b53..61528f4 100644 --- a/ngcsimlib/__init__.py +++ b/ngcsimlib/__init__.py @@ -8,6 +8,7 @@ from ngcsimlib.configManager import init_config, get_config from ngcsimlib.logger import warn from pkg_resources import get_distribution +from ngcsimlib.compilers.process import Process, transition __version__ = get_distribution('ngcsimlib').version ## set software version @@ -36,19 +37,21 @@ def preload_modules(path=None): modules = json.load(file, object_hook=lambda d: SimpleNamespace(**d)) for module in modules: - mod = import_module(module.absolute_path) - utils.modules._Loaded_Modules[module.absolute_path] = mod + load_module(module) - for attribute in module.attributes: - atr = getattr(mod, attribute.name) - utils.modules._Loaded_Attributes[attribute.name] = atr +def load_module(module): + mod = import_module(module.absolute_path) + utils.modules._Loaded_Modules[module.absolute_path] = mod - utils.modules._Loaded_Attributes[".".join([module.absolute_path, attribute.name])] = atr - if hasattr(attribute, "keywords"): - for keyword in attribute.keywords: - utils.modules._Loaded_Attributes[keyword] = atr + for attribute in module.attributes: + atr = getattr(mod, attribute.name) + utils.modules._Loaded_Attributes[attribute.name] = atr - utils.set_loaded(True) + utils.modules._Loaded_Attributes[ + ".".join([module.absolute_path, attribute.name])] = atr + if hasattr(attribute, "keywords"): + for keyword in attribute.keywords: + utils.modules._Loaded_Attributes[keyword] = atr ###### Initialize Config def configure(): diff --git a/ngcsimlib/compartment.py b/ngcsimlib/compartment.py index 9e331dc..9694142 100644 --- a/ngcsimlib/compartment.py +++ b/ngcsimlib/compartment.py @@ -7,13 +7,12 @@ class Compartment: """ Compartments in ngcsimlib are container objects for storing the stateful - values of components. Compartments are - tracked globaly and are automatically linked to components and methods - during compiling to allow for stateful - mechanics to be run without the need for the class object. Compartments - also provide an entry and exit point for - values inside of components allowing for cables to be connected for - sending and receiving values. + values of components. Compartments are tracked globally and are + automatically linked to components and methods during compiling to allow + for stateful mechanics to be run without the need for the class object. + Compartments also provide an entry and exit point for values inside of + components allowing for cables to be connected for sending and receiving + values. """ @classmethod @@ -31,19 +30,18 @@ def is_compartment(cls, obj): """ return hasattr(obj, "_is_compartment") - def __init__(self, initial_value=None, static=False, is_input=False): + def __init__(self, initial_value=None, static=False, is_input=False, + display_name=None, units=None): """ Builds a compartment to be used inside a component. It is important - to note that building compartments - outside of components may cause unexpected behavior as components - interact with their compartments during - construction to finish initializing them. + to note that building compartments outside of components may cause + unexpected behavior as components interact with their compartments + during construction to finish initializing them. Args: initial_value: The initial value of the compartment. As a general - practice it is a good idea to - provide a value that is similar to the values that will - normally be stored here, such as an array of - zeros of the correct length. (default: None) + practice it is a good idea to provide a value that is similar to + the values that will normally be stored here, such as an array of + zeros of the correct length. (default: None) static: a flag to lock a compartment to be static (default: False) """ @@ -56,6 +54,8 @@ def __init__(self, initial_value=None, static=False, is_input=False): self.path = None self.is_input = is_input self._is_destination = False + self._display_name = display_name + self._units = units def _setup(self, current_component, key): """ @@ -103,10 +103,9 @@ def __str__(self): def __lshift__(self, other) -> None: """ Overrides the left shift operation to be used for wiring compartments - into one another - if other is not an Operation it will create an overwrite operation - with other as the argument, - otherwise it will use the provided operation + into one another if other is not an Operation it will create an + overwrite operation with other as the argument, otherwise it will use + the provided operation Args: other: Either another component or an instance of BaseOp @@ -131,3 +130,12 @@ def is_wired(self): return True return self._is_destination + + @property + def display_name(self): + return self._display_name if self._display_name is not None else ( + self.name) + + @property + def units(self): + return self._units if self._units is not None else "dimensionless" diff --git a/ngcsimlib/compilers/__init__.py b/ngcsimlib/compilers/__init__.py index 8c0287e..eb119d8 100644 --- a/ngcsimlib/compilers/__init__.py +++ b/ngcsimlib/compilers/__init__.py @@ -1,3 +1,2 @@ -from .command_compiler import compile_command, dynamic_compile, wrap_command -from .component_compiler import compile as compile_component -from .op_compiler import compile as compile_op +from .legacy_compiler.command_compiler import compile_command, dynamic_compile +from .utils import wrap_command, compose diff --git a/ngcsimlib/compilers/command_compiler.py b/ngcsimlib/compilers/command_compiler.py deleted file mode 100644 index 8c79051..0000000 --- a/ngcsimlib/compilers/command_compiler.py +++ /dev/null @@ -1,157 +0,0 @@ -""" -This is the file that contains the code to compile a given command on a model. - -There are a few ways to compile the commands for a model, firstly if there is a command object already initialized -that has a valid compile key and a list of components the base method of `compile_command(command)` can be used to produce -the desired output. If no command object has been initialized then the `dynamic_compile(*components, compile_key=None)` -can be used to produce the desired output without the need to first go through a command object. The output of either -compile method will be the same. - -The output produced by compiling a command will be two objects. - -The first object produced by compiling a command is the compiled method itself. This method requires at least one -positional argument and then any number of additional arguments. The first argument that is provided to the compiled -method is a python dictionary that contains the state for all compartments this method will need to access, -as discerning this can be a challenge it is normal to just pass it all compartments present on your model. The -remaining list of arguments are all the run time arguments that the various compiled methods need to run properly. -The return value of this compiled method is the final state of all compartments after running through the compiled -command. Note here that the value on the compartments are not automatically updated and that will need to be done after. - -The second object produced by compiling a command is the list of arguments that the compile command is expecting to -be passed in alongside the initial state of all the compartments. It is a good habit to get into printing this list -out after compiling as it can help catch typos present in the compiled methods that will not cause the compiling to -fail but will produce unknown behavior. - -There is a wrapper method offered in this file we recommend using to assist with the design patterned required by the -compiled command. This is done with `wrap_command(command)`. This method will return another method that removes the -need for creating the initial state of the compartments and setting all the compartment values after running. Arguments -are still required to be passed in at run time. - -""" -from ngcsimlib.compilers.component_compiler import parse as parse_component, compile as compile_component -from ngcsimlib.compilers.op_compiler import parse as parse_connection -from ngcsimlib.utils import Get_Compartment_Batch, Set_Compartment_Batch -from ngcsimlib.logger import critical - -def _compile(compile_key, components): - """ - This is the top level compile method for commands. Note this does not actually require you to compile a - specific command object as it works purely off the compile key provided to the method. - The general process that this takes to compile down everything, is by producing an execution order that knows which - methods that are going to be called and where the results of the method are supposed to be stored. - - The execution order is as follows: - - For each component in the provided array of components; - | compile it with the provided key - | resolve the outputs of the compiled function - - Args: - compile_key: The key that is being compiled (mapped to each function that has the @resolver decorator - above it) - - components: The list of components to compile for this function - - Returns: - Produces the two objects described at the top of this file - """ - assert compile_key is not None - ## for each component, get compartments, get output compartments - resolvers = {} - for c_name, component in components.items(): - resolvers[c_name] = parse_component(component, compile_key) - - needed_args = [] - needed_comps = [] - - for c_name, component in components.items(): - _, outs, args, params, comps = resolvers[c_name] - for a in args: - if a not in needed_args: - needed_args.append(a) - - for connection in component.connections: - inputs, _ = parse_connection(connection) - ncs = [str(i) for i in inputs] - for nc in ncs: - if nc not in needed_comps: - needed_comps.append(nc) - - for comp in comps: - path = str(component.__dict__[comp].path) - if path not in needed_comps: - needed_comps.append(path) - - exc_order = [] - for c_name, component in components.items(): - exc_order.extend(compile_component(component, resolvers[c_name])) - - def compiled(compartment_values, **kwargs): - for n in needed_args: - if n not in kwargs: - critical(f"Missing keyword argument \"{n}\" in compiled function." - f"\tExpected keyword arguments {needed_args}") - - for exc, outs, name in exc_order: - _comps = {key: compartment_values[key] for key in needed_comps} - vals = exc(**kwargs, **_comps) - if len(outs) == 1: - compartment_values[outs[0]] = vals - elif len(outs) > 1: - for v, t in zip(vals, outs): - compartment_values[t] = v - return compartment_values - - return compiled, needed_args - - -def compile_command(command): - """ - Compiles a given command object to the spec described at the top of this file - - Args: - command: the command object - - Returns: - compiled_command, needed_arguments - - """ - return _compile(command.compile_key, command.components) - - -def dynamic_compile(*components, compile_key=None): - """ - Dynamically compiles a command without the need of a command object to produce - the spec described at the top of this file. - - Args: - *components: a list of components to be compiled - - compile_key: the compile key specifying what to compile - - Returns: - compiled_command, needed_arguments - """ - if compile_key is None: - critical("Can not compile a command without a compile key") - return _compile(compile_key, {c.name: c for c in components}) - - -def wrap_command(command): - """ - Wraps the provided command to provide the state of all compartments as input - and saves the returned state to all compartments after running. Designed to - be used with compiled commands - - Args: - command: the command to wrap - - Returns: - the output of the command after it's been executed - """ - def _wrapped(**kwargs): - vals = command(Get_Compartment_Batch(), **kwargs) - Set_Compartment_Batch(vals) - return vals - - return _wrapped diff --git a/ngcsimlib/compilers/legacy_compiler/__init__.py b/ngcsimlib/compilers/legacy_compiler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ngcsimlib/compilers/legacy_compiler/command_compiler.py b/ngcsimlib/compilers/legacy_compiler/command_compiler.py new file mode 100644 index 0000000..9107a1b --- /dev/null +++ b/ngcsimlib/compilers/legacy_compiler/command_compiler.py @@ -0,0 +1,159 @@ +""" +This is the file that contains the code to compile a given command on a model. + +There are a few ways to compile the commands for a model, firstly if there is +a command object already initialized that has a valid compile key and a list +of components the base method of `compile_command(command)` can be used to +produce the desired output. If no command object has been initialized then +the `dynamic_compile(*components, compile_key=None)` can be used to produce +the desired output without the need to first go through a command object. The +output of either compile method will be the same. + +The output produced by compiling a command will be two objects. + +The first object produced by compiling a command is the compiled method +itself. This method requires at least one positional argument and then any +number of additional arguments. The first argument that is provided to the +compiled method is a python dictionary that contains the state for all +compartments this method will need to access, as discerning this can be a +challenge it is normal to just pass it all compartments present on your +model. The remaining list of arguments are all the run time arguments that +the various compiled methods need to run properly. The return value of this +compiled method is the final state of all compartments after running through +the compiled command. Note here that the value on the compartments are not +automatically updated and that will need to be done after. + +The second object produced by compiling a command is the list of arguments +that the compile command is expecting to be passed in alongside the initial +state of all the compartments. It is a good habit to get into printing this +list out after compiling as it can help catch typos present in the compiled +methods that will not cause the compiling to fail but will produce unknown +behavior. + +There is a wrapper method offered in this file we recommend using to assist +with the design patterned required by the compiled command. This is done with +`wrap_command(command)`. This method will return another method that removes +the need for creating the initial state of the compartments and setting all +the compartment values after running. Arguments are still required to be +passed in at run time. + +""" +from ngcsimlib.compilers.legacy_compiler.component_compiler import parse as parse_component, \ + compile as compile_component +from ngcsimlib.compilers.legacy_compiler.op_compiler import parse as parse_connection +from ngcsimlib.utils import Get_Compartment_Batch, Set_Compartment_Batch +from ngcsimlib.logger import critical + + +def _compile(compile_key, components): + """ + This is the top level compile method for commands. Note this does not + actually require you to compile a specific command object as it works + purely off the compile key provided to the method. The general process + that this takes to compile down everything, is by producing an execution + order that knows which methods that are going to be called and where the + results of the method are supposed to be stored. + + The execution order is as follows: + + For each component in the provided array of components; + | compile it with the provided key + | resolve the outputs of the compiled function + + Args: + compile_key: The key for the transition that is being compiled + + components: The list of components to compile for this function + + Returns: + Produces the two objects described at the top of this file + """ + assert compile_key is not None + ## for each component, get compartments, get output compartments + transitions = {} + for c_name, component in components.items(): + transitions[c_name] = parse_component(component, compile_key) + + needed_args = [] + needed_comps = [] + + for c_name, component in components.items(): + _, outs, args, params, comps = transitions[c_name] + for a in args: + if a not in needed_args: + needed_args.append(a) + + for connection in component.connections: + inputs, _ = parse_connection(connection) + ncs = [str(i) for i in inputs] + for nc in ncs: + if nc not in needed_comps: + needed_comps.append(nc) + + for comp in comps: + path = str(component.__dict__[comp].path) + if path not in needed_comps: + needed_comps.append(path) + + exc_order = [] + for c_name, component in components.items(): + exc_order.extend(compile_component(component, transitions[c_name])) + + def compiled(compartment_values=None, **kwargs): + if compartment_values is None: + critical( + f"Attempting to call a compiled method without the current " + f"state of the model. " + f"Verify the method is wrapped or a current state is provided") + for n in needed_args: + if n not in kwargs: + critical( + f"Missing keyword argument \"{n}\" in compiled function." + f"\tExpected keyword arguments {needed_args}") + + for exc, outs, name, comp_ids in exc_order: + _comps = {key: compartment_values[key] for key in comp_ids} + vals = exc(**kwargs, **_comps) + if len(outs) == 1: + compartment_values[outs[0]] = vals + elif len(outs) > 1: + for v, t in zip(vals, outs): + compartment_values[t] = v + return compartment_values + + return compiled, needed_args + + +def compile_command(command): + """ + Compiles a given command object to the spec described at the top of this + file + + Args: + command: the command object + + Returns: + compiled_command, needed_arguments + + """ + return _compile(command.compile_key, command.components) + + +def dynamic_compile(*components, compile_key=None): + """ + Dynamically compiles a command without the need of a command object to + produce the spec described at the top of this file. + + Args: + *components: a list of components to be compiled + + compile_key: the compile key specifying what to compile + + Returns: + compiled_command, needed_arguments + """ + if compile_key is None: + critical("Can not compile a command without a compile key") + return _compile(compile_key, {c.name: c for c in components}) + + diff --git a/ngcsimlib/compilers/component_compiler.py b/ngcsimlib/compilers/legacy_compiler/component_compiler.py similarity index 68% rename from ngcsimlib/compilers/component_compiler.py rename to ngcsimlib/compilers/legacy_compiler/component_compiler.py index 28ea42d..4fe12c5 100644 --- a/ngcsimlib/compilers/component_compiler.py +++ b/ngcsimlib/compilers/legacy_compiler/component_compiler.py @@ -15,8 +15,8 @@ the same pattern used by the command compiler. """ -from ngcsimlib.compilers.op_compiler import compile as op_compile -from ngcsimlib.utils import get_resolver +from ngcsimlib.compilers.legacy_compiler.op_compiler import compile as op_compile +from ngcsimlib.utils import get_transition from ngcsimlib.compartment import Compartment from ngcsimlib.logger import critical @@ -37,16 +37,28 @@ def parse(component, compile_key): the compartments needed """ - (pure_fn, output_compartments), ( - args, parameters, compartments, parse_varnames) = \ - get_resolver(component.__class__, compile_key) + if component.__class__.__dict__.get("auto_resolve", True): + (pure_fn, output_compartments), ( + args, parameters, compartments, parse_varnames) = \ + get_transition(component.__class__, compile_key) + else: + build_method = component.__class__.__dict__.get(f"build_{compile_key}", None) + if build_method is None: + critical(f"Component {component.name} if flagged to not use a stored transition but " + f"does not have a build_{compile_key} method") + return build_method(component) if parse_varnames: args = [] parameters = [] compartments = [] - varnames = pure_fn.__func__.__code__.co_varnames[ - :pure_fn.__func__.__code__.co_argcount] + try: + func = pure_fn.__func__ + except: + func = pure_fn + + varnames = func.__code__.co_varnames[ + :func.__code__.co_argcount] for name in varnames: if name not in component.__dict__.keys(): @@ -70,20 +82,20 @@ def parse(component, compile_key): return (pure_fn, output_compartments, args, parameters, compartments) -def compile(component, resolver): +def compile(component, transition): """ compiles down the component to a single pure method Args: component: the component to compile - resolver: the parsed output of the component + transition: the parsed output of the component Returns: the compiled method """ exc_order = [] - pure_fn, outs, _args, params, comps = resolver + pure_fn, outs, _args, params, comps = transition ### Op resolve for connection in component.connections: @@ -95,11 +107,18 @@ def compile(component, resolver): funParams = {narg: component.__dict__[narg] for narg in params} + comp_key_key = [(narg.split('/')[-1], narg) for narg in comp_ids] + + try: + func = pure_fn.__func__ + except: + func = pure_fn + def compiled(**kwargs): funArgs = {narg: kwargs.get(narg) for narg in _args} - funComps = {narg.split('/')[-1]: kwargs.get(narg) for narg in comp_ids} + funComps = {knarg: kwargs.get(narg) for knarg, narg in comp_key_key} - return pure_fn.__func__(**funParams, **funArgs, **funComps) + return func(**funParams, **funArgs, **funComps) - exc_order.append((compiled, out_ids, component.name)) + exc_order.append((compiled, out_ids, component.name, comp_ids)) return exc_order diff --git a/ngcsimlib/compilers/op_compiler.py b/ngcsimlib/compilers/legacy_compiler/op_compiler.py similarity index 89% rename from ngcsimlib/compilers/op_compiler.py rename to ngcsimlib/compilers/legacy_compiler/op_compiler.py index 36efce9..17bbcde 100644 --- a/ngcsimlib/compilers/op_compiler.py +++ b/ngcsimlib/compilers/legacy_compiler/op_compiler.py @@ -10,7 +10,7 @@ The second one is the compile method which returns the execution order for the compile operation. It is important to know that all operation should have an `is_compilable` flag set to true if they are compilable. Some operations -such as the `add` operation are not compilable as their resolve method +such as the `add` operation are not compilable as their transition method contains execution logic that will not be captured by the compiled command. """ from ngcsimlib.operations.baseOp import BaseOp @@ -85,8 +85,13 @@ def compile(op): else: iids.append(str(s.path)) + additional_idds = [] + for _, _, _, _iids in exc_order: + additional_idds.extend(_iids) + + # print(additional_idds) def _op_compiled(**kwargs): - computed_values = [cmd(**kwargs) for cmd, _, _ in exc_order] + computed_values = [cmd(**kwargs) for cmd, _, _, _ in exc_order] compartment_args = [kwargs.get(narg) for narg in iids] _val_loc = 0 @@ -103,4 +108,4 @@ def _op_compiled(**kwargs): return op.operation(*_args) - return (_op_compiled, [str(output)], "op") + return (_op_compiled, [str(output)], op.__class__.__name__, iids + additional_idds) diff --git a/ngcsimlib/compilers/process.py b/ngcsimlib/compilers/process.py new file mode 100644 index 0000000..2462b47 --- /dev/null +++ b/ngcsimlib/compilers/process.py @@ -0,0 +1,241 @@ +from ngcsimlib.compilers.utils import compose +from ngcsimlib.compilers.process_compiler.component_compiler import compile as compile_component +from ngcsimlib.logger import warn +from functools import wraps +from ngcsimlib.utils import add_component_transition, add_transition_meta +from ngcsimlib.utils import get_current_context, infer_context, Set_Compartment_Batch + +class Process(object): + """ + The process is an alternate method for compiling transitions of models into + pure methods. In general this is the preferred method for doing this over + the legacy compiler, however it is not required. The Process composes the + methods used as they are added to the process meaning that partial compiling + is possible for debugging by stopping adding to the chain of transitions in + the process. + + The general use case to create a process is as follows + myProcess = (Process("myProcess") + >> myFirstComponent.firstTransition + >> myFirstComponent.secondTransition + >> mySecondComponent.firstTransition + >> mySecondComponent.secondTransition) + + However, the adding of new methods does not need to happen only at the + initialization of the Process class. The above example can be added to as + follows: + myProcess >> myThirdComponent.firstTransition + myProcess >> myThirdComponent.secondTransition + + In general once all the transition methods are added to the process there + are two ways of actually running the transitions defined in the process. + The first is through the use of myProcess.pure(current_state, **kwargs) this + executes the process as a pure method doing nothing to update the actual + state of the model. + + The other method for running a process is through + myProcess.execute(**kwargs). This runs the process with the current state of + the model. By default, this also does not update the model with the final + state, however this can be changed with the flag "update_state". + """ + def __init__(self, name): + """ + Creates and empty process using the provided name + Args: + name: the name of the process (should be unique per context) + """ + self._method = None + self._calls = [] + self.name = name + self._needed_args = set([]) + self._needed_contexts = set([]) + + cc = get_current_context() + if cc is not None: + cc.register_process(self) + + @staticmethod + def make_process(process_spec, custom_process_klass=None): + """ + Used the in the creation of a process from a json file. Under normal + circumstances this is not normally called by the user. + + Args: + process_spec: the parsed json object to create a process from + custom_process_klass: a custom subclass of a process to build + + Returns: the created process + + """ + if custom_process_klass is None: + custom_process_klass = Process + newProcess = custom_process_klass(process_spec['name']) + + for x in process_spec['calls']: + path = x['path'] + ctx = infer_context(path) + component_name = path.split("/")[-1] + newProcess >> getattr(ctx.get_components(component_name), x['key']) + return newProcess + + @property + def pure(self): + """ + Returns: The current compile method for the process as a pure method + """ + return self._method + + def __rshift__(self, other): + """ + Added wrapper for the transition method + Args: + other: the transition call for the transition method + + Returns: the process for the use of chaining calls + """ + return self.transition(other) + + def transition(self, transition_call): + """ + Adds the given transition call to the Process. The argument call must be + decorated by the @transition decorator. + + Args: + transition_call: Transition method to add to the process + + Returns: the process for the use of chaining calls + + """ + self._calls.append({"path": transition_call.__self__.path, "key": transition_call.resolver_key}) + self._needed_contexts.add(infer_context(transition_call.__self__.path)) + new_step, new_args = compile_component(transition_call) + + for arg in new_args: + self._needed_args.add(arg) + self._method = compose(self._method, new_step) + return self + + def execute(self, update_state=False, **kwargs): + """ + Executes the process using the current state of the model to run. This + method has checks to ensure that the process has transitions added to it + as well as that all the keyword arguments required by each of the + transition call are in the provided keyword arguments. By default, this + does not update the final state of the model but that can be toggled + with the flag "update_state". + + Args: + update_state: Should this method update the final state of the model + **kwargs: The required keyword arguments to execute the process + + Returns: the final state of the process regardless of the model is + updated to reflect this. Will return null if either of the above checks + fail + + """ + if self._method is None: + warn("Attempting to execute a process with no transition steps") + return + for arg in self._needed_args: + if arg not in kwargs.keys(): + warn("Missing kwarg", arg, "in kwargs for Process", self.name) + return + state = self.pure(self.get_required_state(include_special_compartments=True), **kwargs) + if update_state: + self.updated_modified_state(state) + return state + + def as_obj(self): + """ + Returns: Returns this process as an object to be used with json files + """ + return {"name": self.name, "class": self.__class__.__name__, "calls": self._calls} + + def get_required_args(self): + """ + Returns: The needed arguments for all the transition calls in this + process as a set + """ + return self._needed_args + + def get_required_state(self, include_special_compartments=False): + """ + Gets the required compartments needed to run this process, important to + note that if this is going to be used as an argument to the pure method + make sure that the "include_special_compartments" flag is set to True so + that special compartments found in certain components are visible. + Args: + include_special_compartments: A flag to show the compartments that + denoted as special compartments by ngcsimlib (this is any + compartment with * in their name, these are can only be created + dynamically) + + Returns: A subset of the model state based on the required compartments + + """ + compound_state = {} + for context in self._needed_contexts: + compound_state.update(context.get_current_state(include_special_compartments)) + return compound_state + + def updated_modified_state(self, state): + """ + Updates the model with the provided state. It is important to note that + only values that are rquired for the execution of this process will be + affected by this call. If all compartments need to be updated, view + other options found in ngcsimlib.utils. + Args: + state: The state to update the model with + """ + Set_Compartment_Batch({key: value for key, value in state.items() if key in self.get_required_state(include_special_compartments=True)}) + + +def transition(output_compartments, builder=False): + """ + The decorator to be paired with the Process call. This method does + everything that the now outdated resolver did to ensure backward + compatability. This decorator expects to decorate a static method on a + class. + + Through normal patterns these decorated method will never be directly called + by the end user, but if they are for the purpose of debugging there are a + few things to keep in mind. While the process compiler will automatically + link values in the component to the different values to be passed into the + method that does not exist if they are directly called. In addition, if the + method is going to be called at a class level the first value passed into + the method must be None to not mess up the internal decoration. + Args: + output_compartments: The string name of the output compartments the + outputs of this method will be assigned to in the order they are output. + builder: A boolean flag for if this method is a builder method for the + compiler. A builder method is a method that returns the static method to + use in the transition. + + Returns: the wrapped method + + """ + def _wrapper(f): + @wraps(f) + def inner(self, *args, **kwargs): + return f(*args, **kwargs) + + + class_name = ".".join(f.__qualname__.split('.')[:-1]) + resolver_key = f.__qualname__.split('.')[-1] + + + inner.fargs = f.__func__.__code__.co_varnames[:f.__func__.__code__.co_argcount] + inner.f = f + inner.output_compartments = output_compartments + + inner.class_name = class_name + inner.resolver_key = resolver_key + inner.builder = builder + + add_component_transition(class_name, resolver_key, + (f, output_compartments)) + + add_transition_meta(class_name, resolver_key,([], [], [], True)) + + return inner + return _wrapper \ No newline at end of file diff --git a/ngcsimlib/compilers/process_compiler/component_compiler.py b/ngcsimlib/compilers/process_compiler/component_compiler.py new file mode 100644 index 0000000..7245e32 --- /dev/null +++ b/ngcsimlib/compilers/process_compiler/component_compiler.py @@ -0,0 +1,95 @@ +""" +This file contains the methods used to compile methods for the use of Processes. +The general methodology behind this compiler is that if all transitions can be +expressed as f(current_state, **kwargs) -> final_state they can then be composed +together as f(g(current_state, **kwargs) **kwargs) -> final_state. While it is +technically possible to use the compiler outside the process its intended use +case is through the process and thus if error occur though other uses support +may be minimal. +""" + +from ngcsimlib.compilers.process_compiler.op_compiler import compile as op_compile +from ngcsimlib.compartment import Compartment +from ngcsimlib.compilers.utils import compose + +def __make_get_arg(a): + return lambda current_state, **kwargs: kwargs.get(a, None) + +def __make_get_param(p, component): + return lambda current_state, **kwargs: component.__dict__.get(p, None) + +def __make_get_comp(c, component): + return lambda current_state, **kwargs: current_state.get(component.__dict__[c].path, None) + +def _builder(transition_method_to_build): + component = transition_method_to_build.__self__ + builder_method = transition_method_to_build.f + # method, output_compartments, args, params, input_compartments + return builder_method(component) + + +def compile(transition_method): + """ + This method is the main compile method for the process compiler. Unlike the + legacy compiler this compiler is designed to be self-contained and output + the methods that are composed together to make the process compiler function + Args: + transition_method: a method usually component.method that has been + decorated by the @transition decorator. + + Returns: the pure compiled method of the form + f(current_state, **kwargs) -> final_state) + + """ + composition = None + component = transition_method.__self__ + + if transition_method.builder: + pure_fn, output, args, parameters, compartments = _builder(transition_method) + else: + + pure_fn = transition_method.f + output = transition_method.output_compartments + + varnames = transition_method.fargs + + args = [] + compartments = [] + parameters = [] + + for name in varnames: + if name not in component.__dict__.keys(): + args.append(name) + elif Compartment.is_compartment(component.__dict__[name]): + compartments.append(name) + else: + parameters.append(name) + + for conn in component.connections: + composition = compose(composition, op_compile(conn)) + + arg_methods = [] + needed_args = [] + for a in args: + needed_args.append(a) + arg_methods.append((a, __make_get_arg(a))) + + for p in parameters: + arg_methods.append((p, __make_get_param(p, component))) + + for c in compartments: + arg_methods.append((c, __make_get_comp(c, component))) + + def compiled(current_state, **kwargs): + kargvals = {key: m(current_state, **kwargs) for key, m in arg_methods} + vals = pure_fn(**kargvals) + if len(output) > 1: + for v, o in zip(vals, output): + current_state[component.__dict__[o].path] = v + else: + current_state[component.__dict__[output[0]].path] = vals + return current_state + + composition = compose(composition, compiled) + + return composition, needed_args diff --git a/ngcsimlib/compilers/process_compiler/op_compiler.py b/ngcsimlib/compilers/process_compiler/op_compiler.py new file mode 100644 index 0000000..34eb7da --- /dev/null +++ b/ngcsimlib/compilers/process_compiler/op_compiler.py @@ -0,0 +1,32 @@ +from ngcsimlib.operations.baseOp import BaseOp + +def _make_lambda(s): + return lambda current_state, **kwargs: current_state[s.path] + +def compile(op): + """ + compiles root operation down to a single method of + f(current_state, **kwargs) -> final_state + + Args: + op: the operation to compile + + Returns: the compiled operation + """ + arg_methods = [] + for s in op.sources: + if isinstance(s, BaseOp): + arg_methods.append(compile(s)) + else: + arg_methods.append(_make_lambda(s)) + + def compiled(current_state, **kwargs): + argvals = [m(current_state, **kwargs) for m in arg_methods] + val = op.operation(*argvals) + if op.destination is not None: + current_state[op.destination.path] = val + return current_state + else: + return val + + return compiled \ No newline at end of file diff --git a/ngcsimlib/compilers/utils.py b/ngcsimlib/compilers/utils.py new file mode 100644 index 0000000..654f84c --- /dev/null +++ b/ngcsimlib/compilers/utils.py @@ -0,0 +1,30 @@ +from ngcsimlib.utils.compartment import Get_Compartment_Batch, Set_Compartment_Batch + + +def wrap_command(command): + """ + Wraps the provided command to provide the state of all compartments as input + and saves the returned state to all compartments after running. Designed to + be used with compiled commands + + Args: + command: the command to wrap + + Returns: + the output of the command after it's been executed + """ + + def _wrapped(**kwargs): + vals = command(Get_Compartment_Batch(), **kwargs) + Set_Compartment_Batch(vals) + return vals + + return _wrapped + + +def compose(current_composition, next_method): + if current_composition is None: + return next_method + + return lambda current_state, **kwargs: next_method( + current_composition(current_state, **kwargs), **kwargs) diff --git a/ngcsimlib/component.py b/ngcsimlib/component.py index 0220d9f..16a0358 100644 --- a/ngcsimlib/component.py +++ b/ngcsimlib/component.py @@ -3,21 +3,20 @@ from ngcsimlib.compartment import Compartment from ngcsimlib.logger import warn + class Component(metaclass=MetaComponent): """ Components are a foundational part of ngclearn and its component/command structure. In ngclearn, all stateful parts of a model take the form of - components. The internal storage of the state within a component takes one - of two forms, either as a compartment or as a member variable. The member - variables are values such as hyperparameters and weights/synaptic - efficacies, - where the transfer of their individual state from component to component is - not needed. - Compartments, on the other hand, are where the state information, both from - and for other components, are stored. As the components are the stateful - pieces of the model, they also contain the methods and logic behind - advancing - their internal state (values) forward in time. + components. The internal storage of the state within a component takes + one of two forms, either as a compartment or as a member variable. The + member variables are values such as hyperparameters and weights/synaptic + efficacies, where the transfer of their individual state from component + to component is not needed. Compartments, on the other hand, are where + the state information, both from and for other components, are stored. As + the components are the stateful pieces of the model, they also contain + the methods and logic behind advancing their internal state (values) + forward in time. """ def __init__(self, name, **kwargs): @@ -31,8 +30,7 @@ def __init__(self, name, **kwargs): name: the name of the component kwargs: additional keyword arguments. These are not used in the - base class, - but this is here for future use if needed. + base class, but this is here for future use if needed. """ # Component Data self.name = name @@ -77,39 +75,4 @@ def validate(self): msg += f"\nCompartment Description:\t{_help}" warn(msg) valid = False - return valid - - ##Abstract Methods - @abstractmethod - def advance_state(self, **kwargs): - """ - An abstract method to advance the state of the component to the next one - (a component transitions from its current state at time t to a new one - at time t + dt) - """ - pass - - @abstractmethod - def reset(self, **kwargs): - """ - An abstract method that should be implemented to models can be returned - to their original state. - """ - pass - - @abstractmethod - def save(self, directory, **kwargs): - """ - An abstract method to save component specific state to the provided - directory - - Args: - directory: the directory to save the state to - """ - pass - - @classmethod - @abstractmethod - def help(cls): - pass - + return valid \ No newline at end of file diff --git a/ngcsimlib/configManager.py b/ngcsimlib/configManager.py index 26ed85a..7ab0b75 100644 --- a/ngcsimlib/configManager.py +++ b/ngcsimlib/configManager.py @@ -55,7 +55,8 @@ def get_config(configName): configName: configuration section to get Returns: - dictionary representing the configuration section, None if section is not present + dictionary representing the configuration section, None if section + is not present """ return _GlobalConfig.get_config(configName) @@ -68,6 +69,7 @@ def provide_namespace(configName): configName: configuration section to get Returns: - simple namespace representing the configuration section, none if section is not present + simple namespace representing the configuration section, none if + section is not present """ return _GlobalConfig.provide_namespace(configName) diff --git a/ngcsimlib/context.py b/ngcsimlib/context.py index 2c43d68..f42cfd4 100644 --- a/ngcsimlib/context.py +++ b/ngcsimlib/context.py @@ -1,10 +1,15 @@ -from ngcsimlib.utils import make_unique_path, check_attributes, \ - check_serializable, load_from_path, get_compartment_by_name, \ - get_context, add_context, get_current_path, get_current_context, \ - set_new_context, load_module, is_pre_loaded, GuideList +from ngcsimlib.utils import (make_unique_path, check_attributes, \ + check_serializable, load_from_path, + get_compartment_by_name, \ + get_context, add_context, get_current_path, + get_current_context, \ + set_new_context, load_module, GuideList, + infer_context, + Get_Compartment_Batch, Set_Compartment_Batch) from ngcsimlib.logger import warn, info, critical from ngcsimlib import preload_modules -from ngcsimlib.compilers.command_compiler import dynamic_compile, wrap_command +from ngcsimlib.compilers import dynamic_compile, wrap_command +from ngcsimlib.compilers.process import Process from ngcsimlib.component import Component from ngcsimlib.configManager import get_config import json, os, shutil, copy @@ -52,10 +57,9 @@ def __new__(cls, name, *args, **kwargs): def __init__(self, name, should_validate=None): """ Builds the initial context object, if `__new__` provides an already - initialized context do not continue with - construction as it is already initialized. This is where the path to - a context is assigned so their paths are - dependent on the current context path upon creation. + initialized context do not continue with construction as it is + already initialized. This is where the path to a context is assigned + so their paths are dependent on the current context path upon creation. Args: name: The name of the new context can not be empty @@ -74,7 +78,8 @@ def __init__(self, name, should_validate=None): self.path = get_current_path() + "/" + str(name) self._last_context = "" - self._json_objects = {"ops": [], "components": {}, "commands": {}} + self._json_objects = {"ops": [], "components": {}, "commands": {}, + "processes": []} if should_validate is None: _base_config = get_config("context") @@ -112,8 +117,8 @@ def get_components(self, *component_names, unwrap=True): a single component is retrieved Returns: - either a list of components or a single component depending on - the number of components being retrieved + either a list of components or a single component depending on + the number of components being retrieved """ if len(component_names) == 0: return None @@ -163,8 +168,7 @@ def register_command(self, klass, *args, components=None, command_name=None, def register_component(self, component, *args, **kwargs): """ Adds a component to the local json storage for saving, will provide a - warning for all values it fails to - serialize into a json file + warning for all values it fails to serialize into a json file Args: component: the component object to save @@ -202,6 +206,17 @@ def register_component(self, component, *args, **kwargs): "kwargs": _kwargs} self._json_objects['components'][c_path] = obj + def register_process(self, process): + """ + Adds a process to the list of processes to be saved by the context. + Unlike with the other saved parts of the context the actual json objects + for the processes have to be calculated as time of save since they are + constantly changing. + Args: + process: The process to add + """ + self._json_objects['processes'].append(process) + def add_component(self, component): """ Adds a component to the context if it does not exist already in the @@ -235,10 +250,9 @@ def add_command(self, command, name=None): def save_to_json(self, directory, model_name=None, custom_save=True, overwrite=False): """ - Dumps all the required json files to rebuild the current controller - to a specified directory. If there is a - `save` command present on the controller and custom_save is True, - it will run that command as well. + Dumps all the required json files to rebuild the current controller to + a specified directory. If there is a `save` command present on the + controller and custom_save is True, it will run that command as well. Args: directory: The top level directory to save the model to @@ -251,8 +265,7 @@ def save_to_json(self, directory, model_name=None, custom_save=True, command if present on the controller (Default: True) overwrite: A boolean for if the saved model should be in a unique - folder or if it should overwrite - existing folders + folder or if it should overwrite existing folders Returns: a tuple where the first value is the path to the model, and the @@ -282,6 +295,12 @@ def save_to_json(self, directory, model_name=None, custom_save=True, with open(path + "/commands.json", 'w') as fp: json.dump(self._json_objects['commands'], fp, indent=4) + with open(path + "/processes.json", 'w') as fp: + objs = [] + for process in self._json_objects['processes']: + objs.append(process.as_obj()) + json.dump(objs, fp, indent=4) + with open(path + "/components.json", 'w') as fp: hyperparameters = {} _components = copy.deepcopy(self._json_objects['components']) @@ -353,7 +372,7 @@ def load_from_dir(self, directory, custom_folder="/custom"): components. (Default: `/custom`) """ - if os.path.isfile(directory + "/modules.json") and not is_pre_loaded(): + if os.path.isfile(directory + "/modules.json"): info("No modules file loaded, loading from model directory") preload_modules(path=directory + "/modules.json") @@ -361,6 +380,7 @@ def load_from_dir(self, directory, custom_folder="/custom"): directory + custom_folder) self.make_ops(directory + "/ops.json") self.make_commands(directory + "/commands.json") + self.make_process(directory + "/processes.json") def make_components(self, path_to_components_file, custom_file_dir=None): """ @@ -372,9 +392,8 @@ def make_components(self, path_to_components_file, custom_file_dir=None): and extension custom_file_dir: the path to the custom directory for custom load - methods, - this directory is named `custom` if the save_to_json method is - used. (Default: None) + methods, this directory is named `custom` if the save_to_json + method is used. (Default: None) """ made_components = [] with open(path_to_components_file, 'r') as file: @@ -450,7 +469,8 @@ def _make_op(self, op_spec): _sources.append(self._make_op(s)) else: _sources.append( - get_compartment_by_name(get_current_context(), s)) + get_compartment_by_name(infer_context(s, trailing_path=2), + "/".join(s.split("/")[-2:]))) obj = klass(*_sources) @@ -458,17 +478,30 @@ def _make_op(self, op_spec): return obj else: - dest = get_compartment_by_name(get_current_context(), - op_spec['destination']) + d = op_spec['destination'] + dest = get_compartment_by_name(infer_context(d, trailing_path=2), + "/".join(d.split("/")[-2:])) dest << obj + def make_process(self, path_to_process_file): + """ + Will load the processes saved in the provided json file into the model + Args: + path_to_process_file: the path to the saved json file + """ + with open(path_to_process_file, 'r') as file: + process_spec = json.load(file) + for p in process_spec: + klass = load_from_path(p["class"]) + process = Process.make_process(p, klass) + self.__setattr__(process.name, process) + @staticmethod def dynamicCommand(fn): """ Provides a decorator that will automatically bind the decorated - method to the current context. - Note this if this is called from a context object it will still use - the current context not the object + method to the current context. Note this if this is called from a + context object it will still use the current context not the object Args: fn: The wrapped method @@ -482,8 +515,8 @@ def dynamicCommand(fn): def compile_by_key(self, *components, compile_key, name=None): """ - Compiles a given set of components with a given compile key. - It will automatically add it to the context after compiling + Compiles a given set of components with a given compile key. It will + automatically add it to the context after compiling Args: *components: positional arguments for all components @@ -564,6 +597,13 @@ def make_modules(self): modules[module]["attributes"]): modules[module]["attributes"].append({"name": klass}) + jProcesses = self._json_objects['processes'] + for process in jProcesses: + mod = process.__class__.__module__ + klass = process.__class__.__name__ + modules[mod] = {"attributes": []} + modules[mod]["attributes"].append({"name": klass}) + _modules = [] for key, value in modules.items(): _modules.append( @@ -600,6 +640,7 @@ def view_guide(self, guide, skip=None): """ Views the specified guide for each component class in the model, skipping over any classes in skip. + Args: guide: A ngclearn.GuideList value skip: a list of classes to skip, will also skip component classes @@ -629,3 +670,36 @@ def view_guide(self, guide, skip=None): for klass in klasses: guides += klass.guides.__dict__[guide.value] return guides + + def _get_state_keys(self, include_special_compartments=False): + all_keys = [] + for comp_name in self.components.keys(): + all_keys.extend([key for key in Get_Compartment_Batch().keys() + if (self.path + "/" + comp_name in key and (include_special_compartments or "*" not in key))]) + return all_keys + + def get_current_state(self, include_special_compartments=False): + """ + Get the current state of the model based on the components found in this + context. + Args: + include_special_compartments: Should this method include + compartments denotes as special compartments by ngcsimlib. These are + all compartments that include * in their path. (Only creatable + dynamically) + + Returns: All the compartments found in this context. + + """ + return Get_Compartment_Batch(self._get_state_keys(include_special_compartments)) + + def update_current_state(self, state): + """ + Updates the compartments found in this context. While this method can + take a model state that includes compartments from other contexts it + will only update the compartments found in this context. + Args: + state: The state to update the model to + """ + Set_Compartment_Batch({key: value for key, value in state.items() if key in self._get_state_keys()}) + diff --git a/ngcsimlib/controller.py b/ngcsimlib/controller.py index efe5bf6..8b37ff8 100644 --- a/ngcsimlib/controller.py +++ b/ngcsimlib/controller.py @@ -1,8 +1,11 @@ -from ngcsimlib.utils import check_attributes, load_from_path, make_unique_path, check_serializable +from ngcsimlib.utils import (check_attributes, load_from_path, + make_unique_path, \ + check_serializable) from ngcsimlib.logger import warn, error, info import json, os, inspect from ngcsimlib.deprecators import deprecated + @deprecated class Controller: """ @@ -17,8 +20,10 @@ class Controller: def __init__(self): self.steps = [] self.commands = {} - self.components = {} ## components/nodes that characterize system/simulation object - self.connections = [] ## cables that characterize system/simulation object + self.components = {} ## components/nodes that characterize + # system/simulation object + self.connections = [] ## cables that characterize system/simulation + # object self._json_objects = { "commands": [], @@ -54,12 +59,14 @@ def verify_connections(self, skip_components=None): verifying connections (Default: None) """ for component in self.components.keys(): - if skip_components is not None and component.name in skip_components: + if (skip_components is not None and component.name in + skip_components): continue else: self.components[component].verify_connections() - def connect(self, source_component_name, source_compartment_name, target_component_name, + def connect(self, source_component_name, source_compartment_name, + target_component_name, target_compartment_name, bundle=None): """ Creates a cable from one component to another. @@ -80,10 +87,12 @@ def connect(self, source_component_name, source_compartment_name, target_compone cable (Default: None) """ self.components[target_component_name].create_incoming_connection( - self.components[source_component_name].create_outgoing_connection(source_compartment_name), + self.components[source_component_name].create_outgoing_connection( + source_compartment_name), target_compartment_name, bundle) - self.connections.append((source_component_name, source_compartment_name, target_component_name, + self.connections.append((source_component_name, source_compartment_name, + target_component_name, target_compartment_name, bundle)) self._json_objects['connections'].append({ "source_component_name": source_component_name, @@ -115,7 +124,8 @@ def make_components(self, path_to_components_file, custom_file_dir=None): path_to_components_file: the path to the file, including the name and extension - custom_file_dir: the path to the custom directory for custom load methods, + custom_file_dir: the path to the custom directory for custom load + methods, this directory is named `custom` if the save_to_json method is used. (Default: None) """ @@ -126,7 +136,8 @@ def make_components(self, path_to_components_file, custom_file_dir=None): components = componentsConfig["components"] if "hyperparameters" in componentsConfig.keys(): for component in components: - for pKey, pValue in componentsConfig["hyperparameters"].items(): + for pKey, pValue in componentsConfig[ + "hyperparameters"].items(): for cKey, cValue in component.items(): if pKey == cValue: component[cKey] = pValue @@ -142,7 +153,8 @@ def make_steps(self, path_to_steps_file): the specific format of this json file. Args: - path_to_steps_file: the path of the file, including the name and extension + path_to_steps_file: the path of the file, including the name and + extension """ with open(path_to_steps_file, 'r') as file: steps = json.load(file) @@ -151,8 +163,8 @@ def make_steps(self, path_to_steps_file): def make_commands(self, path_to_commands_file): """ - Loads a collection of commands from a json file. Follow `commands.schema` - for the specific format of this json file. + Loads a collection of commands from a json file. Follow + `commands.schema` for the specific format of this json file. Args: path_to_commands_file: the path of the file, including the name @@ -173,38 +185,45 @@ def add_step(self, command_name): command_name: the name of the command to be added """ if command_name not in self.commands.keys(): - raise RuntimeError(str(command_name) + " is not a registered command") + raise RuntimeError( + str(command_name) + " is not a registered command") self.steps.append(command_name) self._json_objects['steps'].append({"command_name": command_name}) - def add_component(self, component_type, match_case=False, absolute_path=False, **kwargs): + def add_component(self, component_type, match_case=False, + absolute_path=False, **kwargs): """ Acts as a component factory for the controller. Args: component_type: A string that is linked to the component class to be created. If the class was loaded with the modules.json file this - can be the keywords defined in that file. Otherwise, it will have - to be dynamically loaded using the functions found in ngcsimlib.utils. + can be the keywords defined in that file. Otherwise, it will + have to be dynamically loaded using the functions found in + ngcsimlib.utils. match_case: A boolean that represents if the exact case should be - matched when dynamically loading the component class (Default: False) + matched when dynamically loading the component class ( + Default: False) absolute_path: A boolean that represents if the component class should be treated as an absolute path when dynamically loading the component class (Default: False) kwargs: All of the keyword arguments that are needed to initialize - the loaded component class. The function will try to crash nicely - if keyword arguments are missing. This list of arguments will - also be stored to allow for the component to be rebuilt, but if - a given value is not serializable it will drop that from the - keyword arguments. + the loaded component class. The function will try to crash + nicely if keyword arguments are missing. This list of arguments + will also be stored to allow for the component to be rebuilt, + but if a given value is not serializable it will drop that from + the keyword arguments. Returns: - the created component (Component is also automatically added to the controller) + the created component (Component is also automatically added to + the controller) """ - Component_class = load_from_path(path=component_type, match_case=match_case, absolute_path=absolute_path) + Component_class = load_from_path(path=component_type, + match_case=match_case, + absolute_path=absolute_path) if inspect.isclass(Component_class): call = Component_class.__init__ @@ -224,11 +243,13 @@ def add_component(self, component_type, match_case=False, absolute_path=False, * check_attributes(component, ["name", "verify_connections"], fatal=True) self.components[component.name] = component - obj = {"component_type": component_type, "match_case": match_case, "absolute_path": absolute_path} | kwargs + obj = {"component_type": component_type, "match_case": match_case, + "absolute_path": absolute_path} | kwargs bad_keys = check_serializable(obj) for key in bad_keys: del obj[key] - info("Failed to serialize \"", key, "\" in ", component.name, sep="") + info("Failed to serialize \"", key, "\" in ", component.name, + sep="") if "directory" in obj.keys(): del obj["directory"] @@ -237,13 +258,17 @@ def add_component(self, component_type, match_case=False, absolute_path=False, * return component - def add_command(self, command_type, command_name, match_case=False, absolute_path=False, component_names=None, + def add_command(self, command_type, command_name, match_case=False, + absolute_path=False, component_names=None, **kwargs): """ Acts as a factory to create/synthesize commands. In addition to adding command objects to the controllers internal - command list, commands are also set to their attributes on the controller. - For example, if a command named `step` is added myController.runCommand("step", ...) + command list, commands are also set to their attributes on the + controller. + + For example, if a command named `step` is added + myController.runCommand("step", ...) is equivalent to myController.step(...). Args: @@ -257,7 +282,8 @@ def add_command(self, command_type, command_name, match_case=False, absolute_pat keyword that will be called elsewhere to execute this command. match_case: A boolean that represents if the exact case should be - matched when dynamically loading the command class (Default: False) + matched when dynamically loading the command class (Default: + False) absolute_path: A boolean that represents if the command class should be treated as an absolute path when dynamically loading the @@ -265,7 +291,8 @@ def add_command(self, command_type, command_name, match_case=False, absolute_pat component_names: A list of component names to be passed to the command's constructor. Internally it will convert the strings to - the actual component objects so they must exist in the controller + the actual component objects so they must exist in the + controller prior to this function being called. kwargs: All the keyword arguments that are needed to initialize the @@ -276,12 +303,16 @@ def add_command(self, command_type, command_name, match_case=False, absolute_pat keyword arguments. Returns: - the created command (Command is also automatically added to the controller) + the created command (Command is also automatically added to the + controller) """ - Command_class = load_from_path(path=command_type, match_case=match_case, absolute_path=absolute_path) + Command_class = load_from_path(path=command_type, match_case=match_case, + absolute_path=absolute_path) if not callable(Command_class): error("The object named \"", Command_class.__name__, - "\" is not callable. Please make sure the object is callable and returns a callable object", sep="") + "\" is not callable. Please make sure the object is " + "callable and returns a callable object", + sep="") if component_names is not None: componentObjs = [self.components[name] for name in component_names] @@ -298,7 +329,8 @@ def add_command(self, command_type, command_name, match_case=False, absolute_pat count = call.__code__.co_argcount - 1 named_args = call.__code__.co_varnames[1:count] try: - command = Command_class(components=componentObjs, controller=self, command_name=command_name, **kwargs) + command = Command_class(components=componentObjs, controller=self, + command_name=command_name, **kwargs) except TypeError as E: error(E, "\nProvided keyword arguments:\t", list(kwargs.keys()), "\nRequired keyword arguments:\t", list(named_args)) @@ -306,8 +338,10 @@ def add_command(self, command_type, command_name, match_case=False, absolute_pat self.commands[command_name] = command self.__setattr__(command_name, command) - obj = {"command_type": command_type, "command_name": command_name, "match_case": match_case, - "absolute_path": absolute_path, "component_names": component_names} | kwargs + obj = {"command_type": command_type, "command_name": command_name, + "match_case": match_case, + "absolute_path": absolute_path, + "component_names": component_names} | kwargs bad_keys = check_serializable(obj) for key in bad_keys: del obj[key] @@ -334,8 +368,9 @@ def runCommand(self, command_name, *args, **kwargs): def save_to_json(self, directory, model_name=None, custom_save=True): """ - Dumps all the required json files to rebuild the current controller to a specified directory. If there is a - `save` command present on the controller and custom_save is True, it will run that command as well. + Dumps all the required json files to rebuild the current controller + to a specified directory. If there is a `save` command present on the + controller and custom_save is True, it will run that command as well. Args: directory: The top level directory to save the model to @@ -389,7 +424,8 @@ def save_to_json(self, directory, model_name=None, custom_save=True): else: warn("Unable to extract hyperparameter", param, - "as it is mismatched between components. Parameter will not be extracted") + "as it is mismatched between components. " + "Parameter will not be extracted") for component in self._json_objects['components']: if "parameterMap" in component.keys(): @@ -409,7 +445,9 @@ def save_to_json(self, directory, model_name=None, custom_save=True): if check_attributes(self, ['save']): self.save(path + "/custom") else: - warn("Controller doesn't have a save command registered. No custom saving happened") + warn( + "Controller doesn't have a save command registered. No " + "custom saving happened") return (path, path + "/custom") if custom_save else (path, None) @@ -424,7 +462,8 @@ def load_from_dir(self, directory, custom_folder="/custom"): custom_folder: The name of the custom data folder for building components. (Default: `/custom`) """ - self.make_components(directory + "/components.json", directory + custom_folder) + self.make_components(directory + "/components.json", + directory + custom_folder) self.make_connections(directory + "/connections.json") self.make_commands(directory + "/commands.json") self.make_steps(directory + "/steps.json") diff --git a/ngcsimlib/logger.py b/ngcsimlib/logger.py index 59dc299..d2a0fbf 100644 --- a/ngcsimlib/logger.py +++ b/ngcsimlib/logger.py @@ -6,6 +6,7 @@ def _concatArgs(func): """Internal Decorator for concatenating arguments into a single string""" + def wrapped(*wargs, sep=" ", end="", **kwargs): msg = sep.join(str(a) for a in wargs) + end return func(msg, **kwargs) @@ -17,6 +18,7 @@ def wrapped(*wargs, sep=" ", end="", **kwargs): _mapped_calls = {} + def addLoggingLevel(levelName, levelNum, methodName=None): """ Comprehensively adds a new logging level to the `logging` module and the @@ -38,20 +40,23 @@ def addLoggingLevel(levelName, levelNum, methodName=None): methodName: The name of the method """ - if not methodName: methodName = levelName.lower() if hasattr(logging, levelName): - raise AttributeError('{} already defined in logging module'.format(levelName)) + raise AttributeError( + '{} already defined in logging module'.format(levelName)) if hasattr(logging, methodName): - raise AttributeError('{} already defined in logging module'.format(methodName)) + raise AttributeError( + '{} already defined in logging module'.format(methodName)) if hasattr(logging.getLoggerClass(), methodName): - raise AttributeError('{} already defined in logger class'.format(methodName)) + raise AttributeError( + '{} already defined in logger class'.format(methodName)) def logForLevel(self, message, *args, **kwargs): if self.isEnabledFor(levelNum): self._log(levelNum, message, args, **kwargs) + def logToRoot(message, *args, **kwargs): logging.log(levelNum, message, *args, **kwargs) @@ -63,6 +68,7 @@ def logToRoot(message, *args, **kwargs): _mapped_calls[levelNum] = getattr(_ngclogger, methodName) _mapped_calls[levelName] = getattr(_ngclogger, methodName) + def init_logging(): loggingConfig = get_config("logging") if loggingConfig is None: @@ -72,10 +78,10 @@ def init_logging(): "custom_levels": {"ANALYSIS": 25}} if loggingConfig.get("custom_levels", None) is not None: - for level_name, level_num in loggingConfig.get("custom_levels", {}).items(): + for level_name, level_num in loggingConfig.get("custom_levels", + {}).items(): addLoggingLevel(level_name.upper(), level_num) - if isinstance(loggingConfig.get("logging_level", None), str): loggingConfig["logging_level"] = \ logging.getLevelName(loggingConfig.get("logging_level", "").upper()) @@ -94,7 +100,8 @@ def init_logging(): fp.write(f"~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" f"New Log {f'{datetime.utcnow():%m/%d/%Y %H:%M:%S}'}" f"\n~~~~~~~~~~~~~~~~~~~~~~~~~~~\n") - file_handler = logging.FileHandler(filename=loggingConfig.get("logging_file", None)) + file_handler = logging.FileHandler( + filename=loggingConfig.get("logging_file", None)) file_handler.setFormatter(formatter) _ngclogger.addHandler(file_handler) @@ -103,7 +110,8 @@ def init_logging(): def warn(msg): """ Logs a warning message - This is decorated to have the same functionality of python's print argument concatenation + This is decorated to have the same functionality of python's print + argument concatenation Args: msg: message to log @@ -115,7 +123,8 @@ def warn(msg): def error(msg): """ Logs an error message - This is decorated to have the same functionality of python's print argument concatenation + This is decorated to have the same functionality of python's print + argument concatenation Args: msg: message to log @@ -128,7 +137,8 @@ def error(msg): def critical(msg): """ Logs a critical message - This is decorated to have the same functionality of python's print argument concatenation + This is decorated to have the same functionality of python's print + argument concatenation Args: msg: message to log @@ -141,7 +151,8 @@ def critical(msg): def info(msg): """ Logs an info message - This is decorated to have the same functionality of python's print argument concatenation + This is decorated to have the same functionality of python's print + argument concatenation Args: msg: message to log @@ -153,7 +164,8 @@ def info(msg): def debug(msg): """ Logs a debug message - This is decorated to have the same functionality of python's print argument concatenation + This is decorated to have the same functionality of python's print + argument concatenation Args: msg: message to log @@ -164,16 +176,17 @@ def debug(msg): @_concatArgs def custom_log(msg, logging_level=None): """ - Logs to a user defined logging level. This will only work for user defined - levels if a builtin logging level is desired please use on of the builtin - logging methods found in this file. To defined logging levels add them to the - configuration file of your project. To add levels here add the map of - `logging_levels` to the top level logging object and have the key be the new - logging level name, and the value be the numerical logging value. To see all - build in logging levels look at the builtin python logger package. - - - This is decorated to have the same functionality of python's print argument concatenation + Logs to a user defined logging level. This will only work for user + defined levels if a builtin logging level is desired please use on of the + builtin logging methods found in this file. To defined logging levels add + them to the configuration file of your project. To add levels here add + the map of `logging_levels` to the top level logging object and have the + key be the new logging level name, and the value be the numerical logging + value. To see all build in logging levels look at the builtin python + logger package. + + This is decorated to have the same functionality of python's print + argument concatenation Args: msg: The message to log @@ -189,4 +202,3 @@ def custom_log(msg, logging_level=None): warn("Attempted to log to undefined level", logging_level) else: _mapped_calls[logging_level](msg) - diff --git a/ngcsimlib/metaComponent.py b/ngcsimlib/metaComponent.py index 7d2b7cb..cca7f5a 100644 --- a/ngcsimlib/metaComponent.py +++ b/ngcsimlib/metaComponent.py @@ -1,14 +1,15 @@ from ngcsimlib.compartment import Compartment from ngcsimlib.utils import get_current_context from ngcsimlib.utils.help import Guides -from ngcsimlib.logger import debug +from ngcsimlib.logger import debug, warn class MetaComponent(type): """ - This is the metaclass for the component objects in ngc-learn. This does a large amount of setup work behind the - scenes to link everything together in the context that it is constructed in. In addition to this it also is - responsible for adding all compartment value to the global hashmap. + This is the metaclass for the component objects in ngc-learn. This does a + large amount of setup work behind the scenes to link everything together + in the context that it is constructed in. In addition to this it also + is responsible for adding all compartment value to the global hashmap. """ @staticmethod @@ -40,7 +41,8 @@ def post_init(self, *args, **kwargs): @staticmethod def add_connection(self, op): """ - A needed function by compartments to be able to add incoming connections to their parent component + A needed function by compartments to be able to add incoming + connections to their parent component """ self.connections.append(op) get_current_context().register_op(op) @@ -48,7 +50,8 @@ def add_connection(self, op): @staticmethod def gather(self): """ - Runs all the connections for the given component to collect values for its own compartments + Runs all the connections for the given component to collect values + for its own compartments """ for comm in self.connections: comm() @@ -70,7 +73,8 @@ def _format_defaults(cls, params): def __new__(cls, *clargs, **clkwargs): """ - Wraps the class adding a pre/post-init method and some additional methods + Wraps the class adding a pre/post-init method and some additional + methods """ x = super().__new__(cls, *clargs, **clkwargs) @@ -82,7 +86,16 @@ def wrapped_init(self, *args, **kwargs): else: cls.pre_init(self, *args, **kwargs) x._orig_init(self, *args, **kwargs) + args_count = self._orig_init.__code__.co_argcount + _kwargs = self._orig_init.__code__.co_varnames[:args_count] + for key, value in kwargs.items(): + if key not in _kwargs: + debug( + f"There is an extra param {key} in component " + f"constructor for {self.name}") cls.post_init(self, *args, **kwargs) + if hasattr(self, "_setup"): + self._setup() x.__init__ = wrapped_init @@ -91,6 +104,7 @@ def wrapped_init(self, *args, **kwargs): if hasattr(x, "help"): _orig_help = x.help + def _wrapped_help(*args): info = _orig_help() if info is not None: @@ -98,8 +112,9 @@ def _wrapped_help(*args): x._format_defaults(x, info["hyperparameters"]) return info + x.help = _wrapped_help - x.guides = Guides(x) + x.guides = Guides(x) return x diff --git a/ngcsimlib/operations/add.py b/ngcsimlib/operations/add.py index cab10f8..b42c66f 100644 --- a/ngcsimlib/operations/add.py +++ b/ngcsimlib/operations/add.py @@ -1,16 +1,16 @@ from .summation import summation + class add(summation): """ Not Compilable - A subclass of summation that also adds the destinations value instead of overwriting it. For a compiler friendly - version of this add the destination as a source to summation. + A subclass of summation that also adds the destinations value instead of + overwriting it. For a compiler friendly version of this add the + destination as a source to summation. """ is_compilable = False def resolve(self, value): if self.destination is not None: self.destination.set(self.destination.value + value) - - diff --git a/ngcsimlib/operations/baseOp.py b/ngcsimlib/operations/baseOp.py index 453507b..5239048 100644 --- a/ngcsimlib/operations/baseOp.py +++ b/ngcsimlib/operations/baseOp.py @@ -13,7 +13,7 @@ class BaseOp(ABC): For commands that can be compiled using ngcsimlib's compiler, all their operational logic must be contained inside the subclass's operation - method. This also means that the resolve method that is defined on the + method. This also means that the transition method that is defined on the base class should not be overwritten. For commands that do not need to be compiled using ngcsimlib's compiler @@ -95,9 +95,9 @@ def dump(self): if isinstance(source, BaseOp): source_array.append(source.dump()) else: - source_array.append(source.name) + source_array.append(source.path) - destination = self.destination.name if self.destination is not None \ + destination = self.destination.path if self.destination is not None \ else None return {"class": class_name, "sources": source_array, diff --git a/ngcsimlib/operations/negate.py b/ngcsimlib/operations/negate.py index 59a1993..b3ff2ae 100644 --- a/ngcsimlib/operations/negate.py +++ b/ngcsimlib/operations/negate.py @@ -1,9 +1,12 @@ from .baseOp import BaseOp + + class negate(BaseOp): """ - negates the first source compartment (other will be ignored) and overwrite the previous value + negates the first source compartment (other will be ignored) and + overwrite the previous value """ @staticmethod def operation(*sources): - return -sources[0] \ No newline at end of file + return -sources[0] diff --git a/ngcsimlib/operations/overwrite.py b/ngcsimlib/operations/overwrite.py index 816c37f..77c4d81 100644 --- a/ngcsimlib/operations/overwrite.py +++ b/ngcsimlib/operations/overwrite.py @@ -1,9 +1,13 @@ from .baseOp import BaseOp + + class overwrite(BaseOp): """ The default operation behavior for cable's - Overwrites the previous value with the first source value (all other sources will be ignored) + Overwrites the previous value with the first source value (all other + sources will be ignored) """ + @staticmethod def operation(*sources): return sources[0] diff --git a/ngcsimlib/operations/summation.py b/ngcsimlib/operations/summation.py index df28d08..753efbe 100644 --- a/ngcsimlib/operations/summation.py +++ b/ngcsimlib/operations/summation.py @@ -1,8 +1,10 @@ from .baseOp import BaseOp + class summation(BaseOp): """ - Adds together all the provided compartment's values and overwrites the previous value + Adds together all the provided compartment's values and overwrites the + previous value """ @staticmethod @@ -13,4 +15,4 @@ def operation(*sources): s = source else: s += source - return s \ No newline at end of file + return s diff --git a/ngcsimlib/resolver.py b/ngcsimlib/resolver.py index 2e04c8e..1751c60 100644 --- a/ngcsimlib/resolver.py +++ b/ngcsimlib/resolver.py @@ -1,35 +1,46 @@ """ -The resolver is an important part of the compiling of components and commands in the ngcsimlib compilers. - -At its core the resolver links a pure (static) method with a class method that knows how to process the output values -of the pure method and set the correct compartment's values from it. - -While a resolver can take in many arguments generally this is not needed as it will automatically map all the values it -needs from argument lines. When writing the resolvers and the pure functions being passed into them the names of the -arguments are very important, this extends to the name of the method the resolver is wrapping as that is the name the -resolver is saved under. When the compiler is searching for how to compile the compile key for a given component it -searches through all the resolvers on the class for any wrapping a method with the same name as the compile key. On -top of serving as the compile key the wrapped function provides the knowledge of where output values of the compiled -function should go and should not contain any runtime logic as this wrapped method will not be called when the -command is compiled. - -The parsing of output_compartments, args, parameters, and compartments is done during compiling the resolvers, -but it will be explained here. - -To parse the output_compartments the resolver looks at the names and order of the arguments in the method it is -wrapping. When it is compiled these names are the names of the compartments the compiled method will try to locate to -place the resultant values in. It is important to note that the number of output values of the pure method and the -number of input values should match for automatic mapping. - -To parse the args, parameters, and compartments the argument names of the pure method are considered. First the -compiler will check to see if the name exists as an attribute on the object. If it does not exist the compiler assume -that this value is an argument being passed in by the user. In the event that it is present on the object it checks -to see if it is an instance of a compartment. In the event that it is it will add it to the compartments list -otherwise it will add it to the parameter list. +Todo: rewrite to have this be the less favorable option + +The resolver is an important part of the compiling of components and commands +in the ngcsimlib compilers. + +At its core the resolver links a pure (static) method with a class method +that knows how to process the output values of the pure method and set the +correct compartment's values from it. + +While a resolver can take in many arguments generally this is not needed as +it will automatically map all the values it needs from argument lines. When +writing the resolvers and the pure functions being passed into them the names +of the arguments are very important, this extends to the name of the method +the resolver is wrapping as that is the name the resolver is saved under. +When the compiler is searching for how to compile the compile key for a given +component it searches through all the resolvers on the class for any wrapping +a method with the same name as the compile key. On top of serving as the +compile key the wrapped function provides the knowledge of where output +values of the compiled function should go and should not contain any runtime +logic as this wrapped method will not be called when the command is compiled. + +The parsing of output_compartments, args, parameters, and compartments is +done during compiling the resolvers, but it will be explained here. + +To parse the output_compartments the resolver looks at the names and order of +the arguments in the method it is wrapping. When it is compiled these names +are the names of the compartments the compiled method will try to locate to +place the resultant values in. It is important to note that the number of +output values of the pure method and the number of input values should match +for automatic mapping. + +To parse the args, parameters, and compartments the argument names of the +pure method are considered. First the compiler will check to see if the name +exists as an attribute on the object. If it does not exist the compiler +assume that this value is an argument being passed in by the user. In the +event that it is present on the object it checks to see if it is an instance +of a compartment. In the event that it is it will add it to the compartments +list otherwise it will add it to the parameter list. """ from ngcsimlib.compartment import Compartment -from ngcsimlib.utils import add_component_resolver, add_resolver_meta +from ngcsimlib.utils import add_component_transition, add_transition_meta def resolver(pure_fn, @@ -40,24 +51,30 @@ def resolver(pure_fn, expand_args=True ): """ - The decorator used to tell the ngcsimlib compiler how to compile methods on components. + The decorator used to tell the ngcsimlib compiler how to compile methods + on components. Args: pure_fn: the pure function where the run time logic is located output_compartments: a list of output compartment names (default: None) - args: a list of arguments being passed into the pure function (default: None) + args: a list of arguments being passed into the pure function + (default: None) - parameters: a list of parameters being passed into the pure function (default: None) + parameters: a list of parameters being passed into the pure function + (default: None) - compartments: a list of compartments being passed into the pure function (default: None) + compartments: a list of compartments being passed into the pure + function (default: None) - expand_args: should the output of the pure method be expanded or kept as a tuple when being passed into the - wrapped method. (default: True) + expand_args: should the output of the pure method be expanded or kept + as a tuple when being passed into the wrapped method. + (default: True) Returns: - A wrapped method that will pass the output of the pure function into the resolve when called. + A wrapped method that will pass the output of the pure function into + the resolve when called. """ if not (args is None and parameters is None and compartments is None): @@ -70,7 +87,8 @@ def resolver(pure_fn, compartments = [] else: parse_varnames = True - varnames = pure_fn.__func__.__code__.co_varnames[:pure_fn.__func__.__code__.co_argcount] + varnames = pure_fn.__func__.__code__.co_varnames[ + :pure_fn.__func__.__code__.co_argcount] if output_compartments is None: output_compartments = [] @@ -83,9 +101,11 @@ def _resolver(fn): class_name = ".".join(fn.__qualname__.split('.')[:-1]) resolver_key = fn.__qualname__.split('.')[-1] - add_component_resolver(class_name, resolver_key, (pure_fn, output_compartments)) + add_component_transition(class_name, resolver_key, + (pure_fn, output_compartments)) - add_resolver_meta(class_name, resolver_key, (args, parameters, compartments, parse_varnames)) + add_transition_meta(class_name, resolver_key, + (args, parameters, compartments, parse_varnames)) def _wrapped(self=None, *_args, **_kwargs): comps = {} diff --git a/ngcsimlib/utils/__init__.py b/ngcsimlib/utils/__init__.py index ecb1590..4e15337 100644 --- a/ngcsimlib/utils/__init__.py +++ b/ngcsimlib/utils/__init__.py @@ -3,5 +3,5 @@ from .io import * from .misc import * from .modules import * -from .resolvers import * +from .transitions import * from .help import * diff --git a/ngcsimlib/utils/compartment.py b/ngcsimlib/utils/compartment.py index e9935b9..24df210 100644 --- a/ngcsimlib/utils/compartment.py +++ b/ngcsimlib/utils/compartment.py @@ -3,11 +3,13 @@ __all_compartments = {} + def Get_Compartment_Batch(compartment_uids=None): """ This method should be used sparingly - Get a subset of all compartment values based on provided paths. If no paths are provided it will grab all of them + Get a subset of all compartment values based on provided paths. If no + paths are provided it will grab all of them Args: compartment_uids: needed ids @@ -25,7 +27,8 @@ def Set_Compartment_Batch(compartment_map=None): """ This method should be used sparingly - Sets a subset of compartments to their corresponding value in the provided dictionary + Sets a subset of compartments to their corresponding value in the + provided dictionary Args: compartment_map: a map of compartment paths to values @@ -53,4 +56,3 @@ def get_compartment_by_name(context, name): """ return __all_compartments.get(context.path + "/" + name, None) - diff --git a/ngcsimlib/utils/context.py b/ngcsimlib/utils/context.py index 47c1503..7bc2845 100644 --- a/ngcsimlib/utils/context.py +++ b/ngcsimlib/utils/context.py @@ -19,11 +19,34 @@ def get_current_path(): def get_context(path): """ - A helper method for getting a context by a provided path + A helper method for getting a context by a provided path, to search from the + root path start the path with '/' """ + if path[0] == "/": + return __contexts.get(path, None) + return __contexts.get(__current_context + "/" + path, None) +def infer_context(path, trailing_path=1): + """ + A helper method that attempts to get the given context by a provided path, if + the context does not exist it will return the current context + Args: + path: path to where the context should be + trailing_path: how many trailing path locations should be ignored in + general 1 for components, 2 for compartments + + + Returns: located context object or the current context if unable to locate + given context + """ + ctx = get_context("/".join(path.split("/")[:-trailing_path])) + + if ctx is None: + return get_current_context() + return ctx + def add_context(name, con): """ A helper method for adding a context to the current path diff --git a/ngcsimlib/utils/io.py b/ngcsimlib/utils/io.py index 2772059..0364bff 100644 --- a/ngcsimlib/utils/io.py +++ b/ngcsimlib/utils/io.py @@ -3,8 +3,9 @@ def make_unique_path(directory, root_name=None): """ - This block of code will make a uniquely named directory inside the specified output folder. - If the root name already exists it will append a UID to the root name to not overwrite data + This block of code will make a uniquely named directory inside the + specified output folder. If the root name already exists it will append a + UID to the root name to not overwrite data Args: directory: The root directory to save models to @@ -22,7 +23,8 @@ def make_unique_path(directory, root_name=None): elif os.path.isdir(directory + "/" + root_name): root_name += "_" + str(uid) - print("root path already exists, generated path will be named \"" + str(root_name) + "\"") + print("root path already exists, generated path will be named \"" + str( + root_name) + "\"") path = directory + "/" + str(root_name) os.mkdir(path) diff --git a/ngcsimlib/utils/misc.py b/ngcsimlib/utils/misc.py index 2c067e6..8477a1c 100644 --- a/ngcsimlib/utils/misc.py +++ b/ngcsimlib/utils/misc.py @@ -1,20 +1,23 @@ def extract_args(keywords=None, *args, **kwargs): """ - Extracts the given keywords from the provided args and kwargs. This method first finds all the matching keywords - then for each missing keyword it takes the next value in args and assigns it. It will throw and error if there are - not enough kwargs and args to satisfy all provided keywords + Extracts the given keywords from the provided args and kwargs. This + method first finds all the matching keywords then for each missing + keyword it takes the next value in args and assigns it. It will throw and + error if there are not enough kwargs and args to satisfy all provided + keywords Args: keywords: a list of keywords to extract - args: the positional arguments to use as a fallback over keyword arguments + args: the positional arguments to use as a fallback over keyword + arguments kwargs: the keyword arguments to first try to extract from Returns: a dictionary for where each keyword is a key, and the value is assigned - argument. Will throw a RuntimeError if it fails to match and argument - to each keyword. + argument. Will throw a RuntimeError if it fails to match and argument to + each keyword. """ if keywords is None: return None diff --git a/ngcsimlib/utils/modules.py b/ngcsimlib/utils/modules.py index 02a8f6a..546d00f 100644 --- a/ngcsimlib/utils/modules.py +++ b/ngcsimlib/utils/modules.py @@ -2,20 +2,15 @@ from importlib import import_module from ngcsimlib.logger import info -## Globally tracking all the modules, and attributes have been dynamically loaded +## Globally tracking all the modules, and attributes have been dynamically +# loaded _Loaded_Attributes = {} _Loaded_Modules = {} -_Loaded = False -def is_pre_loaded(): - return _Loaded - -def set_loaded(val): - global _Loaded - _Loaded = val def check_attributes(obj, required, fatal=False): """ - This function will verify that a provided object has the requested attributes. + This function will verify that a provided object has the requested + attributes. Args: obj: Object that should have the attributes @@ -25,7 +20,8 @@ def check_attributes(obj, required, fatal=False): fatal: If true an Attribute error will be thrown (default False) Returns: - Boolean only returns if not fatal, if the object has the required attributes + Boolean only returns if not fatal, if the object has the required + attributes """ if required is None: return True @@ -34,9 +30,13 @@ def check_attributes(obj, required, fatal=False): if not fatal: return False if hasattr(obj, "name"): - raise AttributeError(str(obj.name) + " is missing the required attribute of " + atr) + raise AttributeError( + str(obj.name) + " is missing the required attribute of " + + atr) else: - raise AttributeError("Checked object is missing the required attribute of " + atr) + raise AttributeError( + "Checked object is missing the required attribute of " + + atr) return True @@ -45,7 +45,8 @@ def load_module(module_path, match_case=False, absolute_path=False): Trys to load a module from the provided path. Args: - module_path: Module path, supports compound modules such as `ngcsimlib.commands` + module_path: Module path, supports compound modules such as + `ngcsimlib.commands` match_case: If true the module must case match exactly (default false) @@ -58,15 +59,15 @@ def load_module(module_path, match_case=False, absolute_path=False): # Return if we have already loaded this module if module_path in _Loaded_Modules.keys(): return _Loaded_Modules[module_path] - # Unkown module + # Unknown module module_name = None if absolute_path: module_name = module_path else: + # Extract the final module from the module_path final_mod = module_path.split('.')[-1] final_mod = final_mod if match_case else final_mod.lower() - # Try to match the final module to any currently loaded module for module in sys.modules: last_mod = module.split('.')[-1] @@ -78,7 +79,8 @@ def load_module(module_path, match_case=False, absolute_path=False): # Will only be None if no imported modules match the import name if module_name is None: - raise RuntimeError("Failed to find dynamic import for \"" + module_path + "\"") + raise RuntimeError( + "Failed to find dynamic import for \"" + module_path + "\"") mod = import_module(module_name) _Loaded_Modules[module_path] = mod @@ -87,15 +89,18 @@ def load_module(module_path, match_case=False, absolute_path=False): def load_from_path(path, match_case=False, absolute_path=False): """ - Loads an attribute/module from a specified path. If not using the absolute path the module name and attribute - names will be assumed to be the same. + Loads an attribute/module from a specified path. If not using the + absolute path the module name and attribute names will be assumed to be + the same. Args: - path: path to attribute/module to load, will try to find the attribute/module if not already loaded + path: path to attribute/module to load, will try to find the + attribute/module if not already loaded match_case: If true the module must case match exactly (default false) - absolute_path: If true tries to import exactly what is passed to module path (default false) + absolute_path: If true tries to import exactly what is passed to + module path (default false) Returns: The attribute at the path @@ -112,7 +117,8 @@ def load_from_path(path, match_case=False, absolute_path=False): match_case=match_case, absolute_path=absolute_path) -def load_attribute(attribute_name, module_path=None, match_case=False, absolute_path=False): +def load_attribute(attribute_name, module_path=None, match_case=False, + absolute_path=False): """ Loads a specific attribute from a specified module @@ -136,16 +142,23 @@ def load_attribute(attribute_name, module_path=None, match_case=False, absolute_ if attribute_name is None: raise RuntimeError() - mod = load_module(attribute_name if module_path is None else module_path, match_case=match_case, + mod = load_module(attribute_name if module_path is None else module_path, + match_case=match_case, absolute_path=absolute_path) - attribute_name = attribute_name if match_case else attribute_name[0].upper() + attribute_name[1:] + attribute_name = attribute_name if match_case else (attribute_name[ + 0].upper() + + attribute_name[ + 1:]) try: attr = getattr(mod, attribute_name) except AttributeError: - raise RuntimeError("Could not find an attribute with name \"" + attribute_name + "\" in module " + - mod.__name__) \ + raise RuntimeError( + "Could not find an attribute with name \"" + attribute_name + "\" " + "in" + " module " + + mod.__name__) \ from None _Loaded_Attributes[attribute_name] = attr diff --git a/ngcsimlib/utils/resolvers.py b/ngcsimlib/utils/resolvers.py deleted file mode 100644 index f4c4e63..0000000 --- a/ngcsimlib/utils/resolvers.py +++ /dev/null @@ -1,64 +0,0 @@ -from ngcsimlib.logger import critical, debug - -__component_resolvers = {} -__resolver_meta_data = {} - - -def get_resolver(klass, resolver_key, root=None): - """ - A helper method for searching through the resolver list - """ - class_name = klass.__name__ - - if class_name + "/" + resolver_key not in __component_resolvers.keys(): - parent_classes = klass.__bases__ - if len(parent_classes) == 0: - return None, None - - resolver = None - for parent in parent_classes: - resolver, meta = get_resolver(parent, resolver_key, root=klass if root is None else root) - if resolver is not None: - return resolver, meta - - if resolver is None and root is None: - critical(class_name, "has no resolver for", resolver_key) - if resolver is None: - return None, None - - if root is not None: - debug(f"{root.__name__} is using the resolver from {class_name} for resolving key \"{resolver_key}\"") - return __component_resolvers[class_name + "/" + resolver_key], __resolver_meta_data[class_name + "/" + resolver_key] - - -def add_component_resolver(class_name, resolver_key, data): - """ - A helper function for adding component resolvers - """ - __component_resolvers[class_name + "/" + resolver_key] = data - - -def add_resolver_meta(class_name, resolver_key, data): - """ - A helper function for adding component resolvers metadata - """ - __resolver_meta_data[class_name + "/" + resolver_key] = data - -def using_resolver(**kwargs): - """ - A decorator for linking resolvers defined in other classes to this class. - the keyword arguments are compile_key=class_to_inherit_resolver_from. This - will add the resolver directly to the class and thus will get used before - any resolvers in parent classes. - - Args: - **kwargs: any number or compile_key=class_to_inherit_resolver_from - """ - def _klass_wrapper(cls): - klass_name = cls.__name__ - for key, value in kwargs.items(): - resolver, data = get_resolver(value, key) - add_component_resolver(klass_name, key, resolver) - add_resolver_meta(klass_name, key, data) - return cls - return _klass_wrapper diff --git a/ngcsimlib/utils/transitions.py b/ngcsimlib/utils/transitions.py new file mode 100644 index 0000000..013e832 --- /dev/null +++ b/ngcsimlib/utils/transitions.py @@ -0,0 +1,71 @@ +from ngcsimlib.logger import critical, debug + +__component_transitions = {} +__transition_meta_data = {} + + +def get_transition(klass, transition_key, root=None): + """ + A helper method for searching through the transition list + """ + class_name = klass.__name__ + + if class_name + "/" + transition_key not in __component_transitions.keys(): + parent_classes = klass.__bases__ + if len(parent_classes) == 0: + return None, None + + transition = None + for parent in parent_classes: + transition, meta = get_transition(parent, transition_key, + root=klass if root is None else root) + if transition is not None: + return transition, meta + + if transition is None and root is None: + critical(class_name, "has no transition for", transition_key) + if transition is None: + return None, None + + if root is not None: + debug( + f"{root.__name__} is using the transition from {class_name} for " + f"resolving key \"{transition_key}\"") + return __component_transitions[class_name + "/" + transition_key], \ + __transition_meta_data[class_name + "/" + transition_key] + + +def add_component_transition(class_name, transition_key, data): + """ + A helper function for adding component transitions + """ + __component_transitions[class_name + "/" + transition_key] = data + + +def add_transition_meta(class_name, transition_key, data): + """ + A helper function for adding component transition metadata + """ + __transition_meta_data[class_name + "/" + transition_key] = data + + +def using_transition(**kwargs): + """ + A decorator for linking transitions defined in other classes to this class. + the keyword arguments are compile_key=class_to_inherit_transition_from. This + will add the transition directly to the class and thus will get used before + any transitions in parent classes. + + Args: + **kwargs: any number or compile_key=class_to_inherit_transition_from + """ + + def _klass_wrapper(cls): + klass_name = cls.__name__ + for key, value in kwargs.items(): + transition, data = get_transition(value, key) + add_component_transition(klass_name, key, transition) + add_transition_meta(klass_name, key, data) + return cls + + return _klass_wrapper diff --git a/pyproject.toml b/pyproject.toml index b551719..4ad7b5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "ngcsimlib" -version = "0.3.beta4" +version = "0.3.beta5" description = "Simulation software backend for ngc-learn." authors = [ diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..5f7e5da --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +# content of pytest.ini +[pytest] +python_files = *_test.py +python_classes = *Test +python_functions = test_* *_test +testpaths = tests \ No newline at end of file diff --git a/tests/compartment_test.py b/tests/compartment_test.py new file mode 100644 index 0000000..85bc3b9 --- /dev/null +++ b/tests/compartment_test.py @@ -0,0 +1,22 @@ + +import pathlib +import sys + +# NOTE: VN: Since we have installed using `pip install -e .` in the workflow file, we +# might not need to manually add to the path +sys.path.append(str(pathlib.Path(__file__).parent.parent)) + +# import ngcsimlib + +class CompartmentTest: + + def test_import(self): + success = False + try: + from ngcsimlib.compartment import Compartment + success = True + except: + success = False + assert success, "Import failed!" + +