From 2a88a0a3fca8249db44afeb44e61eaf1e0aa0415 Mon Sep 17 00:00:00 2001 From: Feiyang Date: Fri, 26 Apr 2024 17:18:25 +0100 Subject: [PATCH 01/13] add type hinting --- cvxpygen/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cvxpygen/utils.py b/cvxpygen/utils.py index 3a9b931..77570c3 100644 --- a/cvxpygen/utils.py +++ b/cvxpygen/utils.py @@ -11,9 +11,13 @@ limitations under the License. """ +from io import TextIOWrapper import numpy as np from datetime import datetime +from cvxpygen.mappings import Configuration, DualVariableInfo, ParameterInfo, VariableInfo +from cvxpygen.solvers import SolverInterface + def write_file(path, mode, function, *args): """Write data to a file using a specific utility function.""" @@ -1258,7 +1262,7 @@ def replace_setup_data(text): return text.replace('%DATE', now.strftime("on %B %d, %Y at %H:%M:%S")) -def write_method(f, configuration, variable_info, dual_variable_info, parameter_info, solver_interface): +def write_method(f: TextIOWrapper, configuration: Configuration, variable_info: VariableInfo, dual_variable_info: DualVariableInfo, parameter_info: ParameterInfo, solver_interface: SolverInterface): """ Write function to be registered as custom CVXPY solve method """ From 706b89db53c09616a8555d4ef4f671d14dc04938 Mon Sep 17 00:00:00 2001 From: Feiyang Date: Fri, 26 Apr 2024 17:18:53 +0100 Subject: [PATCH 02/13] add py interface auto generate --- cvxpygen/utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cvxpygen/utils.py b/cvxpygen/utils.py index 77570c3..255c9b5 100644 --- a/cvxpygen/utils.py +++ b/cvxpygen/utils.py @@ -1262,7 +1262,14 @@ def replace_setup_data(text): return text.replace('%DATE', now.strftime("on %B %d, %Y at %H:%M:%S")) -def write_method(f: TextIOWrapper, configuration: Configuration, variable_info: VariableInfo, dual_variable_info: DualVariableInfo, parameter_info: ParameterInfo, solver_interface: SolverInterface): +def write_method( + f: TextIOWrapper, + configuration: Configuration, + variable_info: VariableInfo, + dual_variable_info: DualVariableInfo, + parameter_info: ParameterInfo, + solver_interface: SolverInterface, +): """ Write function to be registered as custom CVXPY solve method """ From b623e3568fe98e80777c0057832bdc828af7b31d Mon Sep 17 00:00:00 2001 From: Feiyang Date: Fri, 26 Apr 2024 17:27:50 +0100 Subject: [PATCH 03/13] add write interface method --- cvxpygen/cpg.py | 45 ++++++++++++++++++++++++++++++++++----------- cvxpygen/utils.py | 11 +++++++++++ 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/cvxpygen/cpg.py b/cvxpygen/cpg.py index ab9561c..21416a5 100644 --- a/cvxpygen/cpg.py +++ b/cvxpygen/cpg.py @@ -12,24 +12,42 @@ """ import os -import sys -import shutil import pickle +import shutil +import sys import warnings +from subprocess import call -from cvxpygen import utils -from cvxpygen.utils import write_file, read_write_file, write_example_def, write_module_prot, write_module_def, \ - write_canon_cmake, write_method, replace_cmake_data, replace_setup_data, replace_html_data -from cvxpygen.mappings import Configuration, PrimalVariableInfo, DualVariableInfo, ConstraintInfo, \ - ParameterCanon, ParameterInfo -from cvxpygen.solvers import get_interface_class import cvxpy as cp import numpy as np -from scipy import sparse -from subprocess import call -from cvxpy.problems.objective import Maximize from cvxpy.cvxcore.python import canonInterface as cI from cvxpy.expressions.variable import upper_tri_to_full +from cvxpy.problems.objective import Maximize +from scipy import sparse + +from cvxpygen import utils +from cvxpygen.mappings import ( + Configuration, + ConstraintInfo, + DualVariableInfo, + ParameterCanon, + ParameterInfo, + PrimalVariableInfo, +) +from cvxpygen.solvers import get_interface_class +from cvxpygen.utils import ( + read_write_file, + replace_cmake_data, + replace_html_data, + replace_setup_data, + write_canon_cmake, + write_example_def, + write_file, + write_method, + write_interface, + write_module_def, + write_module_prot, +) def generate_code(problem, code_dir='CPG_code', solver=None, solver_opts=None, @@ -375,6 +393,11 @@ def write_c_code(problem: cp.Problem, configuration: Configuration, variable_inf configuration, variable_info, dual_variable_info, parameter_info, solver_interface) + write_file(os.path.join(configuration.code_dir, 'cpg_module.pyi'), 'w', + write_interface, + configuration, variable_info, dual_variable_info, + parameter_info, solver_interface) + write_file(os.path.join(configuration.code_dir, 'problem.pickle'), 'wb', lambda x, y: pickle.dump(y, x), cp.Problem(problem.objective, problem.constraints)) diff --git a/cvxpygen/utils.py b/cvxpygen/utils.py index 255c9b5..9f0b30a 100644 --- a/cvxpygen/utils.py +++ b/cvxpygen/utils.py @@ -1252,6 +1252,17 @@ def write_module_prot(f, configuration, parameter_info, variable_info, dual_vari f'struct {configuration.prefix}CPG_Params_cpp_t& CPG_Params_cpp);\n') +def write_interface( + f: TextIOWrapper, + configuration: Configuration, + variable_info: VariableInfo, + dual_variable_info: DualVariableInfo, + parameter_info: ParameterInfo, + solver_interface: SolverInterface, +): + write_description(f, 'py', 'Python extension stub file.') + + def replace_setup_data(text): """ Replace placeholder strings in setup.py file From e56b18f669b02e585de8c442c37d31029ba18d21 Mon Sep 17 00:00:00 2001 From: Feiyang Date: Fri, 26 Apr 2024 17:32:17 +0100 Subject: [PATCH 04/13] use getattr instead of eval --- cvxpygen/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvxpygen/utils.py b/cvxpygen/utils.py index 9f0b30a..bdc1eb0 100644 --- a/cvxpygen/utils.py +++ b/cvxpygen/utils.py @@ -1311,7 +1311,7 @@ def write_method( f.write(' cpg_module.set_solver_default_settings()\n') f.write(' for key, value in kwargs.items():\n') f.write(' try:\n') - f.write(' eval(f\'cpg_module.set_solver_{standard_settings_names.get(key, key)}(value)\')\n') + f.write(' getattr(cpg_module, f\'set_solver_{standard_settings_names.get(key, key)}(value)\')\n') f.write(' except AttributeError:\n') f.write(' raise AttributeError(f\'Solver setting "{key}" not available.\')\n\n') From 373a5ae3d0fb690cdec0b6a56c0b4d34c8472fd3 Mon Sep 17 00:00:00 2001 From: Feiyang Date: Fri, 26 Apr 2024 17:38:19 +0100 Subject: [PATCH 05/13] add basic interface file --- cvxpygen/utils.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/cvxpygen/utils.py b/cvxpygen/utils.py index bdc1eb0..6910a3e 100644 --- a/cvxpygen/utils.py +++ b/cvxpygen/utils.py @@ -12,6 +12,7 @@ """ from io import TextIOWrapper +import textwrap import numpy as np from datetime import datetime @@ -1261,7 +1262,25 @@ def write_interface( solver_interface: SolverInterface, ): write_description(f, 'py', 'Python extension stub file.') - + interface_content = f""" + def {configuration.prefix}cpg_updated(...): + ... + + def set_solver_default_settings(...): + ... + + # f\'set_solver_{{standard_settings_names.get(key, key)}}(value) + + def {configuration.prefix}cpg_params(): + ... + + def solve(upd, par): + ... + """ + + f.write( + textwrap.dedent(interface_content) + ) def replace_setup_data(text): """ From a87d27d7bdcbe7866249dd57eb698a0d0029685b Mon Sep 17 00:00:00 2001 From: Feiyang Date: Fri, 26 Apr 2024 17:42:20 +0100 Subject: [PATCH 06/13] only import if type checking --- cvxpygen/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cvxpygen/utils.py b/cvxpygen/utils.py index 6910a3e..51c0f9f 100644 --- a/cvxpygen/utils.py +++ b/cvxpygen/utils.py @@ -13,11 +13,14 @@ from io import TextIOWrapper import textwrap +from typing import TYPE_CHECKING import numpy as np from datetime import datetime -from cvxpygen.mappings import Configuration, DualVariableInfo, ParameterInfo, VariableInfo -from cvxpygen.solvers import SolverInterface + +if TYPE_CHECKING: + from cvxpygen.mappings import Configuration, DualVariableInfo, ParameterInfo, VariableInfo + from cvxpygen.solvers import SolverInterface def write_file(path, mode, function, *args): From 271c43bc6572a498e29e1f07baafb1a9b04d0b96 Mon Sep 17 00:00:00 2001 From: Feiyang Date: Fri, 26 Apr 2024 17:43:16 +0100 Subject: [PATCH 07/13] add type hint --- cvxpygen/utils.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cvxpygen/utils.py b/cvxpygen/utils.py index 51c0f9f..3774e4f 100644 --- a/cvxpygen/utils.py +++ b/cvxpygen/utils.py @@ -1258,11 +1258,11 @@ def write_module_prot(f, configuration, parameter_info, variable_info, dual_vari def write_interface( f: TextIOWrapper, - configuration: Configuration, - variable_info: VariableInfo, - dual_variable_info: DualVariableInfo, - parameter_info: ParameterInfo, - solver_interface: SolverInterface, + configuration: "Configuration", + variable_info: "VariableInfo", + dual_variable_info: "DualVariableInfo", + parameter_info: "ParameterInfo", + solver_interface: "SolverInterface", ): write_description(f, 'py', 'Python extension stub file.') interface_content = f""" @@ -1297,11 +1297,11 @@ def replace_setup_data(text): def write_method( f: TextIOWrapper, - configuration: Configuration, - variable_info: VariableInfo, - dual_variable_info: DualVariableInfo, - parameter_info: ParameterInfo, - solver_interface: SolverInterface, + configuration: "Configuration", + variable_info: "VariableInfo", + dual_variable_info: "DualVariableInfo", + parameter_info: "ParameterInfo", + solver_interface: "SolverInterface", ): """ Write function to be registered as custom CVXPY solve method From 730d43e5bb7689c826b3c08a6fbd85a0b0ccdb8d Mon Sep 17 00:00:00 2001 From: Feiyang Date: Fri, 26 Apr 2024 17:47:10 +0100 Subject: [PATCH 08/13] need to operate on valie --- cvxpygen/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvxpygen/utils.py b/cvxpygen/utils.py index 3774e4f..ac5499c 100644 --- a/cvxpygen/utils.py +++ b/cvxpygen/utils.py @@ -1333,7 +1333,7 @@ def write_method( f.write(' cpg_module.set_solver_default_settings()\n') f.write(' for key, value in kwargs.items():\n') f.write(' try:\n') - f.write(' getattr(cpg_module, f\'set_solver_{standard_settings_names.get(key, key)}(value)\')\n') + f.write(' getattr(cpg_module, f\'set_solver_{standard_settings_names.get(key, key)}\')(value)\n') f.write(' except AttributeError:\n') f.write(' raise AttributeError(f\'Solver setting "{key}" not available.\')\n\n') From 95cc56498551de25070e43e3445fe164945c3342 Mon Sep 17 00:00:00 2001 From: Feiyang Date: Fri, 26 Apr 2024 19:39:16 +0100 Subject: [PATCH 09/13] generate interface files consistent with pybind --- cvxpygen/utils.py | 66 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/cvxpygen/utils.py b/cvxpygen/utils.py index ac5499c..5b53928 100644 --- a/cvxpygen/utils.py +++ b/cvxpygen/utils.py @@ -13,7 +13,7 @@ from io import TextIOWrapper import textwrap -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Iterable import numpy as np from datetime import datetime @@ -1265,24 +1265,64 @@ def write_interface( solver_interface: "SolverInterface", ): write_description(f, 'py', 'Python extension stub file.') - interface_content = f""" - def {configuration.prefix}cpg_updated(...): - ... + interface_content = "\n" + + def define_struct( + cls_name: str, + properties: Iterable[str] = [], + methods: Iterable[str] = [], + ): + decl_ = ["\n", f"class {configuration.prefix}{cls_name}:", ""] + for name in properties: + decl_ += [ + " @property", + f" def {name}(self):", + " ...", + "" + ] + for name in methods: + decl_ += [ + f" def {name}(self):" + " ...", + "" + ] + + return "\n".join(decl_) + + interface_content += define_struct("cpg_params", parameter_info.name_to_size_usp.keys()) + interface_content += define_struct("cpg_updated", parameter_info.name_to_size_usp.keys()) + interface_content += define_struct("cpg_prim", variable_info.name_to_init.keys()) - def set_solver_default_settings(...): - ... + if len(dual_variable_info.name_to_init) > 0: + interface_content += define_struct("cpg_dual", dual_variable_info.name_to_init.keys()) + + interface_content += define_struct( + "cpg_info", + properties=[ + "obj_val", + "iter", + "status", + "pri_res", + "dua_res", + "time", + ] + ) + + interface_content += define_struct( + "cpg_result", + ["cpg_prim", "cpg_info"] + (["cpg_dual"] if len(dual_variable_info.name_to_init) > 0 else []) + ) - # f\'set_solver_{{standard_settings_names.get(key, key)}}(value) + interface_content += "\n" - def {configuration.prefix}cpg_params(): - ... + interface_content += "\ndef solve(upd, par):\n ...\n" - def solve(upd, par): - ... - """ + interface_content += "\ndef set_solver_default_settings():\n ...\n" + for name in solver_interface.stgs_names_to_type.keys(): + interface_content += f"\ndef set_solver_{name}():\n ...\n" f.write( - textwrap.dedent(interface_content) + interface_content ) def replace_setup_data(text): From 1f91dfcdbe8dd95851a368c500f5b9c9c608f51a Mon Sep 17 00:00:00 2001 From: Feiyang Date: Fri, 26 Apr 2024 19:54:06 +0100 Subject: [PATCH 10/13] add types to static methods --- cvxpygen/utils.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/cvxpygen/utils.py b/cvxpygen/utils.py index 5b53928..5cb9518 100644 --- a/cvxpygen/utils.py +++ b/cvxpygen/utils.py @@ -1265,14 +1265,14 @@ def write_interface( solver_interface: "SolverInterface", ): write_description(f, 'py', 'Python extension stub file.') - interface_content = "\n" + interface_content = "" def define_struct( cls_name: str, properties: Iterable[str] = [], methods: Iterable[str] = [], ): - decl_ = ["\n", f"class {configuration.prefix}{cls_name}:", ""] + decl_ = ["", f"class {configuration.prefix}{cls_name}:", ""] for name in properties: decl_ += [ " @property", @@ -1287,7 +1287,7 @@ def define_struct( "" ] - return "\n".join(decl_) + return "\n".join(decl_) + "\n" interface_content += define_struct("cpg_params", parameter_info.name_to_size_usp.keys()) interface_content += define_struct("cpg_updated", parameter_info.name_to_size_usp.keys()) @@ -1313,13 +1313,15 @@ def define_struct( ["cpg_prim", "cpg_info"] + (["cpg_dual"] if len(dual_variable_info.name_to_init) > 0 else []) ) - interface_content += "\n" - - interface_content += "\ndef solve(upd, par):\n ...\n" + interface_content += "\ndef solve(arg0: cpg_updated, arg1: cpg_params):\n ...\n" interface_content += "\ndef set_solver_default_settings():\n ...\n" - for name in solver_interface.stgs_names_to_type.keys(): - interface_content += f"\ndef set_solver_{name}():\n ...\n" + for name, type_ in solver_interface.stgs_names_to_type.items(): + pytype = type_.removeprefix("cpg_") + match pytype: + case "const char*": + pytype = "str" + interface_content += f"\ndef set_solver_{name}(arg0: {pytype}):\n ...\n" f.write( interface_content From a0550f10db4fe503dc17ea239c6a5a11103c25f0 Mon Sep 17 00:00:00 2001 From: Feiyang Date: Fri, 26 Apr 2024 20:01:46 +0100 Subject: [PATCH 11/13] add unittest --- tests/test_stub_gen.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/test_stub_gen.py diff --git a/tests/test_stub_gen.py b/tests/test_stub_gen.py new file mode 100644 index 0000000..d8ba0e9 --- /dev/null +++ b/tests/test_stub_gen.py @@ -0,0 +1,17 @@ +import itertools +from unittest import TestCase +import test_E2E_LP +import test_E2E_QP +import test_E2E_SOCP + +class test_stub_gen(TestCase): + def setUp(self) -> None: + self.all_problems = itertools.chain( + test_E2E_LP.name_to_prob.items(), + test_E2E_QP.name_to_prob.items(), + test_E2E_SOCP.name_to_prob.items(), + ) + return super().setUp() + + def test_stub_valid(self): + ... \ No newline at end of file From 43253c1015e2fe1f70a475d3c31f7352344c6d8d Mon Sep 17 00:00:00 2001 From: Feiyang Date: Fri, 26 Apr 2024 20:03:52 +0100 Subject: [PATCH 12/13] add problem type hint --- cvxpygen/cpg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvxpygen/cpg.py b/cvxpygen/cpg.py index 21416a5..55add54 100644 --- a/cvxpygen/cpg.py +++ b/cvxpygen/cpg.py @@ -50,7 +50,7 @@ ) -def generate_code(problem, code_dir='CPG_code', solver=None, solver_opts=None, +def generate_code(problem: cp.Problem, code_dir='CPG_code', solver=None, solver_opts=None, enable_settings=[], unroll=False, prefix='', wrapper=True): """ Generate C code to solve a CVXPY problem From 5385c97e0a40841ed6c0a8c8787ef0bbf40d2c89 Mon Sep 17 00:00:00 2001 From: Feiyang Date: Fri, 26 Apr 2024 20:21:37 +0100 Subject: [PATCH 13/13] test stub syntax --- tests/test_stub_gen.py | 64 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/tests/test_stub_gen.py b/tests/test_stub_gen.py index d8ba0e9..89988d9 100644 --- a/tests/test_stub_gen.py +++ b/tests/test_stub_gen.py @@ -1,9 +1,27 @@ +import ast import itertools +import logging +from io import StringIO +from tempfile import TemporaryDirectory from unittest import TestCase + +import cvxpy as cp import test_E2E_LP import test_E2E_QP import test_E2E_SOCP +from cvxpygen.cpg import ( + get_configuration, + get_constraint_info, + get_dual_variable_info, + get_interface_class, + get_parameter_info, + get_variable_info, + handle_sparsity, +) +from cvxpygen.utils import write_interface + + class test_stub_gen(TestCase): def setUp(self) -> None: self.all_problems = itertools.chain( @@ -11,7 +29,51 @@ def setUp(self) -> None: test_E2E_QP.name_to_prob.items(), test_E2E_SOCP.name_to_prob.items(), ) + self.tempdir = TemporaryDirectory() return super().setUp() + + def tearDown(self) -> None: + self.tempdir.cleanup() + return super().tearDown() + def get_codegen_context(self, problem: cp.Problem): + # problem data + data, solving_chain, inverse_data = problem.get_problem_data( + solver=None, + ) + param_prob = data['param_prob'] + solver_name = solving_chain.solver.name() + interface_class, cvxpy_interface_class = get_interface_class(solver_name) + + # configuration + configuration = get_configuration(self.tempdir, solver_name, False, "") + + # cone problems check + if hasattr(param_prob, 'cone_dims'): + cone_dims = param_prob.cone_dims + interface_class.check_unsupported_cones(cone_dims) + + handle_sparsity(param_prob) + + solver_interface = interface_class(data, param_prob, []) # noqa + variable_info = get_variable_info(problem, inverse_data) + dual_variable_info = get_dual_variable_info(inverse_data, solver_interface, cvxpy_interface_class) + parameter_info = get_parameter_info(param_prob) + constraint_info = get_constraint_info(solver_interface) + return dict( + configuration=configuration, + solver_interface=solver_interface, + variable_info=variable_info, + dual_variable_info=dual_variable_info, + parameter_info=parameter_info, + ) + def test_stub_valid(self): - ... \ No newline at end of file + for name, problem in self.all_problems: + with StringIO() as f: + write_interface(f=f, **self.get_codegen_context(problem)) + try: + ast.parse(f.read()) + except SyntaxError: + logging.exception(f"Generated stub file for problem {name} has incoorect syntax") + raise \ No newline at end of file