Skip to content

Commit ed5e7f8

Browse files
OptHuangOptHuang
committed
Modify action collect info
1 parent 18c81b4 commit ed5e7f8

4 files changed

Lines changed: 285 additions & 83 deletions

File tree

‎.github/actions/collect_info/requirements.txt‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
numpy>=1.17.0
2-
scipy>=1.10.0
3-
pandas>=1.1.0
4-
matplotlib>=3.4.0
5-
h5py>=3.0.0
6-
pypdf>=3.8.0
1+
numpy
2+
scipy
3+
pandas
4+
matplotlib
5+
h5py
6+
pypdf

‎.github/actions/collect_info/s_getInfo.py‎

Lines changed: 24 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,3 @@
1-
"""
2-
s_getInfo.py - Collect problem information from S2MPJ Python problem set
3-
4-
This script is adapted from s2mpj_tools/s_getInfo.py for use in GitHub Actions.
5-
It uses optiprofiler's s2mpj module to load and analyze problems.
6-
"""
7-
81
import numpy as np
92
import pandas as pd
103
from concurrent.futures import ThreadPoolExecutor, TimeoutError
@@ -14,24 +7,21 @@
147
# Add optiprofiler to the system path
158
import os
169
import sys
10+
sys.path.append(os.path.join(os.path.dirname(__file__), 'optiprofiler'))
1711

18-
# Get the repository root directory (three levels up from this script)
12+
# Add problems to the system path
1913
cwd = os.path.dirname(os.path.abspath(__file__))
20-
repo_root = os.path.abspath(os.path.join(cwd, '..', '..', '..'))
21-
22-
# Add optiprofiler to the system path (checked out by GitHub Actions)
23-
sys.path.append(os.path.join(repo_root, 'optiprofiler'))
24-
sys.path.append(os.path.join(repo_root, 'optiprofiler', 'problems'))
25-
from problems.s2mpj.s2mpj_tools import s2mpj_load
14+
# sys.path.append(os.path.join(cwd, 'optiprofiler', 'python', 'optiprofiler', 'problem_libs', 's2mpj'))
15+
from optiprofiler.problem_libs.s2mpj.s2mpj_tools import s2mpj_load
2616

2717
# Set the timeout (seconds) for each problem to be loaded
2818
timeout = 50
2919

30-
# Read problem list from optiprofiler's s2mpj module (this is where the problems actually are)
31-
filename = os.path.join(repo_root, 'optiprofiler', 'problems', 's2mpj', 'src', 'list_of_python_problems')
20+
cwd = os.path.dirname(os.path.abspath(__file__))
21+
filename = os.path.join(cwd, 'optiprofiler', 'problems', 's2mpj', 'src', 'list_of_python_problems')
3222
file = open(filename, 'r')
3323
# Collect the names of the problems from the file
34-
problem_names = [f.strip().replace('.py', '') for f in file.readlines() if f.strip() and not f.startswith('#')]
24+
problem_names = [file.strip().replace('.py', '') for file in file.readlines() if file.strip() and not file.startswith('#')]
3525
file.close()
3626

3727
# Exclude some problems
@@ -54,23 +44,25 @@
5444
timeout_problems = []
5545

5646
# Find problems that are parametric
57-
# The parametric problems list is also in optiprofiler's s2mpj directory
58-
para_file = os.path.join(repo_root, 'optiprofiler', 'problems', 's2mpj', 'list_of_parametric_problems_with_parameters_python.txt')
59-
if os.path.exists(para_file):
60-
with open(para_file, 'r') as file:
61-
para_problem_names = []
62-
problem_argins = []
63-
for line in file:
64-
if line.strip() and not line.startswith('#'):
65-
parts = [x.strip() for x in line.split(',')]
66-
para_problem_names.append(parts[0])
67-
problem_argins.append(parts[1:])
68-
else:
47+
filename = os.path.join(cwd, 'list_of_parametric_problems_with_parameters_python.txt')
48+
# Scan each line, each line only has one problem name, which ends before the first comma
49+
# Give the rest to problem_argins
50+
# In txt file, each line looks like:
51+
# ALJAZZAF,3,100,1000,10000
52+
# or
53+
# TRAINF,{1.5}{2}{11,51,101,01,501,1001,5001,10001}
54+
# ALJAZZAF and TRAINF are problem names
55+
# Then let argins be the rest after the problem name if the problem name is found
56+
with open(filename, 'r') as file:
6957
para_problem_names = []
7058
problem_argins = []
59+
for line in file:
60+
if line.strip() and not line.startswith('#'):
61+
parts = [x.strip() for x in line.split(',')]
62+
para_problem_names.append(parts[0])
63+
problem_argins.append(parts[1:])
7164

72-
# Output path (repository root, where results will be saved)
73-
saving_path = repo_root
65+
saving_path = cwd
7466

7567
# Define the class logger
7668
class Logger(object):
@@ -433,4 +425,4 @@ def has_unknown_values(row):
433425
log_file.close()
434426

435427
sys.stdout = sys.__stdout__ # Reset stdout to default
436-
sys.stderr = sys.__stderr__ # Reset stderr to default
428+
sys.stderr = sys.__stderr__ # Reset stderr to default

‎.github/actions/collect_info/s_get_para.py‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
import os
2+
import re
3+
import shutil
4+
from pathlib import Path
5+
6+
def s_get_para(language='matlab', max_para=500):
7+
"""
8+
Extracts parametric problems from MATLAB files or other specified language files.
9+
"""
10+
11+
# Get the current working directory
12+
cwd = os.getcwd()
13+
14+
# Search and copy the file 'list_of_matlab_problems' to the path './list_of_parametric_problems.txt'
15+
src_file = 'optiprofiler/problems/s2mpj/src/list_of_' + language + '_problems'
16+
dst_file = 'list_of_parametric_problems_' + language + '.txt'
17+
shutil.copy(src_file, dst_file)
18+
19+
# Path to the text file containing the list of names
20+
names_file_path = os.path.join(cwd, dst_file)
21+
22+
# Directory where the language files are located
23+
language_files_folder = os.path.join(cwd, 'optiprofiler/problems/s2mpj/src', language + '_problems')
24+
25+
# Output file
26+
output_file_path = os.path.join(cwd, 'list_of_parametric_problems_with_parameters_' + language + '.txt')
27+
28+
# First Scan the language_files_folder, if the m file contains string '$-PARAMETER', then add the name before .m (or .py) to the txt file names_file_path. Finally sort the names in the names_file_path.
29+
# Open the names file for writing
30+
if language == 'matlab':
31+
suffix = '.m'
32+
elif language == 'python':
33+
suffix = '.py'
34+
else:
35+
raise ValueError("Unsupported language. Supported languages are 'matlab' and 'python'.")
36+
37+
with open(names_file_path, 'w') as output_file:
38+
# Process each name
39+
for file in os.listdir(language_files_folder):
40+
if file.endswith(suffix):
41+
# Construct the full path to the corresponding .m (or .py) file
42+
language_file_path = os.path.join(language_files_folder, file)
43+
44+
# Check if the .m (or .py) file exists
45+
if not os.path.isfile(language_file_path):
46+
print(f"Warning: File not found for '{file}': {language_file_path}")
47+
continue # Skip to the next name if the file doesn't exist
48+
49+
# Read the content of the .m (or .py) file
50+
with open(language_file_path, 'r') as m_file:
51+
content = m_file.readlines()
52+
53+
# Process each line to find the variable assignment and parameters
54+
for line in content:
55+
# If '$-PARAMETER' is in the line, write the name to the output file
56+
if '$-PARAMETER' in line:
57+
if language == 'matlab':
58+
name = file[:-2] # Remove the '.m' (or .py) extension
59+
elif language == 'python':
60+
name = file[:-3]
61+
output_file.write(f'{name}\n')
62+
print(f"Found '$-PARAMETER' in '{file}'. Added '{name}' to the list.")
63+
break # Break after finding the first '$-PARAMETER' line
64+
65+
# Sort the names in the names file
66+
with open(names_file_path, 'r') as f:
67+
names = sorted([line.strip() for line in f if line.strip()])
68+
with open(names_file_path, 'w') as f:
69+
f.write('\n'.join(names))
70+
71+
print(f"First scan complete. Results saved to '{names_file_path}'.")
72+
73+
# Read the list of names from the names file
74+
with open(names_file_path, 'r') as f:
75+
names = [line.strip() for line in f if line.strip()]
76+
77+
# Open the output file for writing
78+
with open(output_file_path, 'w') as output_file:
79+
# Process each name
80+
for name in names:
81+
numbers_set = set()
82+
variable_name = None # Variable to store the VariableName assigned to varargin{1}
83+
84+
# Consider some special cases that we already know
85+
if name == 'NUFFIELD':
86+
output_file.write(f'NUFFIELD,{{5.0}}{{10,20,30,40,100}}\n')
87+
continue
88+
elif name == 'TRAINF':
89+
output_file.write(f'TRAINF,{{1.5}}{{2}}{{11,51,101,201,501}}\n')
90+
continue
91+
elif name == 'QPBAND':
92+
continue
93+
elif name == 'WACHBIEG':
94+
continue
95+
96+
# Construct the full path to the corresponding .m (or .py) file
97+
language_file_path = os.path.join(language_files_folder, f'{name}' + suffix)
98+
99+
# Check if the .m (.py) file exists
100+
if not os.path.isfile(language_file_path):
101+
print(f"Warning: File not found for '{name}': {language_file_path}")
102+
continue # Skip to the next name if the file doesn't exist
103+
104+
# Read the content of the .m (.py) file
105+
with open(language_file_path, 'r') as m_file:
106+
content = m_file.readlines()
107+
108+
# Process each line to find the variable assignment and parameters
109+
for line in content:
110+
# Remove comments and leading/trailing whitespace
111+
if language == 'matlab':
112+
code_line = line.split('%', 1)[0].strip()
113+
pattern = r"v_\(\s*['\"]([^'\"]+)['\"]\s*\)\s*=\s*varargin\{\s*1\s*\};?"
114+
arg_sign = 'varargin{1}'
115+
elif language == 'python':
116+
code_line = line.split('#', 1)[0].strip()
117+
pattern = r"v_\[\s*['\"]([^'\"]+)['\"]\s*\]\s*=\s*(?:int|float|str)?\s*\(?\s*args\[0\]\s*\)?"
118+
arg_sign = 'args[0]'
119+
120+
# Check if arg_sign is in the line
121+
if arg_sign in code_line:
122+
# Use regex to find v_('VariableName') = varargin{1};
123+
match = re.search(pattern, code_line)
124+
if match:
125+
variable_name = match.group(1)
126+
# Since variable_name may include special characters, escape them for regex
127+
variable_name_regex = re.escape(variable_name)
128+
break # Break after finding the first variable assignment
129+
130+
# If variable_name was found, search for $-PARAMETER lines associated with it
131+
if variable_name:
132+
for line in content:
133+
if '$-PARAMETER' in line:
134+
param_line = line.strip()
135+
# Updated regex pattern
136+
if language == 'matlab':
137+
pattern = rf"%\s*\w+\s+({variable_name_regex}(?:[+\-*/]\w+)*)\s+([-+]?\d*\.?\d+)\s*\$-PARAMETER"
138+
elif language == 'python':
139+
pattern = rf"#\s*\w+\s+({variable_name_regex}(?:[+\-*/]\w+)*)\s+([-+]?\d*\.?\d+)\s*\$-PARAMETER"
140+
param_match = re.match(pattern, param_line)
141+
if param_match:
142+
var_in_line = param_match.group(1)
143+
number = param_match.group(2)
144+
# If the float(number) is a number smaller than max_para, and not the case like 5.0 (it should be 5), add it to the set
145+
if float(number) <= max_para:
146+
if '.' not in number:
147+
numbers_set.add(number)
148+
else:
149+
# Add the number and a 'Check' to the problem_name
150+
numbers_set.add(number)
151+
name += 'Check'
152+
153+
154+
# If numbers were found, sort them and write to the output file
155+
if numbers_set:
156+
# Convert numbers to floats for sorting
157+
numbers_list = sorted(numbers_set, key=float)
158+
# Join the numbers with commas
159+
numbers_str = ','.join(numbers_list)
160+
# Write the line to the output file
161+
output_file.write(f'{name},{numbers_str}\n')
162+
163+
# Delete the file 'list_of_parametric_problems.txt'
164+
os.remove(names_file_path)
165+
print(f"Second scan complete. Deleted '{names_file_path}'.")
166+
167+
print(f"Extraction complete. Results saved to '{output_file_path}'.")
168+
print("You need to check all the problems with 'Check' in the name.")
169+
170+
171+
if __name__ == "__main__":
172+
import sys
173+
language_arg = 'matlab' # 默认值
174+
max_para_arg = 500
175+
if len(sys.argv) > 1:
176+
language_arg = sys.argv[1]
177+
if len(sys.argv) > 2:
178+
try:
179+
max_para_arg = int(sys.argv[2])
180+
except ValueError:
181+
print("Invalid max_para value. Using default 500.")
182+
s_get_para(language_arg, max_para_arg)

0 commit comments

Comments
 (0)