diff --git a/test/partitioner/external_tools/3rd-party/CMakeLists.txt b/test/partitioner/external_tools/3rd-party/CMakeLists.txt new file mode 100644 index 00000000..d240400d --- /dev/null +++ b/test/partitioner/external_tools/3rd-party/CMakeLists.txt @@ -0,0 +1,25 @@ +set(3RD_PARTY_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + +set(KAHIP_LIB "${CMAKE_INSTALL_PREFIX}/lib/libkahip.a") + +add_custom_command( + OUTPUT ${KAHIP_LIB} + COMMAND ${CMAKE_COMMAND} -E echo "Building KaHIP." + COMMAND ${CMAKE_COMMAND} -E tar xvzf ${3RD_PARTY_DIR}/KaHIP/KaHIP-3.16.tar.gz + COMMAND cd ${CMAKE_CURRENT_BINARY_DIR}/KaHIP-3.16 && ./compile_withcmake.sh + COMMAND ${CMAKE_COMMAND} -E echo "Installing KaHIP." + COMMAND ${CMAKE_COMMAND} -E rm -rf ${3RD_PARTY_DIR}/KaHIP/deploy + COMMAND mv ${CMAKE_CURRENT_BINARY_DIR}/KaHIP-3.16/deploy ${3RD_PARTY_DIR}/KaHIP/ + + COMMAND ${CMAKE_COMMAND} -E echo "Installing SBG Library." + COMMAND ${CMAKE_COMMAND} -E rm -rf ${3RD_PARTY_DIR}/sbg + COMMAND mkdir -p ${3RD_PARTY_DIR}/sbg + COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/../../../../install/lib/* ${3RD_PARTY_DIR}/sbg +) + +add_custom_target( + 3rd-party-libs + DEPENDS ${KAHIP_LIB} +) + +add_dependencies(graph-partitioner 3rd-party-libs) diff --git a/test/partitioner/external_tools/3rd-party/KaHIP/KaHIP-3.16.tar.gz b/test/partitioner/external_tools/3rd-party/KaHIP/KaHIP-3.16.tar.gz new file mode 100644 index 00000000..32090fcd Binary files /dev/null and b/test/partitioner/external_tools/3rd-party/KaHIP/KaHIP-3.16.tar.gz differ diff --git a/test/partitioner/external_tools/CMakeLists.txt b/test/partitioner/external_tools/CMakeLists.txt new file mode 100644 index 00000000..fd8d5025 --- /dev/null +++ b/test/partitioner/external_tools/CMakeLists.txt @@ -0,0 +1,65 @@ +cmake_minimum_required(VERSION 3.16) + +project(ExtPartitioners) + +set(CMAKE_INSTALL_PREFIX ${PROJECT_SOURCE_DIR}/install) + +# Set default build type to Debug if not specified +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Debug CACHE STRING "Build type" FORCE) +endif() + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# Add libraries +add_library(graph-partitioner STATIC) + +set_target_properties(graph-partitioner PROPERTIES DEBUG_POSTFIX "d") + +target_include_directories(graph-partitioner PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) + +target_include_directories(graph-partitioner PRIVATE 3rd-party/KaHIP/deploy ${PROJECT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/../../../) + +target_sources( + graph-partitioner + PRIVATE + graph_partitioner.cpp +) + + +# Source files. +add_subdirectory(3rd-party) + +# Add executable +add_executable( + graph_partitioner +) + +target_sources( + graph_partitioner + PRIVATE + main.cpp +) + +target_link_directories(graph_partitioner PRIVATE 3rd-party/KaHIP/deploy 3rd-party/sbg) + +target_link_libraries( + graph_partitioner + PRIVATE + graph-partitioner + metis + scotch + scotcherr + kahip.a + sbg-part-lib + sbg-util-lib + sbg-dev +) + +# install +install( + TARGETS graph_partitioner + RUNTIME DESTINATION bin + COMPONENT graph_partitioner +) \ No newline at end of file diff --git a/test/partitioner/external_tools/experiments/adv_dif_reac/adv_dif_reac.mo b/test/partitioner/external_tools/experiments/adv_dif_reac/adv_dif_reac.mo new file mode 100644 index 00000000..1141f63b --- /dev/null +++ b/test/partitioner/external_tools/experiments/adv_dif_reac/adv_dif_reac.mo @@ -0,0 +1,38 @@ +model adv_dif_reac + constant Integer N=1000; + parameter Real a=1; + parameter Real d=1e-4; + parameter Real r=10; + parameter Real L=10; + parameter Real dx=L/N; + Real u[N]; + +initial algorithm + for i in 1:N/3 loop + u[i]:=1; + end for; + +equation + der(u[1])=-a*(u[1]-1)/dx+d*(-2*u[1]+1)/(dx^2)+r*(u[1]^2)*(1-u[1]); + for i in 2:N loop + der(u[i])=-a*(u[i]-u[i-1])/dx+ d*(-2*u[i]+u[i-1])/(dx^2)+ r*(u[i]^2)*(1-u[i]); + end for; + annotation( + + experiment( + MMO_Description=" Advection", + MMO_Solver=LIQSS2, + MMO_Parallel=true, + MMO_PartitionMethod=Metis, + MMO_LPS=2, + MMO_DT_Min= 2, + MMO_Output={u[N]}, + Jacobian=Dense, + MMO_BDF_PDepth=1, + MMO_BDF_Max_Step= 0, + StartTime= 0.0, + StopTime=10, + Tolerance={1e-3}, + AbsTolerance={1e-3} + )); +end adv_dif_reac; diff --git a/test/partitioner/external_tools/experiments/airconds/airconds.mo b/test/partitioner/external_tools/experiments/airconds/airconds.mo new file mode 100644 index 00000000..6aaec3d0 --- /dev/null +++ b/test/partitioner/external_tools/experiments/airconds/airconds.mo @@ -0,0 +1,122 @@ +model airconds + import math; + constant Integer N = 10000; + parameter Real CAP[N], RES[N], POT[N], THA = 32,pmax=0; + Real th[N]; + discrete Real on[N]; + discrete Real tref[N]; + discrete Real nextSample[N]; + discrete Real noise[N]; + discrete Real nextTref[N]; + + initial algorithm + for i in 1:N loop + th[i] := rand(4)+ 18; + CAP[i] := rand(100)+ 550; + RES[i] := rand(0.4)+ 1.8; + POT[i] := rand(2)+ 13; + pmax:=pmax+POT[i]; + nextSample[i] := 1; + noise[i] := rand(2)-1; + tref[i] := 20; + nextTref[i] := 1000; + end for; + for i in 1:N loop + if th[i] - tref[i] - 0.5 > 0 then + on[i] := 1; + end if; + end for; + + equation + for i in 1:N loop + der(th[i]) = (THA/RES[i]-POT[i]*on[i]-th[i]/RES[i]+noise[i]/RES[i])/CAP[i]; + end for; + + algorithm + for i in 1:N loop + when th[i] - tref[i] + on[i] - 0.5 > 0 then + on[i] := 1; + elsewhen th[i] - tref[i] + on[i] - 0.5 < 0 then + on[i] := 0; + end when; + end for; + for i in 1:N loop + when time > nextTref[i] then + if (nextTref[i] == 1000) then + tref[i] := 20.5; + else + tref[i] := 20; + end if; + nextTref[i] := 2000; + end when; + end for; + for i in 1:N loop + when time > nextSample[i] then + nextSample[i] := nextSample[i]+1; + noise[i] := 2*abs(sin(i*time))-1; + end when; + end for; + annotation( + experiment( + MMO_Description = "Power consumption in a large population of air conditioners.", + MMO_Solver = QSS2, + MMO_Period = {3000/5000}, + MMO_Parallel = true, + MMO_PartitionMethod = Scotch, + MMO_LPS = 4, + MMO_DT_Synch = SD_DT_Fixed, + MMO_DT_Min = 3000, + MMO_Output = {th[1]}, + MMO_OutputType = CI_Sampled, + Jacobian = Dense, + MMO_BDF_PDepth = 1, + MMO_BDF_Max_Step = 0, + StartTime = 0, + StopTime = 3000, + Tolerance = {1e-3}, + AbsTolerance = {1e-3} + ) +); + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end airconds; diff --git a/test/partitioner/external_tools/experiments/airconds_advection_experiments.py b/test/partitioner/external_tools/experiments/airconds_advection_experiments.py new file mode 100644 index 00000000..40e34a93 --- /dev/null +++ b/test/partitioner/external_tools/experiments/airconds_advection_experiments.py @@ -0,0 +1,80 @@ +# Airconds and Advection examples to get partition results using classic partitioners. + +# General configuration to access qss-sover methods (the solver needs to be installed first). +# libconf is needded as an external dep, run: `pip install libconf` first.` + +import argparse +import os +import sys + +MMOC_SRC = os.environ['MMOC_SRC'] +MMOC_OUTPUT = os.environ['MMOC_OUTPUT'] +sys.path.append(MMOC_SRC+'/python') +sys.path.append(MMOC_SRC+'/python/qss_solver') + +import qss_solver +import qss_solver.results as solver_results +import qss_solver.model as solver_model +import qss_solver.simulate as solver_sim + +def run_experiments(path, model_name): + # Model constants size. + model_size = [1000, 10000, 100000, 1000000, 10000000] + + # Partitioners + model_partitioners = ['Metis', 'Scotch'] + print(path) + + model_full_path = path+'/'+model_name+'.mo' + + for s in model_size: + for p in model_partitioners: + # The parameter should be the full path name if not stored in the solver default folders. + model_constant = solver_model.constants(model_full_path) + model_constant['N'] = s + + solver_model.set_constants(model_full_path, model_constant) + model_annotations = solver_model.annotations(model_full_path) + model_annotations['MMO_PartitionMethod'] = p + # Store the number of LPS (partitions) we are using. + lps = model_annotations['MMO_LPS'] + solver_model.set_annotations(model_full_path, model_annotations) + + # Compile the model to generate the excecutabe ini file. + solver_sim.compile_model(model_full_path) + + # Now update the ini file to set the 'partitionOnly' flag to avoid running the simulation. + model_config = solver_model.config(model_name) + model_config['partitionOnly'] = 1 + solver_model.set_config(model_name, model_config) + + # Now go ahead and execute the model. + solver_sim.execute_model(model_full_path) + + # Finally, get the partitioner stats into a dict. + # Full size of the advection model. + model_full_size = s + if model_name == 'airconds': + # Update full size for airconds model. + model_full_size = s*4 + partitioner_log_file = model_name+'-'+str(model_full_size)+'-'+lps+'-partition-stats.log' + partition_log = solver_results.simulation_log(MMOC_OUTPUT+'/'+model_name+'/'+partitioner_log_file) + print('Partition method: '+ p) + print('Model size: ' + str(s)) + print('Results') + print(partition_log) + + +# Example usage: python3 ./airconds_experimets.py /home/joaquin/work/sbg-partitioner/src/external_tools/experiments/airconds airconds + +def main(): + parser = argparse.ArgumentParser(description='Run experiments for classic partitioners.') + parser.add_argument('path', type=str, help='Full path to the folder where the airconds model .mo file is located.') + parser.add_argument('model_name', type=str, help='Model name whitout .mo extension. It should be one of \'advection\' or \'airconds\'') + + args = parser.parse_args() + + run_experiments(args.path, args.model_name) + +if __name__ == '__main__': + main() diff --git a/test/partitioner/external_tools/experiments/airconds_cont/airconds_cont.mo b/test/partitioner/external_tools/experiments/airconds_cont/airconds_cont.mo new file mode 100644 index 00000000..ce899f8a --- /dev/null +++ b/test/partitioner/external_tools/experiments/airconds_cont/airconds_cont.mo @@ -0,0 +1,170 @@ +model airconds_cont + function getSection + input Integer x; + output Real y; + external "C" y = getSection(x) annotation( + Include="#include \"sections.c\""); + end getSection; + + import math; + constant Integer N = 10000000; + constant Integer SECTIONS = 4; + parameter Real CAP[N], RES[N], POT[N], THA = 32,pmax=0,Kp=1,Ki=1,tref=20; + parameter Integer sections[N]; + parameter Integer sections_init[SECTIONS]; + parameter Integer sections_end[SECTIONS]; + Real th[N]; + Real ierr; + discrete Real ptotals; + discrete Real partTotal[SECTIONS]; + discrete Real sendPartTotal[SECTIONS]; + Real ptotal; + + discrete Real on[N]; + discrete Real dtref,pref(start=0.5); + discrete Real nextSample(start=1); + discrete Real partSample[SECTIONS](each start=1); + discrete Real noise[N]; + discrete Real update[SECTIONS]; + discrete Real sampleNoise[N]; + discrete Real inputs[N]; + + initial algorithm + for i in 1:N loop + th[i] := random(4)+ 18; + CAP[i] := random(100)+ 550; + RES[i] := random(0.4)+ 1.8; + POT[i] := random(0.2)+ 13; + pmax:=pmax+POT[i]; + noise[i] := 2*sin(i)^2; + sampleNoise[i] := random(2); + sections[i] := getSection(i); + end for; + + for i in 1:SECTIONS loop + sections_init[i] := (i-1)*N/SECTIONS; + sections_end[i] := i*N/SECTIONS; + end for; + + + for i in 1:N loop + if th[i] - tref - 0.5> 0 then + on[i] := 1; + ptotal := ptotal + POT[i]; + end if; + end for; + + equation + for i in 1:N loop + der(th[i]) = (THA/RES[i]-POT[i]*on[i]-th[i]/RES[i]+noise[i]/RES[i])/CAP[i]; + end for; + der(ierr) = pref-ptotals/pmax; + der(ptotal) = 0; + + algorithm + for i in 1:SECTIONS loop + when time > partSample[i] then + partSample[i] := partSample[i]+1; + update[i] := 1; + sendPartTotal[i] := partTotal[i]; + partTotal[i] := 0; + end when annotation(MMO_Event_Id="Ev_1",MMO_HH={(Ev_6[_d2],[1:SECTIONS, sections_init[i]:sections_end[i]])}); + end for; + + for i in 1:SECTIONS loop + when update[i] > 0.5 then + reinit(ptotal, ptotal + sendPartTotal[i]); + update[i] := 0; + end when; + end for; + + when time > 1000 then + pref := 0.4; + end when; + when time > 2000 then + pref := 0.5; + end when; + + when time > nextSample then + nextSample := nextSample+1; + ptotals := ptotal; + dtref := Kp*(ptotals/pmax-pref)-Ki*ierr; + end when; + + for i in 1:N loop + when th[i] - tref -dtref + on[i] - 0.5 > 0 then + on[i] := 1; + partTotal[sections[i]]:= partTotal[sections[i]] + POT[i]; + elsewhen th[i] - tref -dtref + on[i] - 0.5 < 0 then + on[i] := 0; + partTotal[sections[i]]:= partTotal[sections[i]] - POT[i]; + end when annotation(MMO_Event_Id="Ev_6", + MMO_LHS_DSC={partTotal[sections[i]]}, + MMO_HH={Ev_1[sections[i]]} ); + + end for; + + for i in 1:N loop + when time > sampleNoise[i] then + sampleNoise[i] := sampleNoise[i] + 1; + inputs[i] := inputs[i] + 1; + noise[i] := 2*sin(i*inputs[i])^2; + end when; + end for; + annotation( + experiment( + MMO_Description = "Control of the power consumption of a large populaion of air conditioners.", + MMO_Solver = QSS3, + MMO_Period = {3000/5000}, + MMO_Parallel = true, + MMO_PartitionMethod = Metis, + MMO_LPS = 4, + MMO_DT_Synch = SD_DT_Fixed, + MMO_DT_Min = 10, + MMO_Output = {ptotals}, + Jacobian = Dense, + MMO_BDF_PDepth = 1, + MMO_BDF_Max_Step = 0, + StartTime = 0, + StopTime = 3000, + Tolerance = {1e-4}, + AbsTolerance = {1e-4} + ) +); + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end airconds_cont; diff --git a/test/partitioner/external_tools/experiments/airconds_cont_experiments.py b/test/partitioner/external_tools/experiments/airconds_cont_experiments.py new file mode 100644 index 00000000..74662dd2 --- /dev/null +++ b/test/partitioner/external_tools/experiments/airconds_cont_experiments.py @@ -0,0 +1,98 @@ +# Airconds Controller example to get partition results using classic partitioners. + +# General configuration to access qss-sover methods (the solver needs to be installed first). +# libconf is needded as an external dep, run: `pip install libconf` first.` + +import argparse +import os +import shutil +import sys + +MMOC_SRC = os.environ['MMOC_SRC'] +MMOC_OUTPUT = os.environ['MMOC_OUTPUT'] +MMOC_BUILD = os.environ['MMOC_BUILD'] +sys.path.append(MMOC_SRC+'/python') +sys.path.append(MMOC_SRC+'/python/qss_solver') + +import qss_solver +import qss_solver.results as solver_results +import qss_solver.model as solver_model +import qss_solver.simulate as solver_sim + + +def generate_section_file(path, size, sections): + airs_per_section = size / sections + sections_file_code = 'int getSection(int i) {'+'\n int ret = ((i-1)/'+str(airs_per_section)+')+1;\nreturn (ret);\n}' + sections_file = path + '/sections.c' + # Write the modified content back to the file + with open(sections_file, 'w') as file: + file.write(sections_file_code) + +def run_experiments(path, model_name): + # Model constants size. + model_size = [1000, 10000, 100000, 1000000, 10000000] + + # Partitioners + model_partitioners = ['Metis'] + + model_full_path = path+'/'+model_name+'.mo' + + # Just in case we may want to change the number of sections. + sections = [4] + + for s in model_size: + for section in sections: + generate_section_file(path, s, int(section)) + for p in model_partitioners: + # The parameter should be the full path name if not stored in the solver default folders. + model_constant = solver_model.constants(model_full_path) + model_constant['N'] = s + solver_model.set_constants(model_full_path, model_constant) + model_annotations = solver_model.annotations(model_full_path) + model_annotations['MMO_PartitionMethod'] = p + # Store the number of LPS (partitions) we are using. + lps = model_annotations['MMO_LPS'] + solver_model.set_annotations(model_full_path, model_annotations) + + # Copy the generated sections C file to build folder. + sections_file = path + '/sections.c' + build_airconds_cont_folder = MMOC_BUILD + '/'+model_name + print(build_airconds_cont_folder) + if not os.path.exists(build_airconds_cont_folder): + os.makedirs(build_airconds_cont_folder) + shutil.copy(sections_file, build_airconds_cont_folder) + + # Compile the model to generate the excecutabe ini file. + solver_sim.compile_model(model_full_path) + + # Now update the ini file to set the 'partitionOnly' flag to avoid running the simulation. + model_config = solver_model.config(model_name) + model_config['partitionOnly'] = 1 + solver_model.set_config(model_name, model_config) + + # Now go ahead and execute the model. + solver_sim.execute_model(model_full_path) + + # Finally, get the partitioner stats into a dict. + model_full_size = 3 * s + 5 + 2 * int(section) + partitioner_log_file = model_name+'-'+str(model_full_size)+'-'+lps+'-partition-stats.log' + partition_log = solver_results.simulation_log(MMOC_OUTPUT+'/'+model_name+'/'+partitioner_log_file) + print('Partition method: '+ p) + print('Model size: ' + str(s)) + print('Results') + print(partition_log) + + +# Example usage: python3 ./airconds_experimets.py /home/joaquin/work/sbg-partitioner/src/external_tools/experiments/airconds_cont airconds_cont + +def main(): + parser = argparse.ArgumentParser(description='Run experiments for classic partitioners.') + parser.add_argument('path', type=str, help='Full path to the folder where the airconds model .mo file is located.') + parser.add_argument('model_name', type=str, help='Model name whitout .mo extension.') + + args = parser.parse_args() + + run_experiments(args.path, args.model_name) + +if __name__ == '__main__': + main() diff --git a/test/partitioner/external_tools/experiments/spl_neurons/spl_neurons.mo b/test/partitioner/external_tools/experiments/spl_neurons/spl_neurons.mo new file mode 100644 index 00000000..7968aa1a --- /dev/null +++ b/test/partitioner/external_tools/experiments/spl_neurons/spl_neurons.mo @@ -0,0 +1,135 @@ +model spl_neurons + import math; + constant Integer N = 10000; + constant Integer M = 100; + constant Integer OFFSET = 100; + constant Integer CONN = 10; + Real v[N](each start=-60),gex[N],ginh[N]; + parameter Real tau=20, vrest=-60, vthres=-50,Trefrac=5,Eex=0,Einh=-80,tauex=5,tauinh=10,dgex=0.4,dgin=1.6; + discrete Real active[N](each start=1),tfire[N](each start=10000); + discrete Real nextev[M](each start=random(1)); + parameter Real Period=10,lastTime=100; + parameter Real ex[N]; + discrete Real exReinit[N]; + discrete Real inhReinit[N]; + discrete Real nextGlobalTime[M]; + discrete Real inputs[M]; + initial algorithm + + /*for i in 1:600 loop + ex[i]:=1; + end for; + for i in 601:800 loop + ex[i]:=0; + end for; + + for i in 801:1500 loop + ex[i]:=1; + end for;*/ + + equation + for i in 1:N loop + der(v[i])=active[i]*((vrest-v[i])+gex[i]*(Eex-v[i])+ginh[i]*(Einh-v[i]))/tau; + der(gex[i])=-gex[i]/tauex; + der(ginh[i])=-ginh[i]/tauinh; + end for; + + algorithm + for i in 1:N loop + when exReinit[i] > 0.5 then + reinit(gex[i],gex[i]+dgex); + exReinit[i]:=0; + end when; + end for; + + for i in 1:N loop + when inhReinit[i] > 0.5 then + reinit(ginh[i],ginh[i]+dgin); + inhReinit[i]:=0; + end when; + end for; + + for i in 1:N loop + when time>tfire[i]+Trefrac then + active[i]:=1; + end when; + end for; + + for i in 1:M loop + when time>nextev[i] then + exReinit[i+100] := 1; + inputs[i] := inputs[i] + 1; + nextev[i]:=time+10*sin(i*inputs[i])^2; + end when; + end for; + + for i in 1:N-OFFSET-CONN loop + when v[i]*ex[i]>vthres then + reinit(v[i],vrest); + reinit(ginh[i],0); + reinit(gex[i],0); + exReinit[i+100] := 1; + exReinit[i+101] := 1; + exReinit[i+102] := 1; + exReinit[i+103] := 1; + exReinit[i+104] := 1; + exReinit[i+105] := 1; + exReinit[i+106] := 1; + exReinit[i+107] := 1; + exReinit[i+108] := 1; + exReinit[i+109] := 1; + active[i]:=0; + tfire[i]:=time; + end when; + end for; + + for i in 1:N-OFFSET-CONN loop + when v[i]*(1-ex[i])>vthres then + reinit(v[i],vrest); + reinit(ginh[i],0); + reinit(gex[i],0); + inhReinit[i+100] := 1; + inhReinit[i+101] := 1; + inhReinit[i+102] := 1; + inhReinit[i+103] := 1; + inhReinit[i+104] := 1; + inhReinit[i+105] := 1; + inhReinit[i+106] := 1; + inhReinit[i+107] := 1; + inhReinit[i+108] := 1; + inhReinit[i+109] := 1; + active[i]:=0; + tfire[i]:=time; + end when; + end for; + annotation( + experiment( + MMO_Description = "", + MMO_Solver = QSS2, + MMO_SymDiff = false, + MMO_Period = {300/5000}, + MMO_Parallel = true, + MMO_PartitionMethod = Scotch, + MMO_LPS = 2, + MMO_DT_Min = 0.5, + MMO_Output = {v[1:N/20:N]}, + MMO_OutputType = CI_Sampled, + Jacobian = Dense, + MMO_BDF_PDepth = 1, + MMO_BDF_Max_Step = 0, + StartTime = 0.0, + StopTime = 300, + Tolerance = {1e-3}, + AbsTolerance = {1e-3} + ) +); + + + + + + + + + +end spl_neurons; diff --git a/test/partitioner/external_tools/experiments/spl_neurons_experiments.py b/test/partitioner/external_tools/experiments/spl_neurons_experiments.py new file mode 100644 index 00000000..0301cc93 --- /dev/null +++ b/test/partitioner/external_tools/experiments/spl_neurons_experiments.py @@ -0,0 +1,139 @@ +# Spiking Neurons example to get partition results using classic partitioners. + +# General configuration to access qss-sover methods (the solver needs to be installed first). +# libconf is needded as an external dep, run: `pip install libconf` first.` + +import argparse +import os +import re +import sys + +MMOC_SRC = os.environ['MMOC_SRC'] +MMOC_OUTPUT = os.environ['MMOC_OUTPUT'] +MMOC_BUILD = os.environ['MMOC_BUILD'] +sys.path.append(MMOC_SRC+'/python') +sys.path.append(MMOC_SRC+'/python/qss_solver') + +import qss_solver +import qss_solver.results as solver_results +import qss_solver.model as solver_model +import qss_solver.simulate as solver_sim + + +def update_generated_c_code(file_path): + """ + Fix the current bug in the solver that creates wrong expressions for event dependencies in graph merge. + + Args: + file_path (str): The path to the file to be modified. + """ + # Read the file content + with open(file_path, 'r') as file: + lines = file.readlines() + + # Define the lines to be replaced + old_line1 = 'modelData->HZ[_idx_event4(_d1)][events[_idx_event_4(_d1)]++ = _idx_event_1(_d1-100);' + new_line1 = 'modelData->HZ[_idx_event4(_d1)][events[_idx_event_4(_d1)]++ = _idx_event_1(_d1+100);' + + # Define the patterns to match the wrong expressions + pattern = r'modelData->HZ\[_idx_event5\(_d1\)]\[events\[_idx_event_5\(_d1\)]\+\+ = _idx_event_1\(_d1-([0-9]+)\);' + event_6_pattern = r'modelData->HZ\[_idx_event6\(_d1\)]\[events\[_idx_event_6\(_d1\)]\+\+ = _idx_event_2\(_d1-([0-9]+)\);' + + # Replace the old lines with the new lines + modified_lines = [] + for line in lines: + if line.strip() == old_line1: + modified_lines.append(new_line1 + '\n') + elif re.match(pattern, line): + match = re.search(pattern, line) + c = int(match.group(1)) + if 101 <= c <= 109: + new_line = re.sub(r'-([0-9]+)', f'+{c}', line) + modified_lines.append(new_line) + else: + modified_lines.append(line) + elif re.match(event_6_pattern, line): + match = re.search(event_6_pattern, line) + c = int(match.group(1)) + if 101 <= c <= 109: + new_line = re.sub(r'-([0-9]+)', f'+{c}', line) + modified_lines.append(new_line) + else: + modified_lines.append(line) + else: + modified_lines.append(line) + + # Write the modified content back to the file + with open(file_path, 'w') as file: + file.writelines(modified_lines) + +def run_experiments(path, model_name): + + # Model constants size. + model_size = [10000] + + # We won't iterate here, if we need to we can do it. + model_random_inputs = [100] + + # Same as the previous model constant. + model_connection_offset = [100] + + # Partitioners + model_partitioners = ['Metis', 'Scotch'] + print(path) + + model_full_path = path+'/'+model_name+'.mo' + + for s in model_size: + for p in model_partitioners: + # The parameter should be the full path name if not stored in the solver default folders. + model_constant = solver_model.constants(model_full_path) + model_constant['N'] = s + model_constant['M'] = model_random_inputs[0] + model_constant['OFFSET'] = model_connection_offset[0] + + solver_model.set_constants(model_full_path, model_constant) + model_annotations = solver_model.annotations(model_full_path) + model_annotations['MMO_PartitionMethod'] = p + # Store the number of LPS (partitions) we are using. + lps = model_annotations['MMO_LPS'] + solver_model.set_annotations(model_full_path, model_annotations) + + # Fix current code generation bug before running the model. + model_c_code = MMOC_BUILD + '/' + model_name + '/' + model_name + '.c' + update_generated_c_code(model_c_code) + + # Compile the model to generate the excecutabe ini file. + solver_sim.compile_model(model_full_path) + + # Now update the ini file to set the 'partitionOnly' flag to avoid running the simulation. + model_config = solver_model.config(model_name) + model_config['partitionOnly'] = 1 + solver_model.set_config(model_name, model_config) + + # Now go ahead and execute the model. + solver_sim.execute_model(model_full_path) + + # Finally, get the partitioner stats into a dict. + model_full_size = 6 * s + model_constant['M'] + 2 * (s - model_constant['OFFSET'] - model_constant['CONN']) + partitioner_log_file = model_name+'-'+str(model_full_size)+'-'+lps+'-partition-stats.log' + partition_log = solver_results.simulation_log(MMOC_OUTPUT+'/'+model_name+'/'+partitioner_log_file) + print('Partition method: '+ p) + print('Model size: ' + str(s)) + print('Results') + print(partition_log) + + +# Example usage: python3 ./airconds_experimets.py /home/joaquin/work/sbg-partitioner/src/external_tools/experiments/airconds airconds + +def main(): + parser = argparse.ArgumentParser(description='Run experiments for classic partitioners.') + parser.add_argument('path', type=str, help='Full path to the folder where the airconds model .mo file is located.') + parser.add_argument('model_name', type=str, help='Model name whitout .mo extension.') + + args = parser.parse_args() + + run_experiments(args.path, args.model_name) + +if __name__ == '__main__': + main() diff --git a/test/partitioner/external_tools/graph_partitioner.cpp b/test/partitioner/external_tools/graph_partitioner.cpp new file mode 100644 index 00000000..9ca5cf69 --- /dev/null +++ b/test/partitioner/external_tools/graph_partitioner.cpp @@ -0,0 +1,367 @@ +/***************************************************************************** + + This file is part of SBG Partitioner. + + SBG Partitioner is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + SBG Partitioner is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with SBG Partitioner. If not, see . + + *****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "graph_partitioner.hpp" +#include +#include + +constexpr const char *VALID_PARTITION_METHODS = "{ Scotch, Metis, HMetis, Kahip}"; + +// Initialize the map outside the function +static const std::unordered_map PARTITION_METHOD_MAP = {{"Metis", PartitionMethod::Metis}, + {"HMetis", PartitionMethod::HMetis}, + {"Scotch", PartitionMethod::Scotch}, + {"Kahip", PartitionMethod::Kahip}}; + +GraphPartitioner::GraphPartitioner(const std::string &name) : _name(name) { generateInputGraph(); } + +std::string GraphPartitioner::validPartitionMethodsStr() +{ + std::ostringstream valid_methods; + valid_methods << "{"; + for (const auto &pair : PARTITION_METHOD_MAP) { + if (&pair != &*PARTITION_METHOD_MAP.begin()) { + valid_methods << ","; + } + valid_methods << pair.first; + } + valid_methods << "}"; + return valid_methods.str(); +} + +Partition GraphPartitioner::createPartition(const std::string &partition_method_name, unsigned int partitions) +{ + Partition partition; + PartitionMethod partition_method = partitionMethod(partition_method_name); + + if (partition_method == PartitionMethod::Unknown) { + std::cerr << "Unknown partition method, valid values are: " << validPartitionMethodsStr() << std::endl; + return partition; + } + + partition.resize(_nbr_vtxs); + + _nbr_parts = partitions; + + partition.resize(_nbr_vtxs); + + auto start = std::chrono::high_resolution_clock::now(); + + if (_nbr_vtxs > _nbr_parts) { + switch (partition_method) { + case PartitionMethod::Metis: + partitionUsingMetis(partition); + break; + case PartitionMethod::HMetis: + partitionUsingHMetis(partition); + break; + case PartitionMethod::Scotch: + partitionUsingScotch(partition); + break; + case PartitionMethod::Kahip: + partitionUsingKaHip(partition); + break; + default: + break; + } + } + + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration = end - start; + + std::cout << "Partition Time: " << duration.count() << " seconds." << std::endl; + + savePartitionToFile(partition, partition_method_name); + return partition; +} + +bool GraphPartitioner::endsWithJson() { return _name.size() >= 5 && _name.compare(_name.size() - 5, 5, ".json") == 0; } + +void GraphPartitioner::generateInputGraph() +{ + if (endsWithJson()) { + readGraphFromJson(); + } else { + readGraph(); + } +} + +void GraphPartitioner::readGraphFromJson() +{ + auto sb_graph = sbg_partitioner::build_sb_graph(_name); + readGraphFromSBG(sb_graph); +} + +void GraphPartitioner::readGraph() +{ + const std::string graph_file_name = _name + ".graph"; + const std::string graph_size_file_name = graph_file_name + ".size"; + + std::ifstream graph_size_file(graph_size_file_name, std::ios::binary); + if (!graph_size_file) { + std::cerr << "Error opening file: " << graph_size_file_name << std::endl; + return; + } + + std::ifstream graph_file(graph_file_name, std::ios::binary); + if (!graph_file) { + std::cerr << "Error opening file: " << graph_file_name << std::endl; + return; + } + + graph_size_file.read(reinterpret_cast(&_nbr_vtxs), sizeof(_nbr_vtxs)); + graph_size_file.read(reinterpret_cast(&_edges), sizeof(_edges)); + + graph_size_file.close(); + + _xadj.resize(_nbr_vtxs + 1); + _xadj[0] = 0; + + for (int i = 1; i <= _nbr_vtxs; ++i) { + long read_val; + graph_file.read(reinterpret_cast(&read_val), sizeof(long)); + _xadj[i] = read_val; + } + + _adjncy.resize(_edges, 1); + for (int i = 0; i < _edges; ++i) { + long read_val; + graph_file.read(reinterpret_cast(&read_val), sizeof(long)); + _adjncy[i] = read_val; + } + + /* @todo: Add logging, for the moment just comment the code. + for (int i = 0; i < _nbr_vtxs; ++i) { + std::cout << "Node " << i << " Connections: "; + for(int j = _xadj[i]; j < _xadj[i+1]; ++j) { + std::cout << _adjncy[j] << " "; + } + std:: cout << std::endl; + }*/ + + graph_file.close(); + + // @todo: Read weights files. + _vwgt.resize(_nbr_vtxs, 1); + _ewgt.resize(_edges, 1); +} + +PartitionMethod GraphPartitioner::partitionMethod(const std::string &partition_method) const +{ + if (auto it = PARTITION_METHOD_MAP.find(partition_method); it != PARTITION_METHOD_MAP.end()) { + return it->second; + } + return PartitionMethod::Unknown; +} + +void GraphPartitioner::savePartitionToFile(const Partition &partition, const std::string &method_name) const +{ + std::string fileName = _name + "-" + method_name + "-" + std::to_string(_nbr_parts) + "-" + std::to_string(_nbr_vtxs) + ".partition"; + std::ofstream file(fileName); + if (file.is_open()) { + for (const auto &value : partition.values) { + file << value << std::endl; + } + file.close(); + } +} + +void GraphPartitioner::partitionUsingMetis(Partition &partition) +{ + idx_t ncon = 1; + idx_t edgecut; + std::array options; + + METIS_SetDefaultOptions(options.data()); + options[METIS_OPTION_CONTIG] = 1; + options[METIS_OPTION_PTYPE] = METIS_PTYPE_KWAY; + options[METIS_OPTION_OBJTYPE] = METIS_OBJTYPE_VOL; + options[METIS_OPTION_SEED] = 1; + + METIS_PartGraphKway(&_nbr_vtxs, &ncon, _xadj.data(), _adjncy.data(), _vwgt.data(), nullptr, _ewgt.data(), &_nbr_parts, nullptr, nullptr, + options.data(), &edgecut, partition.values.data()); +} + +void GraphPartitioner::partitionUsingHMetis(Partition &partition) +{ + std::string h_graph_name = _name + ".hmetis"; + if (fork() == 0) { + // Child process + createHMetisGraphFile(h_graph_name); + executeKhmetis(h_graph_name); + std::abort(); + } else { + wait(nullptr); + readPartitionFile(h_graph_name, partition); + } +} + +void GraphPartitioner::createHMetisGraphFile(const std::string &file_name) +{ + std::ofstream h_graph_file(file_name); + if (h_graph_file.is_open()) { + h_graph_file << _edges << " " << _nbr_vtxs << "\n"; + for (grp_t i = 0; i < _edges; ++i) { + h_graph_file << _ewgt[i] << " "; + for (grp_t j = _xadj[i]; j < _xadj[i + 1]; ++j) { + h_graph_file << _adjncy[j] + 1 << " "; + } + h_graph_file << "\n"; + } + for (grp_t i = 0; i < _nbr_vtxs; ++i) { + if (!_vwgt.empty()) { + h_graph_file << _vwgt[i] << "\n"; + } + } + h_graph_file.close(); + } +} + +void GraphPartitioner::executeKhmetis(const std::string &h_graph_name) const +{ + std::string parts = std::to_string(_nbr_parts); + execlp("./khmetis", "./khmetis", h_graph_name.c_str(), parts.c_str(), "5", "10", "1", "1", "0", "0", nullptr); +} + +void GraphPartitioner::readPartitionFile(const std::string &file_name, Partition &partition) const +{ + std::ifstream part_file(file_name); + if (!part_file.is_open()) { + std::cerr << "Error opening file: " << file_name << std::endl; + return; + } + + grp_t part; + while (part_file >> part) { // Read until the end of the file + partition.values.push_back(part); + } + part_file.close(); +} + +void GraphPartitioner::partitionUsingScotch(Partition &partition) +{ + SCOTCH_Graph *graph_sc = SCOTCH_graphAlloc(); + if (graph_sc == nullptr) { + std::cerr << "Error allocating graph" << std::endl; + return; + } + + SCOTCH_Strat *strat = SCOTCH_stratAlloc(); + if (SCOTCH_stratInit(strat)) { + std::cerr << "Error allocating graph" << std::endl; + return; + } + + if (SCOTCH_stratGraphMapBuild(strat, SCOTCH_STRATDEFAULT, 4, 0.05)) { + std::cerr << "Error allocating graph" << std::endl; + return; + } + + if (SCOTCH_graphBuild(graph_sc, 0, _nbr_vtxs, _xadj.data(), nullptr, _vwgt.data(), nullptr, _edges, _adjncy.data(), _ewgt.data()) != 0) { + std::cerr << "Error: Scotch Graph Build" << std::endl; + return; + } + if (SCOTCH_graphPart(graph_sc, _nbr_parts, strat, partition.values.data()) != 0) { + std::cerr << "Error: Scotch Graph Partition" << std::endl; + return; + } + + SCOTCH_stratExit(strat); + SCOTCH_graphFree(graph_sc); +} + +void GraphPartitioner::partitionUsingKaHip(Partition &partition) +{ + int edge_cut = 0; + int seed = 0; + const bool SUPPRESS_OUTPUT = false; + + kaffpa(&_nbr_vtxs, nullptr, _xadj.data(), nullptr, _adjncy.data(), &_nbr_parts, &_imbalance, SUPPRESS_OUTPUT, seed, STRONG, &edge_cut, + partition.values.data()); +} + +void GraphPartitioner::readGraphFromSBG(const SBG::LIB::WeightedSBGraph &sbg_graph) +{ + _nbr_vtxs = sbg_graph.V().cardinal(); + _edges = sbg_graph.E().cardinal(); + // asumming all setpiece are unidimensional and have step 1 + int max_node = -1; + for (const auto &v : sbg_graph.V()) { + if (int(v.begin()[0].end()) > max_node) { + max_node = int(v.begin()[0].end()); + } + } + + _xadj.push_back(0); + + for (int i = 0; i <= max_node; i++) { + auto s = SBG::LIB::SET_FACT.createSet(SBG::LIB::Interval(i, 1, i)); + auto edges1 = sbg_graph.map1().preImage(s); + auto nodes1 = sbg_graph.map2().image(edges1); + + auto edges2 = sbg_graph.map2().preImage(s); + auto nodes2 = sbg_graph.map1().image(edges2); + + auto nodes = nodes1.cup(nodes2); + auto nodes_vector = std::vector(); + nodes_vector.reserve(nodes.cardinal()); + + for (const auto &n : nodes) { + for (int val = n.begin()[0].begin(); val <= n.begin()[0].end(); val++) { + nodes_vector.push_back(val); + } + } + + sort(nodes_vector.begin(), nodes_vector.end()); + + for_each(nodes_vector.cbegin(), nodes_vector.cend(), [this](const auto &val) { _adjncy.push_back(int(val)); }); + _xadj.push_back(_xadj.back() + nodes_vector.size()); + } + + /*// @todo: Add logging, for the moment just comment the code. + for (int i = 0; i < _nbr_vtxs; ++i) { + std::cout << "Node " << i << " Connections: "; + for(int j = _xadj[i]; j < _xadj[i+1]; ++j) { + std::cout << _adjncy[j] << " "; + } + std:: cout << std::endl; + }*/ + + // @todo: Read weights files. + _vwgt.resize(_nbr_vtxs, 1); + _ewgt.resize(_edges, 1); +} \ No newline at end of file diff --git a/test/partitioner/external_tools/graph_partitioner.hpp b/test/partitioner/external_tools/graph_partitioner.hpp new file mode 100644 index 00000000..278fe234 --- /dev/null +++ b/test/partitioner/external_tools/graph_partitioner.hpp @@ -0,0 +1,92 @@ +/***************************************************************************** + + This file is part of SBG Partitioner. + + SBG Partitioner is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + SBG Partitioner is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with SBG Partitioner. If not, see . + + *****************************************************************************/ + +#include +#include +#include +#include + + +namespace SBG { + namespace LIB { + class WeightedSBGraph; + } +} + + +using grp_t = int; + +enum class PartitionMethod { + Metis, + HMetis, + Scotch, + Kahip, + Unknown +}; + +struct Partition { + public: + std::vector values; + + void resize(grp_t size) { values.resize(size); } +}; + +class GraphPartitioner { + public: + explicit GraphPartitioner(const std::string &name); + + Partition createPartition(const std::string& partition_method, unsigned int partitions); + + static std::string validPartitionMethodsStr(); + + private: + PartitionMethod partitionMethod(const std::string& partition_method) const; + + void generateInputGraph(); + + bool endsWithJson(); + + void readGraphFromJson(); + void readGraph(); + + void savePartitionToFile(const Partition &partition, const std::string& method_name) const; + void readPartitionFile(const std::string &file_name, Partition &partition) const; + + void partitionUsingMetis(Partition &partition); + void partitionUsingHMetis(Partition &partition); + void createHMetisGraphFile(const std::string &file_name); + void executeKhmetis(const std::string &h_graph_name) const; + + void partitionUsingScotch(Partition &partition); + + void partitionUsingKaHip(Partition &partition); + + void readGraphFromSBG(const SBG::LIB::WeightedSBGraph& sbg_graph); + + std::string _name; + grp_t _edges; + grp_t _nbr_parts; + grp_t _nbr_vtxs; + double _imbalance; + + std::vector _xadj; + std::vector _adjncy; + std::vector _vwgt; + std::vector _ewgt; +}; diff --git a/test/partitioner/external_tools/main.cpp b/test/partitioner/external_tools/main.cpp new file mode 100644 index 00000000..dbb3162d --- /dev/null +++ b/test/partitioner/external_tools/main.cpp @@ -0,0 +1,91 @@ +/***************************************************************************** + + This file is part of SBG Partitioner. + + SBG Partitioner is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + SBG Partitioner is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with SBG Partitioner. If not, see . + + *****************************************************************************/ + +#include +#include +#include +#include +#include +#include + +#include "graph_partitioner.hpp" + +void usage() +{ + std::cout << "Usage: grap_partitioner -f -n -m -i " << std::endl; + std::cout << "Options:" << std::endl; + std::cout << " -f SBG input JSON file name or generated graph input." << std::endl; + std::cout << " -n Number of partitions" << std::endl; + std::cout << " -m Partition method, possible values: " << GraphPartitioner::validPartitionMethodsStr() << std::endl; + std::cout << " -i Partition imbalance" << std::endl; + std::cout << " -h Display this information and exit" << std::endl; +} + +void parseArgs(int argc, char* argv[], std::string& sbg_json_input, int& partitions, std::string& partition_method, int& imbalance) +{ + int opt; + + while ((opt = getopt(argc, argv, "f:n:m:i:h")) != -1) { + switch (opt) { + case 'f': + sbg_json_input = optarg; + break; + case 'n': + partitions = std::stoi(optarg); + break; + case 'm': + partition_method = optarg; + break; + case 'i': + imbalance = std::stoi(optarg); + break; + default: + usage(); + exit(EXIT_SUCCESS); + } + } + + if (sbg_json_input.empty() || partitions <= 0 || partition_method.empty() || imbalance < 0) { + usage(); + exit(EXIT_FAILURE); + } +} + +int main(int argc, char* argv[]) +{ + std::string json_file_name; + int partitions; + std::string partition_method; + int imbalance; + + parseArgs(argc, argv, json_file_name, partitions, partition_method, imbalance); + + GraphPartitioner partitioner(json_file_name); + Partition partition = partitioner.createPartition(partition_method, partitions); + + // write results to a file + std::filesystem::path json_filesystem_path(json_file_name); + json_filesystem_path = json_filesystem_path.replace_extension(); + std::ofstream output_file(json_filesystem_path.filename().string() + "_" + partition_method + "_" + std::to_string(partitions) + + "_output.txt"); + std::for_each(partition.values.cbegin(), partition.values.cend(), + [&output_file](const auto& val) { output_file << std::to_string(val) << "\n"; }); + + return 0; +} \ No newline at end of file