Added some extra error handling to benchmarker and fixed CMakeLists; updated readme with better quickstart.

Author: Theodore Bugnet

CMakeLists.txt

@@ -14,7 +14,7 @@ SET(CMAKE_CXX_FLAGS_DEBUG "-g")# -pg")
 SET(CMAKE_CXX_FLAGS_RELEASE "-O3")
 
 #Customise these values to customise the installation
-SET(INSTALL_PREF "../../install-test/") #Unset for installation to default binary directory, or change to a different portable directory
+SET(INSTALL_PREF "../../../install/") #Unset for installation to default binary directory, or change to a different portable directory
 SET(STORE_PATH "./store/") #Unless installing to a self-contained location, this should probably be an absolute path, e.g. inside ~
 SET(CONF_PREFIX "./conf/") #Same for this, alternatively set this to a path in /etc/
 SET(CONF_PATH_ANALYSER "./graph_analyser.conf") #The next three are names of files inside CONF_PREFIX
@@ -36,14 +36,14 @@ if (Boost_FOUND)
     target_compile_definitions(graph_analyser PRIVATE CONF_PATH="${CONF_PREFIX}${CONF_PATH_ANALYSER}")
     target_compile_definitions(graph_mapper PRIVATE CONF_PATH="${CONF_PREFIX}${CONF_PATH_MAPPER}")
     target_compile_definitions(report_generator PRIVATE CONF_PATH="${CONF_PREFIX}${CONF_PATH_GENERATOR}" STORE_PATH="${STORE_PATH}")
-    target_compile_definitions(benchmarker PRIVATE CONF_PATH="${CONF_PREFIX}${CONF_PATH_BENCHMARKER}" BIN_PATH="${INSTALL_PREF}")
+    target_compile_definitions(benchmarker PRIVATE CONF_PATH="${CONF_PREFIX}${CONF_PATH_BENCHMARKER}" BIN_PATH=".")
     target_compile_definitions(common PRIVATE STORE_PATH="${STORE_PATH}")
     set(CMAKE_INSTALL_PREFIX ${INSTALL_PREF} CACHE PATH "..." FORCE)
     install(TARGETS graph_analyser RUNTIME DESTINATION .)
     install(TARGETS graph_mapper RUNTIME DESTINATION .)
     install(TARGETS benchmarker RUNTIME DESTINATION .)
     install(TARGETS report_generator RUNTIME DESTINATION .)
     install(DIRECTORY store DESTINATION . USE_SOURCE_PERMISSIONS MESSAGE_NEVER)
-    install(PROGRAMS benchmarker/external/exts/compare-extensions DESTINATION ${INSTALL_PREF})
-    install(PROGRAMS benchmarker/external/runsolver/runsolver DESTINATION ${INSTALL_PREF})
+    install(PROGRAMS benchmarker/external/exts/compare-extensions DESTINATION .)
+    install(PROGRAMS benchmarker/external/runsolver/runsolver DESTINATION .)
 endif()

README.md

@@ -4,6 +4,8 @@ AAF and solver benchmarking and analysis toolkit.
 # Quick start
 The archive contains a full, ready to use environment in the `install-final` folder. To rebuild from source: ensure the necessary dependencies (CMake, Boost, a recent version of GCC/Clang) are installed; in the root `CMakeLists.txt`, configure `INSTALL\_PREF` as desired; `cd build` and run `make install`.
 
+Then place some graphs into store/graphs (or use the -g/-d options to specify graphs), run `graph\_analyser` to generate metric scores and/or `benchmarker` with proper options (see `benchmarker -h`) to generate solver performance data, and then either peruse the store or generate an report on the data using `report\_generator -t <type> [other options]` (see `report\_generator -h` for more information).
+
 ## Installation
 ### Prerequisites
 * Cmake
@@ -76,5 +78,17 @@ As an alternative, any executable file in the directory `store/external-metrics`
 
 This allows new metrics to be easily created without recompiling any other code; it also gives a lot of flexibility in how metrics are developed - the executable could be anything from a Python script to a wrapper calling an external tool located elsewhere.
 
+### Graph hashing
+Graphs are hashed with SpookyHash (http://burtleburtle.net/bob/hash/spooky.html) to ensure no duplication. `graph_mapper` can be used to quickly check what a graph's hash is, or what graph file a hash belongs to. The file `store/graphhashmap` stores key-value pairs between graph filepath and hash, as a cache, and can also be easily explored or `grep`ed manually.
+
+## Store
+The store is where all the data is saved; here is an overview of its structure.
+### `graph-scores`
+Each file here corresponds to a graph. The format of the files should be self-explanatory: each line gives the value of a metric.
+### `bench-solutions`
+Here the solutions for every problem are stored, in subdirectories corresponding to each graph. Each problem has its own file, named after the problem, containing the output of the reference solver used verbatim
+### `benchmarks`
+The output of benchmark runs. Each subfolder here corresponds to a solver. Inside a solver's folder, there is a subfolder for every run ID, inside of which the actual run information is stored. Each benchmark run has a subfolder for every graph, inside which each problem has either 1 or 2 corresponding files named after it: a `<PROBLEM>.stat` file and a `<PROBLEM>.output` file. The latter contains the solver output verbatim, and may not be kept depending on the options passed to `benchmarker` (by default, only outputs for incorrect solutions are kept; you can also specify the max size of solutions to keep). The .stat file contains the performance data of the solver.
+
 # License
 This program is released under the GNU General Public License, version 3 or (at your option) any later version. For more information, see LICENSE.txt or https://www.gnu.org/licenses/.

benchmarker/main.cxx

@@ -34,6 +34,8 @@
 namespace po = boost::program_options;
 namespace fs = std::filesystem;
 
+const int magic_number = 143; //randomly selected, used as special return value
+
 unsigned long getSizeInBytes(std::string sizeWithSuffix)
 {
     unsigned long ret = 0;
@@ -316,11 +318,15 @@ int main(int argc, char** argv)
             {
                 if (referenceSolverpath == "")
                 {   if (!quiet)
-                    {   std::cout << "No reference solver specified and no solution for graph " << graphFile
+                    {   std::cout << "INFO: No reference solver specified and no solution for graph " << graphFile
                             << " and problem " << fullproblem << "; skipping." << std::endl;
                         continue;
                     }
                 }
+                else if (!fs::exists(referenceSolverpath) || !(fs::is_regular_file(referenceSolverpath) || fs::is_symlink(referenceSolverpath)))
+                {   std::cerr << "WARNING: Reference solver path does not point to a file. Double-check the filepath. Skipping graphs with no existing solutions." << std::endl;
+                    continue;
+                }
                 else
                 {   if (verbose)
                     {   std::cout << "    No solution exists; running reference solver..." << std::endl;
@@ -341,7 +347,18 @@ int main(int argc, char** argv)
                         continue;
                     }
                     else if (pid > 0) //parent
-                    {   wait(NULL);
+                    {   int status;
+                        wait(&status);
+                        if (!WIFEXITED(status) || (WIFEXITED(status) && WEXITSTATUS(status) == magic_number))
+                        {   std::cerr << "ERROR: Unable to execute reference solver! Skipping graph." << std::endl;
+                            if(!fs::remove(solfp))
+                            {   std::cerr << "Could not remove solution file..." << std::endl;
+                            }
+                            continue;
+                        }
+                        else if (WIFEXITED(status) && WEXITSTATUS(status) != 0)
+                        {   std::cerr << "WARNING: Reference solver exited with non-zero status: " << WEXITSTATUS(status) << std::endl;
+                        }
                     }
                     else //child
                     {   if (dup2(outfd, STDOUT_FILENO) < 0)
@@ -364,7 +381,7 @@ int main(int argc, char** argv)
                         args[argvct.size()] = NULL;
 
                         execv(referenceSolverpath.c_str(), (char**)args);
-                        std::cout << "AAAAAAAAAAAAAAAAAAAAAAAAAA" << std::endl;
+                        return magic_number;
                     }
                 }
             }
@@ -384,7 +401,16 @@ int main(int argc, char** argv)
                 continue;
             }
             else if (pid > 0) //parent
-            {   wait(NULL);
+            {   int status;
+                wait(&status);
+                if (!WIFEXITED(status) || (WIFEXITED(status) && WEXITSTATUS(status) == magic_number))
+                {   std::cerr << "FATAL ERROR: Failed to execute runsolver. Aborting!" << std::endl;
+                    continue;
+                }
+                else if (WIFEXITED(status) && WEXITSTATUS(status) != 0)
+                {   std::cerr << "WARNING: Runsolver exited with non-zero status. Skipping further processing." << std::endl;
+                    continue;
+                }
             }
             else //child
             {   const char* runsolver = BIN_PATH "/runsolver";
@@ -420,7 +446,7 @@ int main(int argc, char** argv)
                 args[argvct.size()] = NULL;
 
                 execv(runsolver, (char**)args);
-                std::cout << "AAAAAAAAAAAAAAAAAAAAAAAAAA" << std::endl;
+                return magic_number;
             }
 
             //runsolver is done
@@ -458,9 +484,6 @@ int main(int argc, char** argv)
                     << ". No correctness report will be generated. Error message: " << status << std::endl;
                     in.close();
                     is_correct = false; //provide default
-                    if (fs::file_size(outfp) >= saveMaxSize)
-                    {   fs::remove(outfp);
-                    }
                 }
                 else
                 {   is_correct = (status == "OK"? true : false);