Skip to content

Commit 0f0fe49

Browse files
author
Hoyt Koepke
committed
Fixes for windows linking issues; disable unneeded spark_unity on windows
1 parent 7e78e1e commit 0f0fe49

10 files changed

Lines changed: 166 additions & 149 deletions

File tree

‎local_cmake/UseCython.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ function( cython_add_module _name )
250250
foreach( _file ${ARGN} )
251251
if( ${_file} MATCHES ".*\\.py[x]?$" )
252252
list( APPEND pyx_module_sources ${_file} )
253-
else()
253+
else()
254254
list( APPEND other_module_sources ${_file} )
255255
endif()
256256
endforeach()

‎oss_src/lambda/CMakeLists.txt‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ make_library(pylambda
88
graph_pylambda_master.cpp
99
# lualambda_master.cpp
1010
REQUIRES
11-
flexible_type cppipc sframe shmipc
11+
flexible_type cppipc sframe shmipc python_callbacks process
1212
# luastate luajit
1313
EXTERNAL_VISIBILITY
1414
)
@@ -17,9 +17,11 @@ make_library(pylambda
1717
make_library(pylambda_worker
1818
SOURCES
1919
pylambda.cpp
20+
pylambda_worker.cpp
2021
graph_pylambda.cpp
2122
REQUIRES
22-
flexible_type cppipc sframe shmipc
23+
flexible_type cppipc sframe shmipc python_callbacks process
24+
EXTERNAL_VISIBILITY
2325
)
2426

2527

‎oss_src/lambda/pylambda_worker.cpp‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
#include <lambda/pylambda_worker.hpp>
2+
#include <cppipc/server/comm_server.hpp>
3+
#include <lambda/pylambda.hpp>
4+
#include <shmipc/shmipc.hpp>
5+
#include <lambda/graph_pylambda.hpp>
6+
#include <logger/logger.hpp>
7+
#include <process/process_util.hpp>
8+
#include <util/try_finally.hpp>
9+
10+
namespace graphlab { namespace lambda {
11+
12+
/** The main function to be called from the python ctypes library to
13+
* create a pylambda worker process.
14+
*
15+
* Different error routes produce different error codes of 101 and
16+
* above.
17+
*/
18+
int EXPORT pylambda_worker_main(const std::string& root_path,
19+
const std::string& server_address, int loglevel) {
20+
21+
/** Set up the debug configuration.
22+
*
23+
* By default, all LOG_ERROR and LOG_FATAL messages are sent to
24+
* stderr, and all messages above loglevel are sent to stdout.
25+
*
26+
* If GRAPHLAB_LAMBDA_WORKER_LOG_FILE is set and is non-empty, then
27+
* all log messages are sent to the file instead of the stdout and
28+
* stderr. In this case, the only errors logged to stderr/stdout
29+
* concern opening the log file.
30+
*
31+
* If GRAPHLAB_LAMBDA_WORKER_DEBUG_MODE is set, then the default
32+
* log level is set to LOG_DEBUG. If a log file is set, then all
33+
* log messages are sent there, otherwise they are sent to stderr.
34+
*/
35+
boost::optional<std::string> debug_mode_str = graphlab::getenv_str("GRAPHLAB_LAMBDA_WORKER_DEBUG_MODE");
36+
boost::optional<std::string> debug_mode_file_str = graphlab::getenv_str("GRAPHLAB_LAMBDA_WORKER_LOG_FILE");
37+
38+
std::string log_file_string = debug_mode_file_str ? *debug_mode_file_str : "";
39+
bool log_to_file = (!log_file_string.empty());
40+
41+
bool debug_mode = (bool)(debug_mode_str);
42+
43+
global_logger().set_log_level(loglevel);
44+
global_logger().set_log_to_console(true);
45+
46+
// Logging using the LOG_DEBUG_WITH_PID macro requires this_pid to be set.
47+
size_t this_pid = get_my_pid();
48+
global_logger().set_pid(this_pid);
49+
50+
// Set up the logging to file if needed.
51+
if(log_to_file) {
52+
// Set up the logging to the file, with any errors being fully logged.
53+
global_logger().set_log_to_console(true, true);
54+
global_logger().set_log_file(log_file_string);
55+
LOG_DEBUG_WITH_PID("Logging lambda worker logs to " << log_file_string);
56+
global_logger().set_log_to_console(false);
57+
}
58+
59+
// Now, set the log mode for debug
60+
if(debug_mode) {
61+
global_logger().set_log_level(LOG_DEBUG);
62+
if(!log_to_file) {
63+
// Set logging to console, with everything logged to stderr.
64+
global_logger().set_log_to_console(true, true);
65+
}
66+
}
67+
68+
// Log the basic information about parameters.
69+
size_t parent_pid = get_parent_pid();
70+
71+
LOG_DEBUG_WITH_PID("root_path = '" << root_path << "'");
72+
LOG_DEBUG_WITH_PID("server_address = '" << server_address << "'");
73+
LOG_DEBUG_WITH_PID("parend pid = " << parent_pid);
74+
75+
try {
76+
77+
LOG_DEBUG_WITH_PID("Library function entered successfully.");
78+
79+
if(server_address == "debug") {
80+
logstream(LOG_INFO) << "Exiting dry run." << std::endl;
81+
return 1;
82+
}
83+
84+
graphlab::shmipc::server shm_comm_server;
85+
bool has_shm = shm_comm_server.bind();
86+
87+
LOG_DEBUG_WITH_PID("shm_comm_server bind: has_shm=" << has_shm);
88+
89+
// construct the server
90+
cppipc::comm_server server(std::vector<std::string>(), "", server_address);
91+
92+
server.register_type<graphlab::lambda::lambda_evaluator_interface>([&](){
93+
if (has_shm) {
94+
auto n = new graphlab::lambda::pylambda_evaluator(&shm_comm_server);
95+
LOG_DEBUG_WITH_PID("creation of pylambda_evaluator with SHM complete.");
96+
return n;
97+
} else {
98+
auto n = new graphlab::lambda::pylambda_evaluator();
99+
LOG_DEBUG_WITH_PID("creation of pylambda_evaluator without SHM complete.");
100+
return n;
101+
}
102+
});
103+
server.register_type<graphlab::lambda::graph_lambda_evaluator_interface>([&](){
104+
auto n = new graphlab::lambda::graph_pylambda_evaluator();
105+
LOG_DEBUG_WITH_PID("creation of graph_pylambda_evaluator complete.");
106+
return n;
107+
});
108+
109+
LOG_DEBUG_WITH_PID("Starting server.");
110+
server.start();
111+
112+
wait_for_parent_exit(parent_pid);
113+
114+
return 0;
115+
116+
/** Any exceptions happening?
117+
*/
118+
} catch (const std::string& error) {
119+
logstream(LOG_ERROR) << "Internal PyLambda Error: " << error << std::endl;
120+
return 103;
121+
} catch (const std::exception& error) {
122+
logstream(LOG_ERROR) << "PyLambda C++ Error: " << error.what() << std::endl;
123+
return 104;
124+
} catch (...) {
125+
logstream(LOG_ERROR) << "Unknown PyLambda Error." << std::endl;
126+
return 105;
127+
}
128+
}
129+
130+
}}
131+

‎oss_src/lambda/pylambda_worker.hpp‎

Lines changed: 3 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -8,134 +8,15 @@
88
#ifndef GRAPHLAB_LAMBDA_PYLAMBDA_WORKER_H_
99
#define GRAPHLAB_LAMBDA_PYLAMBDA_WORKER_H_
1010

11-
#include <cppipc/server/comm_server.hpp>
12-
#include <lambda/pylambda.hpp>
13-
#include <shmipc/shmipc.hpp>
14-
#include <lambda/graph_pylambda.hpp>
15-
#include <logger/logger.hpp>
16-
#include <process/process_util.hpp>
17-
#include <util/try_finally.hpp>
11+
#include <string>
1812

1913
namespace graphlab { namespace lambda {
2014

2115
/** The main function to be called from the python ctypes library to
2216
* create a pylambda worker process.
23-
*
24-
* Different error routes produce different error codes of 101 and
25-
* above.
2617
*/
27-
static int pylambda_worker_main(const char* _root_path, const char* _server_address, int loglevel) {
28-
29-
/** Set up the debug configuration.
30-
*
31-
* By default, all LOG_ERROR and LOG_FATAL messages are sent to
32-
* stderr, and all messages above loglevel are sent to stdout.
33-
*
34-
* If GRAPHLAB_LAMBDA_WORKER_LOG_FILE is set and is non-empty, then
35-
* all log messages are sent to the file instead of the stdout and
36-
* stderr. In this case, the only errors logged to stderr/stdout
37-
* concern opening the log file.
38-
*
39-
* If GRAPHLAB_LAMBDA_WORKER_DEBUG_MODE is set, then the default
40-
* log level is set to LOG_DEBUG. If a log file is set, then all
41-
* log messages are sent there, otherwise they are sent to stderr.
42-
*/
43-
boost::optional<std::string> debug_mode_str = graphlab::getenv_str("GRAPHLAB_LAMBDA_WORKER_DEBUG_MODE");
44-
boost::optional<std::string> debug_mode_file_str = graphlab::getenv_str("GRAPHLAB_LAMBDA_WORKER_LOG_FILE");
45-
46-
std::string log_file_string = debug_mode_file_str ? *debug_mode_file_str : "";
47-
bool log_to_file = (!log_file_string.empty());
48-
49-
bool debug_mode = (bool)(debug_mode_str);
50-
51-
global_logger().set_log_level(loglevel);
52-
global_logger().set_log_to_console(true);
53-
54-
// Logging using the LOG_DEBUG_WITH_PID macro requires this_pid to be set.
55-
size_t this_pid = get_my_pid();
56-
global_logger().set_pid(this_pid);
57-
58-
// Set up the logging to file if needed.
59-
if(log_to_file) {
60-
// Set up the logging to the file, with any errors being fully logged.
61-
global_logger().set_log_to_console(true, true);
62-
global_logger().set_log_file(log_file_string);
63-
LOG_DEBUG_WITH_PID("Logging lambda worker logs to " << log_file_string);
64-
global_logger().set_log_to_console(false);
65-
}
66-
67-
// Now, set the log mode for debug
68-
if(debug_mode) {
69-
global_logger().set_log_level(LOG_DEBUG);
70-
if(!log_to_file) {
71-
// Set logging to console, with everything logged to stderr.
72-
global_logger().set_log_to_console(true, true);
73-
}
74-
}
75-
76-
// Log the basic information about parameters.
77-
std::string server_address = _server_address;
78-
std::string root_path = _root_path;
79-
size_t parent_pid = get_parent_pid();
80-
81-
LOG_DEBUG_WITH_PID("root_path = '" << root_path << "'");
82-
LOG_DEBUG_WITH_PID("server_address = '" << server_address << "'");
83-
LOG_DEBUG_WITH_PID("parend pid = " << parent_pid);
84-
85-
try {
86-
87-
LOG_DEBUG_WITH_PID("Library function entered successfully.");
88-
89-
if(server_address == "debug") {
90-
logstream(LOG_INFO) << "Exiting dry run." << std::endl;
91-
return 1;
92-
}
93-
94-
graphlab::shmipc::server shm_comm_server;
95-
bool has_shm = shm_comm_server.bind();
96-
97-
LOG_DEBUG_WITH_PID("shm_comm_server bind: has_shm=" << has_shm);
98-
99-
// construct the server
100-
cppipc::comm_server server(std::vector<std::string>(), "", server_address);
101-
102-
server.register_type<graphlab::lambda::lambda_evaluator_interface>([&](){
103-
if (has_shm) {
104-
auto n = new graphlab::lambda::pylambda_evaluator(&shm_comm_server);
105-
LOG_DEBUG_WITH_PID("creation of pylambda_evaluator with SHM complete.");
106-
return n;
107-
} else {
108-
auto n = new graphlab::lambda::pylambda_evaluator();
109-
LOG_DEBUG_WITH_PID("creation of pylambda_evaluator without SHM complete.");
110-
return n;
111-
}
112-
});
113-
server.register_type<graphlab::lambda::graph_lambda_evaluator_interface>([&](){
114-
auto n = new graphlab::lambda::graph_pylambda_evaluator();
115-
LOG_DEBUG_WITH_PID("creation of graph_pylambda_evaluator complete.");
116-
return n;
117-
});
118-
119-
LOG_DEBUG_WITH_PID("Starting server.");
120-
server.start();
121-
122-
wait_for_parent_exit(parent_pid);
123-
124-
return 0;
125-
126-
/** Any exceptions happening?
127-
*/
128-
} catch (const std::string& error) {
129-
logstream(LOG_ERROR) << "Internal PyLambda Error: " << error << std::endl;
130-
return 103;
131-
} catch (const std::exception& error) {
132-
logstream(LOG_ERROR) << "PyLambda C++ Error: " << error.what() << std::endl;
133-
return 104;
134-
} catch (...) {
135-
logstream(LOG_ERROR) << "Unknown PyLambda Error." << std::endl;
136-
return 105;
137-
}
138-
}
18+
int pylambda_worker_main(const std::string& root_path,
19+
const std::string& server_address, int loglevel);
13920

14021
}}
14122

‎oss_src/unity/lib/CMakeLists.txt‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,14 @@ make_library(unity_core
2929
../extensions/additional_sframe_utilities.cpp
3030
REQUIRES
3131
flexible_type
32-
pylambda table_printer
33-
sframe pylambda cancel_serverside_ops
32+
pylambda
33+
table_printer
34+
sframe cancel_serverside_ops
3435
sframe_query_engine
3536
libjson sgraph
3637
image_type image_io
3738
startup_teardown
38-
python_callbacks
39+
pylambda_worker
3940
EXTERNAL_VISIBILITY
4041
)
4142

‎oss_src/unity/python/sframe/_scripts/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
project(unity_cython_scripts)
22

3+
if(NOT WIN32)
4+
35
set_source_files_properties("pyspark_unity.pyx" PROPERTIES CYTHON_IS_CXX TRUE)
46
set_source_files_properties("../cython/cy_flexible_type.pyx" PROPERTIES CYTHON_IS_CXX TRUE)
57
set_source_files_properties("../cython/cy_callback.pyx" PROPERTIES CYTHON_IS_CXX TRUE)
@@ -23,3 +25,4 @@ set_property(TARGET pyspark_unity
2325
target_link_libraries(pyspark_unity python_callbacks spark_unity unity_core)
2426
add_dependencies(pyspark_unity release_binaries)
2527

28+
endif()

‎oss_src/unity/python/sframe/cython/CMakeLists.txt‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,6 @@ foreach(source ${cython_sources})
2121
add_dependencies(cython_targets ${target})
2222
endforeach()
2323

24-
target_link_libraries("cy_pylambda_workers" pylambda_worker)
25-
26-
2724
ADD_CUSTOM_TARGET (oss_clean_cython)
2825
ADD_CUSTOM_COMMAND(
2926
COMMENT "clean cython"

0 commit comments

Comments
 (0)