|
| 1 | +# Common.py |
| 2 | +# |
| 3 | +# requires Python 2.4 or later |
| 4 | +# |
| 5 | +# a collection of useful tools |
| 6 | +# |
| 7 | +# writen by John Dickinson |
| 8 | +# with help from a few other sources (the Internet and books) |
| 9 | + |
| 10 | +import sys # for logged_function, handle_exception |
| 11 | +import time # for deferred_thread,logged_function |
| 12 | +import os # for deferred_fork |
| 13 | +import cPickle # for deferred_fork |
| 14 | +import signal # for deferred_fork |
| 15 | +from threading import Thread # for deferred_thread |
| 16 | +import copy # for deep_update |
| 17 | +import traceback # for handle_exception |
| 18 | +import logging # for handle_exception |
| 19 | + |
| 20 | +######################### |
| 21 | +# example usage: |
| 22 | +# |
| 23 | +# @deferred_thread |
| 24 | +# def foo(): |
| 25 | +# ...long running task... |
| 26 | +# |
| 27 | +# ...in your code... |
| 28 | +# x = foo() |
| 29 | +# ...some time passes... |
| 30 | +# print x() <-- prints result from foo |
| 31 | + |
| 32 | +class DeferredThread(object): |
| 33 | + def __init__(self,f,*a,**kw): |
| 34 | + self.__f = f |
| 35 | + self.__exception = None |
| 36 | + self.__thread = Thread(target=self.runfunc,args=a,kwargs=kw) |
| 37 | + time.sleep(0) # give other threads a chance to run |
| 38 | + self.__thread.start() |
| 39 | + |
| 40 | + def __call__(self, *a, **kw): |
| 41 | + self.__thread.join() |
| 42 | + if self.__exception is not None: |
| 43 | + raise self.__exception |
| 44 | + return self.__val |
| 45 | + |
| 46 | + def runfunc(self,*a,**kw): |
| 47 | + try: |
| 48 | + self.__val = self.__f(*a,**kw) |
| 49 | + except Exception, e: |
| 50 | + self.__exception = e |
| 51 | + |
| 52 | +def deferred_thread(f): |
| 53 | + def wrapper(*a,**kw): |
| 54 | + return DeferredThread(f,*a,**kw) |
| 55 | + wrapper.__name__ = f.__name__ |
| 56 | + return wrapper |
| 57 | + |
| 58 | +######################### |
| 59 | +# example usage: |
| 60 | +# |
| 61 | +# @deferred_fork |
| 62 | +# def foo(): |
| 63 | +# ...long running task... |
| 64 | +# |
| 65 | +# ...in your code... |
| 66 | +# x = foo() |
| 67 | +# ...some time passes... |
| 68 | +# print x() <-- prints result from foo |
| 69 | +# |
| 70 | +# Caveat: The function you wrap must return pickle'able objects. |
| 71 | + |
| 72 | +class DeferredForkError(Exception): |
| 73 | + pass |
| 74 | + |
| 75 | +class DeferredFork(object): |
| 76 | + def __init__(self,f,*a,**kw): |
| 77 | + self.__result = None |
| 78 | + self.__called_before = False |
| 79 | + self.__exit_status = 0 |
| 80 | + self.__r,w = os.pipe() |
| 81 | + self.__pid = os.fork() |
| 82 | + if self.__pid: |
| 83 | + # we are the parent |
| 84 | + os.close(w) |
| 85 | + else: |
| 86 | + # we are the child |
| 87 | + os.close(self.__r) |
| 88 | + ret_code = 0 |
| 89 | + out_msg = None |
| 90 | + try: |
| 91 | + out_msg = f(*a,**kw) |
| 92 | + except: |
| 93 | + ret_code = 1 |
| 94 | + import traceback, StringIO |
| 95 | + err = StringIO.StringIO() |
| 96 | + traceback.print_exc(file=err) |
| 97 | + out_msg = err.getvalue() |
| 98 | + err.close() |
| 99 | + w = os.fdopen(w,'wb') |
| 100 | + w.write(cPickle.dumps(out_msg)) |
| 101 | + w.close() |
| 102 | + os._exit(ret_code) |
| 103 | + #sys.exit(ret_code) |
| 104 | + |
| 105 | + def __call__(self, *a, **kw): |
| 106 | + if not self.__called_before: |
| 107 | + r = os.fdopen(self.__r, 'rb') |
| 108 | + self.__result = cPickle.loads(r.read()) |
| 109 | + r.close() |
| 110 | + self.__exit_status = os.waitpid(self.__pid,0)[1] >> 8 |
| 111 | + self.__called_before = True |
| 112 | + if self.__exit_status: |
| 113 | + raise DeferredForkError,self.__result |
| 114 | + return self.__result |
| 115 | + |
| 116 | +def deferred_fork(f): |
| 117 | + def wrapper(*a, **kw): |
| 118 | + return DeferredFork(f,*a,**kw) |
| 119 | + wrapper.__name__ = f.__name__ |
| 120 | + return wrapper |
| 121 | + |
| 122 | +########################## |
| 123 | +# example usage: |
| 124 | +# |
| 125 | +# times2 = curry(operator.mul,2) |
| 126 | +# print times2(4) <-- prints 8 |
| 127 | + |
| 128 | +def curry(f, *a, **kw): |
| 129 | + def curried(*more_a, **more_kw): |
| 130 | + return f(*(a+more_a), **dict(kw, **more_kw)) |
| 131 | + return curried |
| 132 | + |
| 133 | + |
| 134 | +########################## |
| 135 | +# example usage: |
| 136 | +# |
| 137 | +# @logged_function |
| 138 | +# def foo(): |
| 139 | +# pass |
| 140 | +# |
| 141 | +# @logged_funtion('post',open('/tmp/log')) |
| 142 | +# def bar(): |
| 143 | +# pass |
| 144 | +# |
| 145 | +# @logged_funtion(when='post',out=open('/tmp/log')) |
| 146 | +# def bar(): |
| 147 | +# pass |
| 148 | + |
| 149 | +def logged_function(*args,**kw): |
| 150 | + # handle variations of calls for decorators |
| 151 | + try: |
| 152 | + callit = args[0] |
| 153 | + except IndexError: |
| 154 | + callit = lambda func: logged_function(func,*args,**kw) |
| 155 | + if not callable(callit): |
| 156 | + return lambda func: logged_function(func,*args,**kw) |
| 157 | + |
| 158 | + # set parameters (or use defaults) |
| 159 | + when = 'pre' |
| 160 | + out = sys.stderr |
| 161 | + try: |
| 162 | + when = kw.get('when') or args[1] |
| 163 | + except: |
| 164 | + pass |
| 165 | + try: |
| 166 | + out = kw.get('out') or args[2] |
| 167 | + except: |
| 168 | + pass |
| 169 | + |
| 170 | + # now, the work |
| 171 | + def pre_logged(f): |
| 172 | + def wrapper(*a, **kw): |
| 173 | + a_list = ','.join(`i` for i in a) |
| 174 | + kw_list = ','.join('%s=%s'%(k,`v`) for k,v in kw.items()) |
| 175 | + params = ','.join([a_list,kw_list]).strip(',') |
| 176 | + print >>out, 'Log: %s(%s) called' % (f.__name__,params) |
| 177 | + out.flush() |
| 178 | + return f(*a, **kw) |
| 179 | + wrapper.__name__ = f.__name__ |
| 180 | + return wrapper |
| 181 | + def post_logged(f): |
| 182 | + def wrapper(*a, **kw): |
| 183 | + try: |
| 184 | + start = time.time() |
| 185 | + return f(*a, **kw) |
| 186 | + finally: |
| 187 | + stop = time.time() |
| 188 | + a_list = ','.join(`i` for i in a) |
| 189 | + kw_list = ','.join('%s=%s'%(k,`v`) for k,v in kw.items()) |
| 190 | + params = ','.join([a_list,kw_list]).strip(',') |
| 191 | + print >>out, 'Log: %s(%s) called (duration : %.4f)' % (f.__name__,params, stop-start) |
| 192 | + out.flush() |
| 193 | + wrapper.__name__ = f.__name__ |
| 194 | + return wrapper |
| 195 | + return {'pre':pre_logged,'post':post_logged}[when](callit) |
| 196 | + |
| 197 | + |
| 198 | +########################## |
| 199 | +# like a typical dictionary update, but if for a key |
| 200 | +# both values are a dict, merge those recursively |
| 201 | + |
| 202 | +def deep_update(d1,d2): |
| 203 | + if type(d1) is not type({}) or type(d2) is not type({}): |
| 204 | + return None |
| 205 | + |
| 206 | + res = copy.deepcopy(d1) |
| 207 | + for key in d2: |
| 208 | + if not d1.has_key(key): |
| 209 | + res[key] = None |
| 210 | + t1 = type(d1.get(key)) |
| 211 | + t2 = type(d2.get(key)) |
| 212 | + if t1 is t2 and t1 is type({}): |
| 213 | + res[key] = deep_update(d1[key],d2[key]) |
| 214 | + else: |
| 215 | + res[key] = d2[key] |
| 216 | + |
| 217 | + return res |
| 218 | + |
| 219 | + |
| 220 | +########################## |
| 221 | +# arbitrary infix notation |
| 222 | +# |
| 223 | +# example usage: |
| 224 | +# |
| 225 | +# @infix |
| 226 | +# def foo(left,right): |
| 227 | +# return left+right |
| 228 | +# print 2 |foo| 2 # prints 4 |
| 229 | +# print 2 <<foo>> 2 # prints 4 |
| 230 | +# print 2 >>foo<< 2 # prints 4 |
| 231 | + |
| 232 | +class Infix(object): |
| 233 | + def __init__(self,func): |
| 234 | + self._func = func |
| 235 | + |
| 236 | + # |foo| |
| 237 | + def __or__(self,other): |
| 238 | + return self._func(other) |
| 239 | + def __ror__(self,other): |
| 240 | + return Infix(lambda x: self._func(other,x)) |
| 241 | + |
| 242 | + # <<foo>> |
| 243 | + def __rshift__(self,other): |
| 244 | + return self._func(other) |
| 245 | + def __rlshift__(self,other): |
| 246 | + return Infix(lambda x: self._func(other,x)) |
| 247 | + |
| 248 | + # >>foo<< |
| 249 | + def __lshift__(self,other): |
| 250 | + return self._func(other) |
| 251 | + def __rrshift__(self,other): |
| 252 | + return Infix(lambda x: self._func(other,x)) |
| 253 | + |
| 254 | + def __call__(self,a,b): |
| 255 | + return self._func(a,b) |
| 256 | +def infix(f): |
| 257 | + return Infix(f) |
| 258 | + |
| 259 | + |
| 260 | +########################## |
| 261 | +# callback decorator |
| 262 | +# |
| 263 | +# example usage: |
| 264 | +# |
| 265 | +# def my_callback(called_proc,called_proc_result): |
| 266 | +# pass |
| 267 | +# |
| 268 | +# @add_callback(my_callback) # addl args will be passed to callback funtion |
| 269 | +# def long_running(): |
| 270 | +# pass |
| 271 | + |
| 272 | +def add_callback(callback_proc,*args,**kw): |
| 273 | + def executor(f): |
| 274 | + def wrapped(*wrapped_a,**wrapped_kw): |
| 275 | + try: |
| 276 | + res = f(*wrapped_a,**wrapped_kw) |
| 277 | + finally: |
| 278 | + callback_proc(f,res,*args,**kw) |
| 279 | + wrapped.__name__ = f.__name__ |
| 280 | + return wrapped |
| 281 | + return executor |
| 282 | + |
| 283 | +########################## |
| 284 | +# handle_exception decorator |
| 285 | +# |
| 286 | +# example usage: |
| 287 | +# |
| 288 | +# @handle_exception |
| 289 | +# def foo(a,b,c): |
| 290 | +# raise Exception, 'Oh no!' |
| 291 | +# |
| 292 | +# @handle_exception(open('/tmp/error.log','ab')) |
| 293 | +# def foo(a,b,c): |
| 294 | +# raise Exception, 'Oh no!' |
| 295 | + |
| 296 | +def handle_exception(out_file=None): |
| 297 | + '''wraps a function and catches errors in it |
| 298 | + if out is given, it is an open file object |
| 299 | + if out is None (default), messages will be logged with logging.getLogger().error(...) |
| 300 | + ''' |
| 301 | + def wrapper(f): |
| 302 | + def wrapped(*a, **kw): |
| 303 | + try: |
| 304 | + ret = f(*a, **kw) |
| 305 | + except: |
| 306 | + try: |
| 307 | + t, v, _tb = sys.exc_info() |
| 308 | + tb = traceback.format_exc(_tb) |
| 309 | + finally: # see http://docs.python.org/lib/module-sys.html |
| 310 | + del _tb |
| 311 | + msg = '%s caught in %s:\n%s' % (t, f.__name__, tb) |
| 312 | + if out_file is None: |
| 313 | + logging.getLogger().error(msg) |
| 314 | + else: |
| 315 | + out_file.write('%s\n' % msg) |
| 316 | + wrapped.__name__ = f.__name__ |
| 317 | + wrapped.__doc__ = f.__doc__ |
| 318 | + return wrapped |
| 319 | + if callable(out_file): |
| 320 | + f = out_file |
| 321 | + out_file = None |
| 322 | + return wrapper(f) |
| 323 | + else: |
| 324 | + return wrapper |
0 commit comments