gluon.newcron

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 4 """ 5 Created by Attila Csipa <web2py@csipa.in.rs> 6 Modified by Massimo Di Pierro <mdipierro@cs.depaul.edu> 7 """ 8 9 import sys 10 import os 11 import threading 12 import logging 13 import time 14 import sched 15 import re 16 import datetime 17 import platform 18 import portalocker 19 import fileutils 20 import cPickle 21 from gluon.settings import global_settings 22 23 logger = logging.getLogger("web2py.cron") 24 _cron_stopping = False 25 _cron_subprocs = [] 26 27

28 -def absolute_path_link(path):

29 """ 30 Return an absolute path for the destination of a symlink 31 32 """ 33 if os.path.islink(path): 34 link = os.readlink(path) 35 if not os.path.isabs(link): 36 link = os.path.join(os.path.dirname(path), link) 37 else: 38 link = os.path.abspath(path) 39 return link

40 41

42 -def stopcron():

43 "graceful shutdown of cron" 44 global _cron_stopping 45 _cron_stopping = True 46 while _cron_subprocs: 47 _cron_subprocs.pop().terminate()

48

49 -class extcron(threading.Thread):

50

51 - def __init__(self, applications_parent, apps=None):

52 threading.Thread.__init__(self) 53 self.setDaemon(False) 54 self.path = applications_parent 55 self.apps = apps

56 # crondance(self.path, 'external', startup=True, apps=self.apps) 57

58 - def run(self):

59 if not _cron_stopping: 60 logger.debug('external cron invocation') 61 crondance(self.path, 'external', startup=False, apps=self.apps)

62 63

64 -class hardcron(threading.Thread):

65

66 - def __init__(self, applications_parent):

67 threading.Thread.__init__(self) 68 self.setDaemon(True) 69 self.path = applications_parent 70 crondance(self.path, 'hard', startup=True)

71

72 - def launch(self):

73 if not _cron_stopping: 74 logger.debug('hard cron invocation') 75 crondance(self.path, 'hard', startup=False)

76

77 - def run(self):

78 s = sched.scheduler(time.time, time.sleep) 79 logger.info('Hard cron daemon started') 80 while not _cron_stopping: 81 now = time.time() 82 s.enter(60 - now % 60, 1, self.launch, ()) 83 s.run()

84 85

86 -class softcron(threading.Thread):

87

88 - def __init__(self, applications_parent):

89 threading.Thread.__init__(self) 90 self.path = applications_parent

91 # crondance(self.path, 'soft', startup=True) 92

93 - def run(self):

94 if not _cron_stopping: 95 logger.debug('soft cron invocation') 96 crondance(self.path, 'soft', startup=False)

97 98

99 -class Token(object):

100

101 - def __init__(self, path):

102 self.path = os.path.join(path, 'cron.master') 103 if not os.path.exists(self.path): 104 fileutils.write_file(self.path, '', 'wb') 105 self.master = None 106 self.now = time.time()

107

108 - def acquire(self, startup=False):

109 """ 110 returns the time when the lock is acquired or 111 None if cron already running 112 113 lock is implemented by writing a pickle (start, stop) in cron.master 114 start is time when cron job starts and stop is time when cron completed 115 stop == 0 if job started but did not yet complete 116 if a cron job started within less than 60 seconds, acquire returns None 117 if a cron job started before 60 seconds and did not stop, 118 a warning is issue "Stale cron.master detected" 119 """ 120 if sys.platform == 'win32': 121 locktime = 59.5 122 else: 123 locktime = 59.99 124 if portalocker.LOCK_EX is None: 125 logger.warning('WEB2PY CRON: Disabled because no file locking') 126 return None 127 self.master = open(self.path, 'rb+') 128 try: 129 ret = None 130 portalocker.lock(self.master, portalocker.LOCK_EX) 131 try: 132 (start, stop) = cPickle.load(self.master) 133 except: 134 (start, stop) = (0, 1) 135 if startup or self.now - start > locktime: 136 ret = self.now 137 if not stop: 138 # this happens if previous cron job longer than 1 minute 139 logger.warning('WEB2PY CRON: Stale cron.master detected') 140 logger.debug('WEB2PY CRON: Acquiring lock') 141 self.master.seek(0) 142 cPickle.dump((self.now, 0), self.master) 143 self.master.flush() 144 finally: 145 portalocker.unlock(self.master) 146 if not ret: 147 # do this so no need to release 148 self.master.close() 149 return ret

150

151 - def release(self):

152 """ 153 this function writes into cron.master the time when cron job 154 was completed 155 """ 156 if not self.master.closed: 157 portalocker.lock(self.master, portalocker.LOCK_EX) 158 logger.debug('WEB2PY CRON: Releasing cron lock') 159 self.master.seek(0) 160 (start, stop) = cPickle.load(self.master) 161 if start == self.now: # if this is my lock 162 self.master.seek(0) 163 cPickle.dump((self.now, time.time()), self.master) 164 portalocker.unlock(self.master) 165 self.master.close()

166 167

168 -def rangetolist(s, period='min'):

169 retval = [] 170 if s.startswith('*'): 171 if period == 'min': 172 s = s.replace('*', '0-59', 1) 173 elif period == 'hr': 174 s = s.replace('*', '0-23', 1) 175 elif period == 'dom': 176 s = s.replace('*', '1-31', 1) 177 elif period == 'mon': 178 s = s.replace('*', '1-12', 1) 179 elif period == 'dow': 180 s = s.replace('*', '0-6', 1) 181 m = re.compile(r'(\d+)-(\d+)/(\d+)') 182 match = m.match(s) 183 if match: 184 for i in range(int(match.group(1)), int(match.group(2)) + 1): 185 if i % int(match.group(3)) == 0: 186 retval.append(i) 187 return retval

188 189

190 -def parsecronline(line):

191 task = {} 192 if line.startswith('@reboot'): 193 line = line.replace('@reboot', '-1 * * * *') 194 elif line.startswith('@yearly'): 195 line = line.replace('@yearly', '0 0 1 1 *') 196 elif line.startswith('@annually'): 197 line = line.replace('@annually', '0 0 1 1 *') 198 elif line.startswith('@monthly'): 199 line = line.replace('@monthly', '0 0 1 * *') 200 elif line.startswith('@weekly'): 201 line = line.replace('@weekly', '0 0 * * 0') 202 elif line.startswith('@daily'): 203 line = line.replace('@daily', '0 0 * * *') 204 elif line.startswith('@midnight'): 205 line = line.replace('@midnight', '0 0 * * *') 206 elif line.startswith('@hourly'): 207 line = line.replace('@hourly', '0 * * * *') 208 params = line.strip().split(None, 6) 209 if len(params) < 7: 210 return None 211 daysofweek = {'sun': 0, 'mon': 1, 'tue': 2, 'wed': 3, 'thu': 4, 212 'fri': 5, 'sat': 6} 213 for (s, id) in zip(params[:5], ['min', 'hr', 'dom', 'mon', 'dow']): 214 if not s in [None, '*']: 215 task[id] = [] 216 vals = s.split(',') 217 for val in vals: 218 if val != '-1' and '-' in val and '/' not in val: 219 val = '%s/1' % val 220 if '/' in val: 221 task[id] += rangetolist(val, id) 222 elif val.isdigit() or val == '-1': 223 task[id].append(int(val)) 224 elif id == 'dow' and val[:3].lower() in daysofweek: 225 task[id].append(daysofweek(val[:3].lower())) 226 task['user'] = params[5] 227 task['cmd'] = params[6] 228 return task

229 230

231 -class cronlauncher(threading.Thread):

232

233 - def __init__(self, cmd, shell=True):

234 threading.Thread.__init__(self) 235 if platform.system() == 'Windows': 236 shell = False 237 self.cmd = cmd 238 self.shell = shell

239

240 - def run(self):

241 import subprocess 242 global _cron_subprocs 243 if isinstance(self.cmd, (list, tuple)): 244 cmd = self.cmd 245 else: 246 cmd = self.cmd.split() 247 proc = subprocess.Popen(cmd, 248 stdin=subprocess.PIPE, 249 stdout=subprocess.PIPE, 250 stderr=subprocess.PIPE, 251 shell=self.shell) 252 _cron_subprocs.append(proc) 253 (stdoutdata, stderrdata) = proc.communicate() 254 _cron_subprocs.remove(proc) 255 if proc.returncode != 0: 256 logger.warning( 257 'WEB2PY CRON Call returned code %s:\n%s' % 258 (proc.returncode, stdoutdata + stderrdata)) 259 else: 260 logger.debug('WEB2PY CRON Call returned success:\n%s' 261 % stdoutdata)

262 263

264 -def crondance(applications_parent, ctype='soft', startup=False, apps=None):

265 apppath = os.path.join(applications_parent, 'applications') 266 cron_path = os.path.join(applications_parent) 267 token = Token(cron_path) 268 cronmaster = token.acquire(startup=startup) 269 if not cronmaster: 270 return 271 now_s = time.localtime() 272 checks = (('min', now_s.tm_min), 273 ('hr', now_s.tm_hour), 274 ('mon', now_s.tm_mon), 275 ('dom', now_s.tm_mday), 276 ('dow', (now_s.tm_wday + 1) % 7)) 277 278 if apps is None: 279 apps = [x for x in os.listdir(apppath) 280 if os.path.isdir(os.path.join(apppath, x))] 281 282 full_apath_links = set() 283 284 for app in apps: 285 if _cron_stopping: 286 break 287 apath = os.path.join(apppath, app) 288 289 # if app is a symbolic link to other app, skip it 290 full_apath_link = absolute_path_link(apath) 291 if full_apath_link in full_apath_links: 292 continue 293 else: 294 full_apath_links.add(full_apath_link) 295 296 cronpath = os.path.join(apath, 'cron') 297 crontab = os.path.join(cronpath, 'crontab') 298 if not os.path.exists(crontab): 299 continue 300 try: 301 cronlines = fileutils.readlines_file(crontab, 'rt') 302 lines = [x.strip() for x in cronlines if x.strip( 303 ) and not x.strip().startswith('#')] 304 tasks = [parsecronline(cline) for cline in lines] 305 except Exception, e: 306 logger.error('WEB2PY CRON: crontab read error %s' % e) 307 continue 308 309 for task in tasks: 310 if _cron_stopping: 311 break 312 if sys.executable.lower().endswith('pythonservice.exe'): 313 _python_exe = os.path.join(sys.exec_prefix, 'python.exe') 314 else: 315 _python_exe = sys.executable 316 commands = [_python_exe] 317 w2p_path = fileutils.abspath('web2py.py', gluon=True) 318 if os.path.exists(w2p_path): 319 commands.append(w2p_path) 320 if global_settings.applications_parent != global_settings.gluon_parent: 321 commands.extend(('-f', global_settings.applications_parent)) 322 citems = [(k in task and not v in task[k]) for k, v in checks] 323 task_min = task.get('min', []) 324 if not task: 325 continue 326 elif not startup and task_min == [-1]: 327 continue 328 elif task_min != [-1] and reduce(lambda a, b: a or b, citems): 329 continue 330 logger.info('WEB2PY CRON (%s): %s executing %s in %s at %s' 331 % (ctype, app, task.get('cmd'), 332 os.getcwd(), datetime.datetime.now())) 333 action, command, models = False, task['cmd'], '' 334 if command.startswith('**'): 335 (action, models, command) = (True, '', command[2:]) 336 elif command.startswith('*'): 337 (action, models, command) = (True, '-M', command[1:]) 338 else: 339 action = False 340 341 if action and command.endswith('.py'): 342 commands.extend(('-J', # cron job 343 models, # import models? 344 '-S', app, # app name 345 '-a', '"<recycle>"', # password 346 '-R', command)) # command 347 elif action: 348 commands.extend(('-J', # cron job 349 models, # import models? 350 '-S', app + '/' + command, # app name 351 '-a', '"<recycle>"')) # password 352 else: 353 commands = command 354 355 # from python docs: 356 # You do not need shell=True to run a batch file or 357 # console-based executable. 358 shell = False 359 360 try: 361 cronlauncher(commands, shell=shell).start() 362 except Exception, e: 363 logger.warning( 364 'WEB2PY CRON: Execution error for %s: %s' 365 % (task.get('cmd'), e)) 366 token.release()

367

Source Code for Module gluon.newcron