1
2
3
4 """
5 This file is part of the web2py Web Framework
6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8
9 Functions required to execute app components
10 ============================================
11
12 FOR INTERNAL USE ONLY
13 """
14
15 import re
16 import fnmatch
17 import os
18 import copy
19 import random
20 import __builtin__
21 from gluon.storage import Storage, List
22 from gluon.template import parse_template
23 from gluon.restricted import restricted, compile2
24 from gluon.fileutils import mktree, listdir, read_file, write_file
25 from gluon.myregex import regex_expose
26 from gluon.languages import translator
27 from gluon.dal import BaseAdapter, SQLDB, SQLField, DAL, Field
28 from gluon.sqlhtml import SQLFORM, SQLTABLE
29 from gluon.cache import Cache
30 from gluon.globals import current, Response
31 from gluon import settings
32 from gluon.cfs import getcfs
33 from gluon import html
34 from gluon import validators
35 from gluon.http import HTTP, redirect
36 import marshal
37 import shutil
38 import imp
39 import logging
40 logger = logging.getLogger("web2py")
41 from gluon import rewrite
42 from custom_import import custom_import_install
43
44 try:
45 import py_compile
46 except:
47 logger.warning('unable to import py_compile')
48
49 is_pypy = settings.global_settings.is_pypy
50 is_gae = settings.global_settings.web2py_runtime_gae
51 is_jython = settings.global_settings.is_jython
52
53 pjoin = os.path.join
54
55 TEST_CODE = \
56 r"""
57 def _TEST():
58 import doctest, sys, cStringIO, types, cgi, gluon.fileutils
59 if not gluon.fileutils.check_credentials(request):
60 raise HTTP(401, web2py_error='invalid credentials')
61 stdout = sys.stdout
62 html = '<h2>Testing controller "%s.py" ... done.</h2><br/>\n' \
63 % request.controller
64 for key in sorted([key for key in globals() if not key in __symbols__+['_TEST']]):
65 eval_key = eval(key)
66 if type(eval_key) == types.FunctionType:
67 number_doctests = sum([len(ds.examples) for ds in doctest.DocTestFinder().find(eval_key)])
68 if number_doctests>0:
69 sys.stdout = cStringIO.StringIO()
70 name = '%s/controllers/%s.py in %s.__doc__' \
71 % (request.folder, request.controller, key)
72 doctest.run_docstring_examples(eval_key,
73 globals(), False, name=name)
74 report = sys.stdout.getvalue().strip()
75 if report:
76 pf = 'failed'
77 else:
78 pf = 'passed'
79 html += '<h3 class="%s">Function %s [%s]</h3>\n' \
80 % (pf, key, pf)
81 if report:
82 html += CODE(report, language='web2py', \
83 link='/examples/global/vars/').xml()
84 html += '<br/>\n'
85 else:
86 html += \
87 '<h3 class="nodoctests">Function %s [no doctests]</h3><br/>\n' \
88 % (key)
89 response._vars = html
90 sys.stdout = stdout
91 _TEST()
92 """
93
94 CACHED_REGEXES = {}
95 CACHED_REGEXES_MAX_SIZE = 1000
96
97
106
107
109 """
110 NOTE could simple use a dict and populate it,
111 NOTE not sure if this changes things though if monkey patching import.....
112 """
113
115 try:
116 return getattr(__builtin__, key)
117 except AttributeError:
118 raise KeyError(key)
119
121 setattr(self, key, value)
122
123
124 -def LOAD(c=None, f='index', args=None, vars=None,
125 extension=None, target=None, ajax=False, ajax_trap=False,
126 url=None, user_signature=False, timeout=None, times=1,
127 content='loading...', **attr):
128 """ LOAD a component into the action's document
129
130 Timing options:
131 -times: An integer or string ("infinity"/"continuous")
132 specifies how many times the component is requested
133 -timeout (milliseconds): specifies the time to wait before
134 starting the request or the frequency if times is greater than
135 1 or "infinity".
136 Timing options default to the normal behavior. The component
137 is added on page loading without delay.
138 """
139 from html import TAG, DIV, URL, SCRIPT, XML
140 if args is None:
141 args = []
142 vars = Storage(vars or {})
143 target = target or 'c' + str(random.random())[2:]
144 attr['_id'] = target
145 request = current.request
146 if '.' in f:
147 f, extension = f.rsplit('.', 1)
148 if url or ajax:
149 url = url or URL(request.application, c, f, r=request,
150 args=args, vars=vars, extension=extension,
151 user_signature=user_signature)
152
153 if isinstance(times, basestring):
154 if times.upper() in ("INFINITY", "CONTINUOUS"):
155 times = "Infinity"
156 else:
157 raise TypeError("Unsupported times argument %s" % times)
158 elif isinstance(times, int):
159 if times <= 0:
160 raise ValueError("Times argument must be greater than zero, 'Infinity' or None")
161 else:
162 raise TypeError("Unsupported times argument type %s" % type(times))
163 if timeout is not None:
164 if not isinstance(timeout, (int, long)):
165 raise ValueError("Timeout argument must be an integer or None")
166 elif timeout <= 0:
167 raise ValueError(
168 "Timeout argument must be greater than zero or None")
169 statement = "$.web2py.component('%s','%s', %s, %s);" \
170 % (url, target, timeout, times)
171 attr['_data-w2p_timeout'] = timeout
172 attr['_data-w2p_times'] = times
173 else:
174 statement = "$.web2py.component('%s','%s');" % (url, target)
175 attr['_data-w2p_remote'] = url
176 if not target is None:
177 return DIV(content, **attr)
178
179 else:
180 if not isinstance(args, (list, tuple)):
181 args = [args]
182 c = c or request.controller
183 other_request = Storage(request)
184 other_request['env'] = Storage(request.env)
185 other_request.controller = c
186 other_request.function = f
187 other_request.extension = extension or request.extension
188 other_request.args = List(args)
189 other_request.vars = vars
190 other_request.get_vars = vars
191 other_request.post_vars = Storage()
192 other_response = Response()
193 other_request.env.path_info = '/' + \
194 '/'.join([request.application, c, f] +
195 map(str, other_request.args))
196 other_request.env.query_string = \
197 vars and URL(vars=vars).split('?')[1] or ''
198 other_request.env.http_web2py_component_location = \
199 request.env.path_info
200 other_request.cid = target
201 other_request.env.http_web2py_component_element = target
202 other_response.view = '%s/%s.%s' % (c, f, other_request.extension)
203
204 other_environment = copy.copy(current.globalenv)
205
206 other_response._view_environment = other_environment
207 other_response.generic_patterns = \
208 copy.copy(current.response.generic_patterns)
209 other_environment['request'] = other_request
210 other_environment['response'] = other_response
211
212
213
214 original_request, current.request = current.request, other_request
215 original_response, current.response = current.response, other_response
216 page = run_controller_in(c, f, other_environment)
217 if isinstance(page, dict):
218 other_response._vars = page
219 other_response._view_environment.update(page)
220 run_view_in(other_response._view_environment)
221 page = other_response.body.getvalue()
222 current.request, current.response = original_request, original_response
223 js = None
224 if ajax_trap:
225 link = URL(request.application, c, f, r=request,
226 args=args, vars=vars, extension=extension,
227 user_signature=user_signature)
228 js = "$.web2py.trap_form('%s','%s');" % (link, target)
229 script = js and SCRIPT(js, _type="text/javascript") or ''
230 return TAG[''](DIV(XML(page), **attr), script)
231
232
234 """
235 Attention: this helper is new and experimental
236 """
238 self.environment = environment
239
240 - def __call__(self, c=None, f='index', args=None, vars=None,
241 extension=None, target=None, ajax=False, ajax_trap=False,
242 url=None, user_signature=False, content='loading...', **attr):
243 if args is None:
244 args = []
245 vars = Storage(vars or {})
246 import globals
247 target = target or 'c' + str(random.random())[2:]
248 attr['_id'] = target
249 request = self.environment['request']
250 if '.' in f:
251 f, extension = f.rsplit('.', 1)
252 if url or ajax:
253 url = url or html.URL(request.application, c, f, r=request,
254 args=args, vars=vars, extension=extension,
255 user_signature=user_signature)
256 script = html.SCRIPT('$.web2py.component("%s","%s")' % (url, target),
257 _type="text/javascript")
258 return html.TAG[''](script, html.DIV(content, **attr))
259 else:
260 if not isinstance(args, (list, tuple)):
261 args = [args]
262 c = c or request.controller
263
264 other_request = Storage(request)
265 other_request['env'] = Storage(request.env)
266 other_request.controller = c
267 other_request.function = f
268 other_request.extension = extension or request.extension
269 other_request.args = List(args)
270 other_request.vars = vars
271 other_request.get_vars = vars
272 other_request.post_vars = Storage()
273 other_response = globals.Response()
274 other_request.env.path_info = '/' + \
275 '/'.join([request.application, c, f] +
276 map(str, other_request.args))
277 other_request.env.query_string = \
278 vars and html.URL(vars=vars).split('?')[1] or ''
279 other_request.env.http_web2py_component_location = \
280 request.env.path_info
281 other_request.cid = target
282 other_request.env.http_web2py_component_element = target
283 other_response.view = '%s/%s.%s' % (c, f, other_request.extension)
284 other_environment = copy.copy(self.environment)
285 other_response._view_environment = other_environment
286 other_response.generic_patterns = \
287 copy.copy(current.response.generic_patterns)
288 other_environment['request'] = other_request
289 other_environment['response'] = other_response
290
291
292
293 original_request, current.request = current.request, other_request
294 original_response, current.response = current.response, other_response
295 page = run_controller_in(c, f, other_environment)
296 if isinstance(page, dict):
297 other_response._vars = page
298 other_response._view_environment.update(page)
299 run_view_in(other_response._view_environment)
300 page = other_response.body.getvalue()
301 current.request, current.response = original_request, original_response
302 js = None
303 if ajax_trap:
304 link = html.URL(request.application, c, f, r=request,
305 args=args, vars=vars, extension=extension,
306 user_signature=user_signature)
307 js = "$.web2py.trap_form('%s','%s');" % (link, target)
308 script = js and html.SCRIPT(js, _type="text/javascript") or ''
309 return html.TAG[''](html.DIV(html.XML(page), **attr), script)
310
311
313 """
314 In apps, instead of importing a local module
315 (in applications/app/modules) with::
316
317 import a.b.c as d
318
319 you should do::
320
321 d = local_import('a.b.c')
322
323 or (to force a reload):
324
325 d = local_import('a.b.c', reload=True)
326
327 This prevents conflict between applications and un-necessary execs.
328 It can be used to import any module, including regular Python modules.
329 """
330 items = name.replace('/', '.')
331 name = "applications.%s.modules.%s" % (app, items)
332 module = __import__(name)
333 for item in name.split(".")[1:]:
334 module = getattr(module, item)
335 if reload_force:
336 reload(module)
337 return module
338
339
340 """
341 OLD IMPLEMENTATION:
342 items = name.replace('/','.').split('.')
343 filename, modulepath = items[-1], pjoin(apath,'modules',*items[:-1])
344 imp.acquire_lock()
345 try:
346 file=None
347 (file,path,desc) = imp.find_module(filename,[modulepath]+sys.path)
348 if not path in sys.modules or reload:
349 if is_gae:
350 module={}
351 execfile(path,{},module)
352 module=Storage(module)
353 else:
354 module = imp.load_module(path,file,path,desc)
355 sys.modules[path] = module
356 else:
357 module = sys.modules[path]
358 except Exception, e:
359 module = None
360 if file:
361 file.close()
362 imp.release_lock()
363 if not module:
364 raise ImportError, "cannot find module %s in %s" % (
365 filename, modulepath)
366 return module
367 """
368
369 _base_environment_ = dict((k, getattr(html, k)) for k in html.__all__)
370 _base_environment_.update(
371 (k, getattr(validators, k)) for k in validators.__all__)
372 _base_environment_['__builtins__'] = __builtins__
373 _base_environment_['HTTP'] = HTTP
374 _base_environment_['redirect'] = redirect
375 _base_environment_['DAL'] = DAL
376 _base_environment_['Field'] = Field
377 _base_environment_['SQLDB'] = SQLDB
378 _base_environment_['SQLField'] = SQLField
379 _base_environment_['SQLFORM'] = SQLFORM
380 _base_environment_['SQLTABLE'] = SQLTABLE
381 _base_environment_['LOAD'] = LOAD
382
384 """
385 Build the environment dictionary into which web2py files are executed.
386 """
387
388 environment = dict(_base_environment_)
389
390 if not request.env:
391 request.env = Storage()
392
393
394 response.models_to_run = [
395 r'^\w+\.py$',
396 r'^%s/\w+\.py$' % request.controller,
397 r'^%s/%s/\w+\.py$' % (request.controller, request.function)
398 ]
399
400 t = environment['T'] = translator(os.path.join(request.folder,'languages'),
401 request.env.http_accept_language)
402 c = environment['cache'] = Cache(request)
403
404 if store_current:
405 current.globalenv = environment
406 current.request = request
407 current.response = response
408 current.session = session
409 current.T = t
410 current.cache = c
411
412 global __builtins__
413 if is_jython:
414 __builtins__ = mybuiltin()
415 elif is_pypy:
416 __builtins__ = mybuiltin()
417 else:
418 __builtins__['__import__'] = __builtin__.__import__
419 environment['request'] = request
420 environment['response'] = response
421 environment['session'] = session
422 environment['local_import'] = \
423 lambda name, reload=False, app=request.application:\
424 local_import_aux(name, reload, app)
425 BaseAdapter.set_folder(pjoin(request.folder, 'databases'))
426 response._view_environment = copy.copy(environment)
427 custom_import_install()
428 return environment
429
430
432 """
433 Bytecode compiles the file `filename`
434 """
435 py_compile.compile(filename)
436
437
439 """
440 Read the code inside a bytecode compiled file if the MAGIC number is
441 compatible
442
443 :returns: a code object
444 """
445 data = read_file(filename, 'rb')
446 if not is_gae and data[:4] != imp.get_magic():
447 raise SystemError('compiled code is incompatible')
448 return marshal.loads(data[8:])
449
450
452 """
453 Compiles all the views in the application specified by `folder`
454 """
455
456 path = pjoin(folder, 'views')
457 for fname in listdir(path, '^[\w/\-]+(\.\w+)*$'):
458 try:
459 data = parse_template(fname, path)
460 except Exception, e:
461 raise Exception("%s in %s" % (e, fname))
462 filename = 'views.%s.py' % fname.replace(os.path.sep, '.')
463 filename = pjoin(folder, 'compiled', filename)
464 write_file(filename, data)
465 save_pyc(filename)
466 os.unlink(filename)
467
468
470 """
471 Compiles all the models in the application specified by `folder`
472 """
473
474 path = pjoin(folder, 'models')
475 for fname in listdir(path, '.+\.py$'):
476 data = read_file(pjoin(path, fname))
477 modelfile = 'models.'+fname.replace(os.path.sep,'.')
478 filename = pjoin(folder, 'compiled', modelfile)
479 mktree(filename)
480 write_file(filename, data)
481 save_pyc(filename)
482 os.unlink(filename)
483
484
486 """
487 Compiles all the controllers in the application specified by `folder`
488 """
489
490 path = pjoin(folder, 'controllers')
491 for fname in listdir(path, '.+\.py$'):
492
493 data = read_file(pjoin(path, fname))
494 exposed = regex_expose.findall(data)
495 for function in exposed:
496 command = data + "\nresponse._vars=response._caller(%s)\n" % \
497 function
498 filename = pjoin(folder, 'compiled',
499 'controllers.%s.%s.py' % (fname[:-3],function))
500 write_file(filename, command)
501 save_pyc(filename)
502 os.unlink(filename)
503
506
508 """
509 Runs all models (in the app specified by the current folder)
510 It tries pre-compiled models first before compiling them.
511 """
512
513 folder = environment['request'].folder
514 c = environment['request'].controller
515 f = environment['request'].function
516 response = environment['response']
517
518 path = pjoin(folder, 'models')
519 cpath = pjoin(folder, 'compiled')
520 compiled = os.path.exists(cpath)
521 if compiled:
522 models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0),model_cmp)
523 else:
524 models = sorted(listdir(path, '^\w+\.py$', 0, sort=False),model_cmp)
525
526 models_to_run = None
527 for model in models:
528 if response.models_to_run != models_to_run:
529 regex = models_to_run = response.models_to_run
530 if isinstance(regex, list):
531 regex = re_compile('|'.join(regex))
532 if models_to_run:
533 if compiled:
534 n = len(cpath)+8
535 fname = model[n:-4].replace('.','/')+'.py'
536 else:
537 n = len(path)+1
538 fname = model[n:].replace(os.path.sep,'/')
539 if not regex.search(fname) and c != 'appadmin':
540 continue
541 elif compiled:
542 code = read_pyc(model)
543 elif is_gae:
544 code = getcfs(model, model,
545 lambda: compile2(read_file(model), model))
546 else:
547 code = getcfs(model, model, None)
548 restricted(code, environment, layer=model)
549
550
552 """
553 Runs the controller.function() (for the app specified by
554 the current folder).
555 It tries pre-compiled controller_function.pyc first before compiling it.
556 """
557
558
559 folder = environment['request'].folder
560 path = pjoin(folder, 'compiled')
561 badc = 'invalid controller (%s/%s)' % (controller, function)
562 badf = 'invalid function (%s/%s)' % (controller, function)
563 if os.path.exists(path):
564 filename = pjoin(path, 'controllers.%s.%s.pyc'
565 % (controller, function))
566 if not os.path.exists(filename):
567
568 filename = pjoin(path, 'controllers_%s_%s.pyc'
569 % (controller, function))
570
571 if not os.path.exists(filename):
572 raise HTTP(404,
573 rewrite.THREAD_LOCAL.routes.error_message % badf,
574 web2py_error=badf)
575 restricted(read_pyc(filename), environment, layer=filename)
576 elif function == '_TEST':
577
578 from settings import global_settings
579 from admin import abspath, add_path_first
580 paths = (global_settings.gluon_parent, abspath(
581 'site-packages', gluon=True), abspath('gluon', gluon=True), '')
582 [add_path_first(path) for path in paths]
583
584
585 filename = pjoin(folder, 'controllers/%s.py'
586 % controller)
587 if not os.path.exists(filename):
588 raise HTTP(404,
589 rewrite.THREAD_LOCAL.routes.error_message % badc,
590 web2py_error=badc)
591 environment['__symbols__'] = environment.keys()
592 code = read_file(filename)
593 code += TEST_CODE
594 restricted(code, environment, layer=filename)
595 else:
596 filename = pjoin(folder, 'controllers/%s.py'
597 % controller)
598 if not os.path.exists(filename):
599 raise HTTP(404,
600 rewrite.THREAD_LOCAL.routes.error_message % badc,
601 web2py_error=badc)
602 code = read_file(filename)
603 exposed = regex_expose.findall(code)
604 if not function in exposed:
605 raise HTTP(404,
606 rewrite.THREAD_LOCAL.routes.error_message % badf,
607 web2py_error=badf)
608 code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function)
609 if is_gae:
610 layer = filename + ':' + function
611 code = getcfs(layer, filename, lambda: compile2(code, layer))
612 restricted(code, environment, filename)
613 response = environment['response']
614 vars = response._vars
615 if response.postprocessing:
616 vars = reduce(lambda vars, p: p(vars), response.postprocessing, vars)
617 if isinstance(vars, unicode):
618 vars = vars.encode('utf8')
619 elif hasattr(vars, 'xml') and callable(vars.xml):
620 vars = vars.xml()
621 return vars
622
623
625 """
626 Executes the view for the requested action.
627 The view is the one specified in `response.view` or determined by the url
628 or `view/generic.extension`
629 It tries the pre-compiled views_controller_function.pyc before compiling it.
630 """
631 request = environment['request']
632 response = environment['response']
633 view = response.view
634 folder = request.folder
635 path = pjoin(folder, 'compiled')
636 badv = 'invalid view (%s)' % view
637 if response.generic_patterns:
638 patterns = response.generic_patterns
639 regex = re_compile('|'.join(map(fnmatch.translate, patterns)))
640 short_action = '%(controller)s/%(function)s.%(extension)s' % request
641 allow_generic = regex.search(short_action)
642 else:
643 allow_generic = False
644 if not isinstance(view, str):
645 ccode = parse_template(view, pjoin(folder, 'views'),
646 context=environment)
647 restricted(ccode, environment, 'file stream')
648 elif os.path.exists(path):
649 x = view.replace('/', '.')
650 files = ['views.%s.pyc' % x]
651 if allow_generic:
652 files.append('views.generic.%s.pyc' % request.extension)
653
654 x = view.replace('/', '_')
655 files.append('views_%s.pyc' % x)
656 if allow_generic:
657 files.append('views_generic.%s.pyc' % request.extension)
658 if request.extension == 'html':
659 files.append('views_%s.pyc' % x[:-5])
660 if allow_generic:
661 files.append('views_generic.pyc')
662
663 for f in files:
664 filename = pjoin(path, f)
665 if os.path.exists(filename):
666 code = read_pyc(filename)
667 restricted(code, environment, layer=filename)
668 return
669 raise HTTP(404,
670 rewrite.THREAD_LOCAL.routes.error_message % badv,
671 web2py_error=badv)
672 else:
673 filename = pjoin(folder, 'views', view)
674 if not os.path.exists(filename) and allow_generic:
675 view = 'generic.' + request.extension
676 filename = pjoin(folder, 'views', view)
677 if not os.path.exists(filename):
678 raise HTTP(404,
679 rewrite.THREAD_LOCAL.routes.error_message % badv,
680 web2py_error=badv)
681 layer = filename
682 if is_gae:
683 ccode = getcfs(layer, filename,
684 lambda: compile2(parse_template(view,
685 pjoin(folder, 'views'),
686 context=environment), layer))
687 else:
688 ccode = parse_template(view,
689 pjoin(folder, 'views'),
690 context=environment)
691 restricted(ccode, environment, layer)
692
693
695 """
696 Deletes the folder `compiled` containing the compiled application.
697 """
698 try:
699 shutil.rmtree(pjoin(folder, 'compiled'))
700 path = pjoin(folder, 'controllers')
701 for file in listdir(path, '.*\.pyc$', drop=False):
702 os.unlink(file)
703 except OSError:
704 pass
705
706
716
717
719 """
720 Example::
721
722 >>> import traceback, types
723 >>> environment={'x':1}
724 >>> open('a.py', 'w').write('print 1/x')
725 >>> save_pyc('a.py')
726 >>> os.unlink('a.py')
727 >>> if type(read_pyc('a.pyc'))==types.CodeType: print 'code'
728 code
729 >>> exec read_pyc('a.pyc') in environment
730 1
731 """
732
733 return
734
735
736 if __name__ == '__main__':
737 import doctest
738 doctest.testmod()
739