1
2
3
4 """
5 This file is part of the web2py Web Framework
6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8 """
9
10 import storage
11 import os
12 import re
13 import tarfile
14 import glob
15 import time
16 import datetime
17 import logging
18 from http import HTTP
19 from gzip import open as gzopen
20
21
22 __all__ = [
23 'parse_version',
24 'read_file',
25 'write_file',
26 'readlines_file',
27 'up',
28 'abspath',
29 'mktree',
30 'listdir',
31 'recursive_unlink',
32 'cleanpath',
33 'tar',
34 'untar',
35 'tar_compiled',
36 'get_session',
37 'check_credentials',
38 'w2p_pack',
39 'w2p_unpack',
40 'w2p_pack_plugin',
41 'w2p_unpack_plugin',
42 'fix_newlines',
43 'make_fake_file_like_object',
44 ]
45
46
47 -def parse_semantic(version="Version 1.99.0-rc.1+timestamp.2011.09.19.08.23.26"):
48 "http://semver.org/"
49 re_version = re.compile('(\d+)\.(\d+)\.(\d+)(\-(?P<pre>[^\s+]*))?(\+(?P<build>\S*))')
50 m = re_version.match(version.strip().split()[-1])
51 if not m:
52 return None
53 a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3))
54 pre_release = m.group('pre') or ''
55 build = m.group('build') or ''
56 if build.startswith('timestamp'):
57 build = datetime.datetime.strptime(build.split('.',1)[1], '%Y.%m.%d.%H.%M.%S')
58 return (a, b, c, pre_release, build)
59
60 -def parse_legacy(version="Version 1.99.0 (2011-09-19 08:23:26)"):
61 re_version = re.compile('[^\d]+ (\d+)\.(\d+)\.(\d+)\s*\((?P<datetime>.+?)\)\s*(?P<type>[a-z]+)?')
62 m = re_version.match(version)
63 a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)),
64 pre_release = m.group('type') or 'dev'
65 build = datetime.datetime.strptime(m.group('datetime'), '%Y-%m-%d %H:%M:%S')
66 return (a, b, c, pre_release, build)
67
69 version_tuple = parse_semantic(version)
70 if not version_tuple:
71 version_tuple = parse_legacy(version)
72 return version_tuple
73
75 "returns content from filename, making sure to close the file explicitly on exit."
76 f = open(filename, mode)
77 try:
78 return f.read()
79 finally:
80 f.close()
81
82
84 "writes <value> to filename, making sure to close the file explicitly on exit."
85 f = open(filename, mode)
86 try:
87 return f.write(value)
88 finally:
89 f.close()
90
91
93 "applies .split('\n') to the output of read_file()"
94 return read_file(filename, mode).split('\n')
95
96
104
105
106 -def listdir(
107 path,
108 expression='^.+$',
109 drop=True,
110 add_dirs=False,
111 sort=True,
112 ):
138
139
147
148
150 """
151 turns any expression/path into a valid filename. replaces / with _ and
152 removes special characters.
153 """
154
155 items = path.split('.')
156 if len(items) > 1:
157 path = re.sub('[^\w\.]+', '_', '_'.join(items[:-1]) + '.'
158 + ''.join(items[-1:]))
159 else:
160 path = re.sub('[^\w\.]+', '_', ''.join(items[-1:]))
161 return path
162
163
165 if not hasattr(tarfile.TarFile, 'extractall'):
166 from tarfile import ExtractError
167
168 class TarFile(tarfile.TarFile):
169
170 def extractall(self, path='.', members=None):
171 """Extract all members from the archive to the current working
172 directory and set owner, modification time and permissions on
173 directories afterwards. `path' specifies a different directory
174 to extract to. `members' is optional and must be a subset of the
175 list returned by getmembers().
176 """
177
178 directories = []
179 if members is None:
180 members = self
181 for tarinfo in members:
182 if tarinfo.isdir():
183
184
185
186
187 try:
188 os.makedirs(os.path.join(path,
189 tarinfo.name), 0777)
190 except EnvironmentError:
191 pass
192 directories.append(tarinfo)
193 else:
194 self.extract(tarinfo, path)
195
196
197
198 directories.sort(lambda a, b: cmp(a.name, b.name))
199 directories.reverse()
200
201
202
203 for tarinfo in directories:
204 path = os.path.join(path, tarinfo.name)
205 try:
206 self.chown(tarinfo, path)
207 self.utime(tarinfo, path)
208 self.chmod(tarinfo, path)
209 except ExtractError, e:
210 if self.errorlevel > 1:
211 raise
212 else:
213 self._dbg(1, 'tarfile: %s' % e)
214
215 _cls = TarFile
216 else:
217 _cls = tarfile.TarFile
218
219 tar = _cls(filename, 'r')
220 ret = tar.extractall(path, members)
221 tar.close()
222 return ret
223
224
225 -def tar(file, dir, expression='^.+$', filenames=None):
226 """
227 tars dir into file, only tars file that match expression
228 """
229
230 tar = tarfile.TarFile(file, 'w')
231 try:
232 if filenames is None:
233 filenames = listdir(dir, expression, add_dirs=True)
234 for file in filenames:
235 tar.add(os.path.join(dir, file), file, False)
236 finally:
237 tar.close()
238
240 """
241 untar file into dir
242 """
243
244 _extractall(file, dir)
245
246
247 -def w2p_pack(filename, path, compiled=False, filenames=None):
248 filename = abspath(filename)
249 path = abspath(path)
250 tarname = filename + '.tar'
251 if compiled:
252 tar_compiled(tarname, path, '^[\w\.\-]+$')
253 else:
254 tar(tarname, path, '^[\w\.\-]+$', filenames=filenames)
255 w2pfp = gzopen(filename, 'wb')
256 tarfp = open(tarname, 'rb')
257 w2pfp.write(tarfp.read())
258 w2pfp.close()
259 tarfp.close()
260 os.unlink(tarname)
261
263 if not os.path.exists('welcome.w2p') or os.path.exists('NEWINSTALL'):
264 try:
265 w2p_pack('welcome.w2p', 'applications/welcome')
266 os.unlink('NEWINSTALL')
267 logging.info("New installation: created welcome.w2p file")
268 except:
269 logging.error("New installation error: unable to create welcome.w2p file")
270
271
273
274 if filename=='welcome.w2p':
275 create_welcome_w2p()
276 filename = abspath(filename)
277 path = abspath(path)
278 if filename[-4:] == '.w2p' or filename[-3:] == '.gz':
279 if filename[-4:] == '.w2p':
280 tarname = filename[:-4] + '.tar'
281 else:
282 tarname = filename[:-3] + '.tar'
283 fgzipped = gzopen(filename, 'rb')
284 tarfile = open(tarname, 'wb')
285 tarfile.write(fgzipped.read())
286 tarfile.close()
287 fgzipped.close()
288 else:
289 tarname = filename
290 untar(tarname, path)
291 if delete_tar:
292 os.unlink(tarname)
293
294
296 """Pack the given plugin into a w2p file.
297 Will match files at:
298 <path>/*/plugin_[name].*
299 <path>/*/plugin_[name]/*
300 """
301 filename = abspath(filename)
302 path = abspath(path)
303 if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name):
304 raise Exception("Not a web2py plugin name")
305 plugin_tarball = tarfile.open(filename, 'w:gz')
306 try:
307 app_dir = path
308 while app_dir[-1] == '/':
309 app_dir = app_dir[:-1]
310 files1 = glob.glob(
311 os.path.join(app_dir, '*/plugin_%s.*' % plugin_name))
312 files2 = glob.glob(
313 os.path.join(app_dir, '*/plugin_%s/*' % plugin_name))
314 for file in files1 + files2:
315 plugin_tarball.add(file, arcname=file[len(app_dir) + 1:])
316 finally:
317 plugin_tarball.close()
318
319
326
327
329 """
330 used to tar a compiled application.
331 the content of models, views, controllers is not stored in the tar file.
332 """
333
334 tar = tarfile.TarFile(file, 'w')
335 for file in listdir(dir, expression, add_dirs=True):
336 filename = os.path.join(dir, file)
337 if os.path.islink(filename):
338 continue
339 if os.path.isfile(filename) and file[-4:] != '.pyc':
340 if file[:6] == 'models':
341 continue
342 if file[:5] == 'views':
343 continue
344 if file[:11] == 'controllers':
345 continue
346 if file[:7] == 'modules':
347 continue
348 tar.add(filename, file, False)
349 tar.close()
350
351
354
355
357 """ checks that user is authorized to access other_application"""
358 if request.application == other_application:
359 raise KeyError
360 try:
361 session_id = request.cookies['session_id_' + other_application].value
362 session_filename = os.path.join(
363 up(request.folder), other_application, 'sessions', session_id)
364 osession = storage.load_storage(session_filename)
365 except Exception, e:
366 osession = storage.Storage()
367 return osession
368
369 -def set_session(request, session, other_application='admin'):
370 """ checks that user is authorized to access other_application"""
371 if request.application == other_application:
372 raise KeyError
373 session_id = request.cookies['session_id_' + other_application].value
374 session_filename = os.path.join(
375 up(request.folder), other_application, 'sessions', session_id)
376 storage.save_storage(session,session_filename)
377
378 -def check_credentials(request, other_application='admin',
379 expiration=60 * 60, gae_login=True):
380 """ checks that user is authorized to access other_application"""
381 if request.env.web2py_runtime_gae:
382 from google.appengine.api import users
383 if users.is_current_user_admin():
384 return True
385 elif gae_login:
386 login_html = '<a href="%s">Sign in with your google account</a>.' \
387 % users.create_login_url(request.env.path_info)
388 raise HTTP(200, '<html><body>%s</body></html>' % login_html)
389 else:
390 return False
391 else:
392 t0 = time.time()
393 dt = t0 - expiration
394 s = get_session(request, other_application)
395 r = (s.authorized and s.last_time and s.last_time > dt)
396 if r:
397 s.last_time = t0
398 set_session(request,s,other_application)
399 return r
400
401
403 regex = re.compile(r'''(\r
404 |\r|
405 )''')
406 for filename in listdir(path, '.*\.(py|html)$', drop=False):
407 rdata = read_file(filename, 'rb')
408 wdata = regex.sub('\n', rdata)
409 if wdata != rdata:
410 write_file(filename, wdata, 'wb')
411
412
413 -def copystream(
414 src,
415 dest,
416 size,
417 chunk_size=10 ** 5,
418 ):
419 """
420 this is here because I think there is a bug in shutil.copyfileobj
421 """
422 while size > 0:
423 if size < chunk_size:
424 data = src.read(size)
425 else:
426 data = src.read(chunk_size)
427 length = len(data)
428 if length > size:
429 (data, length) = (data[:size], size)
430 size -= length
431 if length == 0:
432 break
433 dest.write(data)
434 if length < chunk_size:
435 break
436 dest.seek(0)
437 return
438
439
441 class LogFile(object):
442 def write(self, value):
443 pass
444
445 def close(self):
446 pass
447 return LogFile()
448
449
450 from settings import global_settings
451
452
453
463