From fijal at codespeak.net Thu Feb 1 12:28:48 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Feb 2007 12:28:48 +0100 (CET) Subject: [py-svn] r37720 - py/trunk/py/test/rsession Message-ID: <20070201112848.C085E10088@code0.codespeak.net> Author: fijal Date: Thu Feb 1 12:28:45 2007 New Revision: 37720 Modified: py/trunk/py/test/rsession/webjs.py Log: Kill dead import Modified: py/trunk/py/test/rsession/webjs.py ============================================================================== --- py/trunk/py/test/rsession/webjs.py (original) +++ py/trunk/py/test/rsession/webjs.py Thu Feb 1 12:28:45 2007 @@ -7,7 +7,6 @@ try: from pypy.translator.js.modules import dom from pypy.translator.js.helper import __show_traceback - from pypy.translator.transformer.debug import traceback_handler except ImportError: py.test.skip("PyPy not found") From fijal at codespeak.net Thu Feb 1 12:50:38 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Feb 2007 12:50:38 +0100 (CET) Subject: [py-svn] r37721 - py/trunk/py/test/rsession Message-ID: <20070201115038.BF36810071@code0.codespeak.net> Author: fijal Date: Thu Feb 1 12:50:29 2007 New Revision: 37721 Modified: py/trunk/py/test/rsession/executor.py Log: Fix. !Tests needed! Modified: py/trunk/py/test/rsession/executor.py ============================================================================== --- py/trunk/py/test/rsession/executor.py (original) +++ py/trunk/py/test/rsession/executor.py Thu Feb 1 12:50:29 2007 @@ -60,21 +60,19 @@ self.tracer = tracer return super(ApigenExecutor, self).execute() - def wrap_underlaying(self, target): - def f(*args): - try: - self.tracer.start_tracing() - return target(*args) - finally: - self.tracer.end_tracing() - return f + def wrap_underlaying(self, target, *args): + try: + self.tracer.start_tracing() + return target(*args) + finally: + self.tracer.end_tracing() def run(self): """ We want to trace *only* function objects here. Unsure what to do with custom collectors at all """ - if hasattr(self.item, 'obj') and type(self.item.obj) is py.test.Function: - self.item.obj = self.wrap_underlaying(self.item.obj) + if hasattr(self.item, 'obj') and type(self.item) is py.test.Function: + self.item.execute = self.wrap_underlaying self.item.run() class BoxExecutor(RunExecutor): From fijal at codespeak.net Thu Feb 1 12:59:48 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Feb 2007 12:59:48 +0100 (CET) Subject: [py-svn] r37723 - py/trunk/py/test/rsession/testing Message-ID: <20070201115948.BB13F10077@code0.codespeak.net> Author: fijal Date: Thu Feb 1 12:59:47 2007 New Revision: 37723 Modified: py/trunk/py/test/rsession/testing/test_executor.py Log: Add a test Modified: py/trunk/py/test/rsession/testing/test_executor.py ============================================================================== --- py/trunk/py/test/rsession/testing/test_executor.py (original) +++ py/trunk/py/test/rsession/testing/test_executor.py Thu Feb 1 12:59:47 2007 @@ -3,7 +3,7 @@ import example1 from py.__.test.rsession.executor import RunExecutor, BoxExecutor,\ - AsyncExecutor + AsyncExecutor, ApigenExecutor from py.__.test.rsession.outcome import ReprOutcome from py.__.test.rsession.testing.test_slave import funcprint_spec, \ funcprintfail_spec @@ -120,3 +120,51 @@ outcome = ReprOutcome(outcome_repr) assert not outcome.passed assert outcome.stdout.find("samfing elz") != -1 + +def test_apigen_executor(): + class Tracer(object): + def __init__(self): + self.starts = 0 + self.ends = 0 + + def start_tracing(self): + self.starts += 1 + + def end_tracing(self): + self.ends += 1 + + tmpdir = py.test.ensuretemp("apigen_executor") + tmpdir.ensure("__init__.py") + tmpdir.ensure("test_one.py").write(py.code.Source(""" + def g(): + pass + + def test_1(): + g() + + class TestX(object): + def setup_method(self, m): + self.ttt = 1 + + def test_one(self): + self.ttt += 1 + + def test_raise(self): + 1/0 + """)) + rootcol = py.test.collect.Directory(tmpdir) + tracer = Tracer() + item = rootcol.getitembynames("test_one.py/test_1") + ex = ApigenExecutor(item, config=config) + out1 = ex.execute(tracer) + item = rootcol.getitembynames("test_one.py/TestX/()/test_one") + ex = ApigenExecutor(item, config=config) + out2 = ex.execute(tracer) + item = rootcol.getitembynames("test_one.py/TestX/()/test_raise") + ex = ApigenExecutor(item, config=config) + out3 = ex.execute(tracer) + assert tracer.starts == 3 + assert tracer.ends == 3 + assert out1.passed + assert out2.passed + assert not out3.passed From guido at codespeak.net Thu Feb 1 14:56:33 2007 From: guido at codespeak.net (guido at codespeak.net) Date: Thu, 1 Feb 2007 14:56:33 +0100 (CET) Subject: [py-svn] r37729 - in py/trunk/py: apigen io io/test test/rsession Message-ID: <20070201135633.7A2E210070@code0.codespeak.net> Author: guido Date: Thu Feb 1 14:56:31 2007 New Revision: 37729 Modified: py/trunk/py/apigen/apigen.py py/trunk/py/io/capture.py py/trunk/py/io/test/test_capture.py py/trunk/py/test/rsession/rsession.py Log: Added some code to py.io.FDCapture and py.io.OutErrCapture to allow writing to the original (patched) file descriptor. Also made that the capturing object is passed to apigen.py's build() function (from rsession.py), which uses the new methods to print progress information. Modified: py/trunk/py/apigen/apigen.py ============================================================================== --- py/trunk/py/apigen/apigen.py (original) +++ py/trunk/py/apigen/apigen.py Thu Feb 1 14:56:31 2007 @@ -17,7 +17,7 @@ rootmod = __import__(pkgdir.basename) return 'py', pkg_to_dict(rootmod) -def build(pkgdir, dsa): +def build(pkgdir, dsa, capture): l = linker.Linker() proj = project.Project() @@ -32,16 +32,25 @@ apb = htmlgen.ApiPageBuilder(targetdir, l, dsa, pkgdir, namespace_tree) spb = htmlgen.SourcePageBuilder(targetdir, l, pkgdir) + capture.writeorgerr('preparing namespace pages\n') ns_data = apb.prepare_namespace_pages() + capture.writeorgerr('preparing class pages\n') class_names = dsa.get_class_names() class_data = apb.prepare_class_pages(class_names) + capture.writeorgerr('preparing function pages\n') function_names = dsa.get_function_names() func_data = apb.prepare_function_pages(function_names) + capture.writeorgerr('preparing source pages\n') source_data = spb.prepare_pages(pkgdir) + capture.writeorgerr('building namespace pages\n') apb.build_namespace_pages(ns_data, proj) + capture.writeorgerr('building class pages\n') apb.build_class_pages(class_data, proj) - #apb.build_method_pages(method_data, proj) + apb.build_method_pages(method_data, proj) + capture.writeorgerr('building function pages\n') apb.build_function_pages(func_data, proj) + capture.writeorgerr('building source pages\n') spb.build_pages(source_data, proj, pkgdir) + capture.writeorgerr('done building documentation\n') Modified: py/trunk/py/io/capture.py ============================================================================== --- py/trunk/py/io/capture.py (original) +++ py/trunk/py/io/capture.py Thu Feb 1 14:56:31 2007 @@ -1,9 +1,12 @@ -import os, sys +import os +import sys +import thread import py class FDCapture: """ Capture IO to/from a given os-level filedescriptor. """ + def __init__(self, targetfd, tmpfile=None): self.targetfd = targetfd if tmpfile is None: @@ -14,16 +17,22 @@ self._patched = [] def setasfile(self, name, module=sys): + """ patch . to self.tmpfile + """ key = (module, name) self._patched.append((key, getattr(module, name))) setattr(module, name, self.tmpfile) def unsetfiles(self): + """ unpatch all patched items + """ while self._patched: (module, name), value = self._patched.pop() setattr(module, name, value) def done(self): + """ unpatch and clean up, returns the self.tmpfile (file object) + """ os.dup2(self._savefd, self.targetfd) self.unsetfiles() os.close(self._savefd) @@ -31,11 +40,23 @@ return self.tmpfile def maketmpfile(self): + """ create a temporary file + """ f = os.tmpfile() newf = py.io.dupfile(f) f.close() return newf + def writeorg(self, str): + """ write a string to the original file descriptor + """ + tempfp = os.tmpfile() + try: + os.dup2(self._savefd, tempfp.fileno()) + tempfp.write(str) + finally: + tempfp.close() + class OutErrCapture: """ capture Stdout and Stderr both on filedescriptor and sys.stdout/stderr level. @@ -51,6 +72,11 @@ self.err.setasfile('stderr') def reset(self): + """ reset sys.stdout and sys.stderr + + returns a tuple of file objects (out, err) for the captured + data + """ out = err = "" if hasattr(self, 'out'): outfile = self.out.done() @@ -60,6 +86,20 @@ err = errfile.read() return out, err + def writeorgout(self, str): + """ write something to the original stdout + """ + if not hasattr(self, 'out'): + raise IOError('stdout not patched') + self.out.writeorg(str) + + def writeorgerr(self, str): + """ write something to the original stderr + """ + if not hasattr(self, 'err'): + raise IOError('stderr not patched') + self.err.writeorg(str) + def callcapture(func, *args, **kwargs): """ call the given function with args/kwargs and return a (res, out, err) tuple where Modified: py/trunk/py/io/test/test_capture.py ============================================================================== --- py/trunk/py/io/test/test_capture.py (original) +++ py/trunk/py/io/test/test_capture.py Thu Feb 1 14:56:31 2007 @@ -30,6 +30,33 @@ f = cap.done() assert x == "3" + def test_writeorg(self): + tmppath = py.test.ensuretemp('test_writeorg').ensure('stderr', + file=True) + tmpfp = tmppath.open('w+b') + try: + cap = py.io.FDCapture(tmpfp.fileno()) + print >>tmpfp, 'foo' + cap.writeorg('bar\n') + finally: + tmpfp.close() + f = cap.done() + scap = f.read() + assert scap == 'foo\n' + stmp = tmppath.read() + assert stmp == "bar\n" + + def test_writeorg_wrongtype(self): + tmppath = py.test.ensuretemp('test_writeorg').ensure('stdout', + file=True) + tmpfp = tmppath.open('r') + try: + cap = py.io.FDCapture(tmpfp.fileno()) + py.test.raises(IOError, "cap.writeorg('bar\\n')") + finally: + tmpfp.close() + f = cap.done() + class TestCapturing: def getcapture(self): return py.io.OutErrCapture() Modified: py/trunk/py/test/rsession/rsession.py ============================================================================== --- py/trunk/py/test/rsession/rsession.py (original) +++ py/trunk/py/test/rsession/rsession.py Thu Feb 1 14:56:31 2007 @@ -263,7 +263,8 @@ try: pkgdir = self.getpkgdir(self.config.args[0]) apigen.build(pkgdir, - DocStorageAccessor(self.docstorage)) + DocStorageAccessor(self.docstorage), + capture) finally: capture.reset() From guido at codespeak.net Thu Feb 1 15:14:06 2007 From: guido at codespeak.net (guido at codespeak.net) Date: Thu, 1 Feb 2007 15:14:06 +0100 (CET) Subject: [py-svn] r37733 - in py/trunk/py: apigen io Message-ID: <20070201141406.1F26D10084@code0.codespeak.net> Author: guido Date: Thu Feb 1 15:13:59 2007 New Revision: 37733 Modified: py/trunk/py/apigen/apigen.py py/trunk/py/io/capture.py Log: Removed two rather useless methods that just delegated to methods on the underlying object, in favour of calling the underlying object's methods directly (py.io.OutErrCapture.writeorg*). Modified: py/trunk/py/apigen/apigen.py ============================================================================== --- py/trunk/py/apigen/apigen.py (original) +++ py/trunk/py/apigen/apigen.py Thu Feb 1 15:13:59 2007 @@ -32,25 +32,25 @@ apb = htmlgen.ApiPageBuilder(targetdir, l, dsa, pkgdir, namespace_tree) spb = htmlgen.SourcePageBuilder(targetdir, l, pkgdir) - capture.writeorgerr('preparing namespace pages\n') + capture.err.writeorg('preparing namespace pages\n') ns_data = apb.prepare_namespace_pages() - capture.writeorgerr('preparing class pages\n') + capture.err.writeorg('preparing class pages\n') class_names = dsa.get_class_names() class_data = apb.prepare_class_pages(class_names) - capture.writeorgerr('preparing function pages\n') + capture.err.writeorg('preparing function pages\n') function_names = dsa.get_function_names() func_data = apb.prepare_function_pages(function_names) - capture.writeorgerr('preparing source pages\n') + capture.err.writeorg('preparing source pages\n') source_data = spb.prepare_pages(pkgdir) - capture.writeorgerr('building namespace pages\n') + capture.err.writeorg('building namespace pages\n') apb.build_namespace_pages(ns_data, proj) - capture.writeorgerr('building class pages\n') + capture.err.writeorg('building class pages\n') apb.build_class_pages(class_data, proj) apb.build_method_pages(method_data, proj) - capture.writeorgerr('building function pages\n') + capture.err.writeorg('building function pages\n') apb.build_function_pages(func_data, proj) - capture.writeorgerr('building source pages\n') + capture.err.writeorg('building source pages\n') spb.build_pages(source_data, proj, pkgdir) - capture.writeorgerr('done building documentation\n') + capture.err.writeorg('done building documentation\n') Modified: py/trunk/py/io/capture.py ============================================================================== --- py/trunk/py/io/capture.py (original) +++ py/trunk/py/io/capture.py Thu Feb 1 15:13:59 2007 @@ -86,20 +86,6 @@ err = errfile.read() return out, err - def writeorgout(self, str): - """ write something to the original stdout - """ - if not hasattr(self, 'out'): - raise IOError('stdout not patched') - self.out.writeorg(str) - - def writeorgerr(self, str): - """ write something to the original stderr - """ - if not hasattr(self, 'err'): - raise IOError('stderr not patched') - self.err.writeorg(str) - def callcapture(func, *args, **kwargs): """ call the given function with args/kwargs and return a (res, out, err) tuple where From hpk at codespeak.net Thu Feb 1 15:35:07 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 1 Feb 2007 15:35:07 +0100 (CET) Subject: [py-svn] r37736 - py/trunk/py/io Message-ID: <20070201143507.E72E810097@code0.codespeak.net> Author: hpk Date: Thu Feb 1 15:35:07 2007 New Revision: 37736 Modified: py/trunk/py/io/dupfile.py Log: cosmetic docstring change Modified: py/trunk/py/io/dupfile.py ============================================================================== --- py/trunk/py/io/dupfile.py (original) +++ py/trunk/py/io/dupfile.py Thu Feb 1 15:35:07 2007 @@ -4,8 +4,8 @@ def dupfile(f, mode=None, buffering=0, raising=False): """ return a new open file object that's a duplicate of f - mode is duplicated if not given, buffering controls - buffer size (defaulting to no buffering) and raising + mode is duplicated if not given, 'buffering' controls + buffer size (defaulting to no buffering) and 'raising' defines whether an exception is raised when an incompatible file object is passed in (if raising is False, the file object itself will be returned) From guido at codespeak.net Thu Feb 1 15:55:24 2007 From: guido at codespeak.net (guido at codespeak.net) Date: Thu, 1 Feb 2007 15:55:24 +0100 (CET) Subject: [py-svn] r37738 - py/trunk/py/apigen Message-ID: <20070201145524.4B5F510088@code0.codespeak.net> Author: guido Date: Thu Feb 1 15:55:23 2007 New Revision: 37738 Modified: py/trunk/py/apigen/apigen.py Log: Fixed problem probably caused by removing some comment or something... Modified: py/trunk/py/apigen/apigen.py ============================================================================== --- py/trunk/py/apigen/apigen.py (original) +++ py/trunk/py/apigen/apigen.py Thu Feb 1 15:55:23 2007 @@ -47,7 +47,6 @@ apb.build_namespace_pages(ns_data, proj) capture.err.writeorg('building class pages\n') apb.build_class_pages(class_data, proj) - apb.build_method_pages(method_data, proj) capture.err.writeorg('building function pages\n') apb.build_function_pages(func_data, proj) capture.err.writeorg('building source pages\n') From guido at codespeak.net Thu Feb 1 15:57:36 2007 From: guido at codespeak.net (guido at codespeak.net) Date: Thu, 1 Feb 2007 15:57:36 +0100 (CET) Subject: [py-svn] r37739 - in py/trunk/py/apigen/source: . testing Message-ID: <20070201145736.3D7821008B@code0.codespeak.net> Author: guido Date: Thu Feb 1 15:57:34 2007 New Revision: 37739 Modified: py/trunk/py/apigen/source/html.py py/trunk/py/apigen/source/testing/test_html.py Log: Made that only the first two lines of a source file are examined for the encoding. Modified: py/trunk/py/apigen/source/html.py ============================================================================== --- py/trunk/py/apigen/source/html.py (original) +++ py/trunk/py/apigen/source/html.py Thu Feb 1 15:57:34 2007 @@ -261,8 +261,15 @@ if path[-1] in ['c', 'o']: path = path[:-1] fpath = py.path.local(path) - code = fpath.read() - match = _reg_enc.search(code) + fp = fpath.open() + lines = [] + try: + # encoding is only allowed in the first two lines + for i in range(2): + lines.append(fp.readline()) + finally: + fp.close() + match = _reg_enc.search('\n'.join(lines)) if match: return match.group(1) return 'ISO-8859-1' Modified: py/trunk/py/apigen/source/testing/test_html.py ============================================================================== --- py/trunk/py/apigen/source/testing/test_html.py (original) +++ py/trunk/py/apigen/source/testing/test_html.py Thu Feb 1 15:57:34 2007 @@ -177,3 +177,14 @@ """))) assert get_module_encoding(fpath.strpath) == 'UTF-8' +def test_get_encoding_matching_pattern_elsewhere(): + temp = py.test.ensuretemp('test_get_encoding') + fpath = temp.join('matching_pattern.py') + fpath.write(str(py.code.Source("""\ + #!/usr/bin/env python + + def foo(coding=None): + pass + """))) + assert get_module_encoding(fpath.strpath) == 'ISO-8859-1' + From hpk at codespeak.net Thu Feb 1 16:20:45 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 1 Feb 2007 16:20:45 +0100 (CET) Subject: [py-svn] r37741 - in py/trunk/py: . doc io io/test log/testing misc misc/testing test test/rsession test/rsession/testing test/testing thread/testing Message-ID: <20070201152045.A98601009A@code0.codespeak.net> Author: hpk Date: Thu Feb 1 16:20:39 2007 New Revision: 37741 Added: py/trunk/py/doc/io.txt (contents, props changed) py/trunk/py/io/fdcapture.py - copied, changed from r37733, py/trunk/py/io/capture.py py/trunk/py/io/stdcapture.py - copied, changed from r37733, py/trunk/py/io/capture.py py/trunk/py/io/test/test_simplecapture.py - copied, changed from r37733, py/trunk/py/misc/testing/test_simplecapture.py Removed: py/trunk/py/io/capture.py py/trunk/py/misc/capture.py py/trunk/py/misc/simplecapture.py py/trunk/py/misc/testing/test_simplecapture.py py/trunk/py/test/todo-cleanup.txt Modified: py/trunk/py/__init__.py py/trunk/py/doc/TODO.txt py/trunk/py/doc/index.txt py/trunk/py/io/test/test_capture.py py/trunk/py/log/testing/test_log.py py/trunk/py/test/collect.py py/trunk/py/test/item.py py/trunk/py/test/rsession/local.py py/trunk/py/test/rsession/rsession.py py/trunk/py/test/rsession/testing/test_reporter.py py/trunk/py/test/testing/test_session.py py/trunk/py/thread/testing/test_pool.py Log: monster checking for * unifying IO capturing methods * py.io.StdCapture and py.io.StdCaptureFD (and both have a classmethod 'call' that is a shortcut for capturing output while executing a function) * removing lots of duplicate code * providing some examples in py/doc/io.txt at least tests on win32 and linux seem to pass all for me. Modified: py/trunk/py/__init__.py ============================================================================== --- py/trunk/py/__init__.py (original) +++ py/trunk/py/__init__.py Thu Feb 1 16:20:39 2007 @@ -16,7 +16,7 @@ download_url = "http://codespeak.net/download/py/%s.tar.gz" %(version,), license = "MIT license", platforms = ['unix', 'linux', 'cygwin'], - author = "holger krekel & others", + author = "holger krekel, Armin Rigo, Guido Wesdorp, Maciej Fijalkowski & others", author_email = "py-dev at codespeak.net", long_description = globals()['__doc__'], @@ -94,9 +94,9 @@ # input-output helping 'io.dupfile' : ('./io/dupfile.py', 'dupfile'), - 'io.FDCapture' : ('./io/capture.py', 'FDCapture'), - 'io.OutErrCapture' : ('./io/capture.py', 'OutErrCapture'), - 'io.callcapture' : ('./io/capture.py', 'callcapture'), + 'io.FDCapture' : ('./io/fdcapture.py', 'FDCapture'), + 'io.StdCapture' : ('./io/stdcapture.py', 'StdCapture'), + 'io.StdCaptureFD' : ('./io/stdcapture.py', 'StdCaptureFD'), # error module, defining all errno's as Classes 'error' : ('./misc/error.py', 'error'), Modified: py/trunk/py/doc/TODO.txt ============================================================================== --- py/trunk/py/doc/TODO.txt (original) +++ py/trunk/py/doc/TODO.txt Thu Feb 1 16:20:39 2007 @@ -97,7 +97,7 @@ os.fork to work)) * see why startcapture() used to not use FD-based - "py.io.OutErrCapture" to isolate standard output. + "py.io.StdCaptureFD" to isolate standard output. use that check if all py and PyPy tests pass as good as they do without. Modified: py/trunk/py/doc/index.txt ============================================================================== --- py/trunk/py/doc/index.txt (original) +++ py/trunk/py/doc/index.txt Thu Feb 1 16:20:39 2007 @@ -11,7 +11,9 @@ `py.code`_: High-level access/manipulation of Python code and traceback objects. -`py.xml`_ a fast'n'easy way to generate xml/html documents (including CSS-styling) +`py.xml`_ for generating in-memory xml/html object trees + +`py.io`_ Helper Classes for Capturing of Input/Output `py.log`_ an alpha document about the ad-hoc logging facilities Added: py/trunk/py/doc/io.txt ============================================================================== --- (empty file) +++ py/trunk/py/doc/io.txt Thu Feb 1 16:20:39 2007 @@ -0,0 +1,45 @@ +======= +py.io +======= + +.. contents:: +.. sectnum:: + +The 'py' lib provides helper classes for capturing IO during +execution of a program. + +IO Capturing examples +=============================================== + +:api:`py.io.StdCapture` +--------------------------- + +Basic Example: + + >>> import py + >>> capture = py.io.StdCapture() + >>> print "hello" + >>> out,err = capture.reset() + >>> out.strip() == "hello" + True + +For calling functions you may use a shortcut: + >>> import py + >>> def f(): print "hello" + >>> res, out, err = py.io.StdCapture.call(f) + >>> out.strip() == "hello" + True + +:api:`py.io.StdCaptureFD` +--------------------------- + +If you also want to capture writes to the stdout/stdin +filedescriptors you may invoke: + + >>> import py, sys + >>> capture = py.io.StdCaptureFD() + >>> sys.stdout.write("hello") + >>> sys.stderr.write("world") + >>> out,err = capture.reset() + >>> out.strip() + err.strip() == "helloworld" + True Deleted: /py/trunk/py/io/capture.py ============================================================================== --- /py/trunk/py/io/capture.py Thu Feb 1 16:20:39 2007 +++ (empty file) @@ -1,101 +0,0 @@ - -import os -import sys -import thread -import py - -class FDCapture: - """ Capture IO to/from a given os-level filedescriptor. """ - - def __init__(self, targetfd, tmpfile=None): - self.targetfd = targetfd - if tmpfile is None: - tmpfile = self.maketmpfile() - self.tmpfile = tmpfile - self._savefd = os.dup(targetfd) - os.dup2(self.tmpfile.fileno(), targetfd) - self._patched = [] - - def setasfile(self, name, module=sys): - """ patch . to self.tmpfile - """ - key = (module, name) - self._patched.append((key, getattr(module, name))) - setattr(module, name, self.tmpfile) - - def unsetfiles(self): - """ unpatch all patched items - """ - while self._patched: - (module, name), value = self._patched.pop() - setattr(module, name, value) - - def done(self): - """ unpatch and clean up, returns the self.tmpfile (file object) - """ - os.dup2(self._savefd, self.targetfd) - self.unsetfiles() - os.close(self._savefd) - self.tmpfile.seek(0) - return self.tmpfile - - def maketmpfile(self): - """ create a temporary file - """ - f = os.tmpfile() - newf = py.io.dupfile(f) - f.close() - return newf - - def writeorg(self, str): - """ write a string to the original file descriptor - """ - tempfp = os.tmpfile() - try: - os.dup2(self._savefd, tempfp.fileno()) - tempfp.write(str) - finally: - tempfp.close() - -class OutErrCapture: - """ capture Stdout and Stderr both on filedescriptor - and sys.stdout/stderr level. - """ - def __init__(self, out=True, err=True, patchsys=True): - if out: - self.out = FDCapture(1) - if patchsys: - self.out.setasfile('stdout') - if err: - self.err = FDCapture(2) - if patchsys: - self.err.setasfile('stderr') - - def reset(self): - """ reset sys.stdout and sys.stderr - - returns a tuple of file objects (out, err) for the captured - data - """ - out = err = "" - if hasattr(self, 'out'): - outfile = self.out.done() - out = outfile.read() - if hasattr(self, 'err'): - errfile = self.err.done() - err = errfile.read() - return out, err - -def callcapture(func, *args, **kwargs): - """ call the given function with args/kwargs - and return a (res, out, err) tuple where - out and err represent the output/error output - during function execution. - """ - so = OutErrCapture() - try: - res = func(*args, **kwargs) - finally: - out, err = so.reset() - return res, out, err - Copied: py/trunk/py/io/fdcapture.py (from r37733, py/trunk/py/io/capture.py) ============================================================================== --- py/trunk/py/io/capture.py (original) +++ py/trunk/py/io/fdcapture.py Thu Feb 1 16:20:39 2007 @@ -57,45 +57,3 @@ finally: tempfp.close() -class OutErrCapture: - """ capture Stdout and Stderr both on filedescriptor - and sys.stdout/stderr level. - """ - def __init__(self, out=True, err=True, patchsys=True): - if out: - self.out = FDCapture(1) - if patchsys: - self.out.setasfile('stdout') - if err: - self.err = FDCapture(2) - if patchsys: - self.err.setasfile('stderr') - - def reset(self): - """ reset sys.stdout and sys.stderr - - returns a tuple of file objects (out, err) for the captured - data - """ - out = err = "" - if hasattr(self, 'out'): - outfile = self.out.done() - out = outfile.read() - if hasattr(self, 'err'): - errfile = self.err.done() - err = errfile.read() - return out, err - -def callcapture(func, *args, **kwargs): - """ call the given function with args/kwargs - and return a (res, out, err) tuple where - out and err represent the output/error output - during function execution. - """ - so = OutErrCapture() - try: - res = func(*args, **kwargs) - finally: - out, err = so.reset() - return res, out, err - Copied: py/trunk/py/io/stdcapture.py (from r37733, py/trunk/py/io/capture.py) ============================================================================== --- py/trunk/py/io/capture.py (original) +++ py/trunk/py/io/stdcapture.py Thu Feb 1 16:20:39 2007 @@ -1,73 +1,36 @@ - import os import sys -import thread import py +try: from cStringIO import StringIO +except ImportError: from StringIO import StringIO -class FDCapture: - """ Capture IO to/from a given os-level filedescriptor. """ - - def __init__(self, targetfd, tmpfile=None): - self.targetfd = targetfd - if tmpfile is None: - tmpfile = self.maketmpfile() - self.tmpfile = tmpfile - self._savefd = os.dup(targetfd) - os.dup2(self.tmpfile.fileno(), targetfd) - self._patched = [] - - def setasfile(self, name, module=sys): - """ patch . to self.tmpfile - """ - key = (module, name) - self._patched.append((key, getattr(module, name))) - setattr(module, name, self.tmpfile) - - def unsetfiles(self): - """ unpatch all patched items - """ - while self._patched: - (module, name), value = self._patched.pop() - setattr(module, name, value) - - def done(self): - """ unpatch and clean up, returns the self.tmpfile (file object) - """ - os.dup2(self._savefd, self.targetfd) - self.unsetfiles() - os.close(self._savefd) - self.tmpfile.seek(0) - return self.tmpfile - - def maketmpfile(self): - """ create a temporary file - """ - f = os.tmpfile() - newf = py.io.dupfile(f) - f.close() - return newf - - def writeorg(self, str): - """ write a string to the original file descriptor - """ - tempfp = os.tmpfile() - try: - os.dup2(self._savefd, tempfp.fileno()) - tempfp.write(str) - finally: - tempfp.close() +class Capture(object): + def call(cls, func, *args, **kwargs): + """ return a (res, out, err) tuple where + out and err represent the output/error output + during function execution. + call the given function with args/kwargs + and capture output/error during its execution. + """ + so = cls() + try: + res = func(*args, **kwargs) + finally: + out, err = so.reset() + return res, out, err + call = classmethod(call) -class OutErrCapture: +class StdCaptureFD(Capture): """ capture Stdout and Stderr both on filedescriptor and sys.stdout/stderr level. """ def __init__(self, out=True, err=True, patchsys=True): if out: - self.out = FDCapture(1) + self.out = py.io.FDCapture(1) if patchsys: self.out.setasfile('stdout') if err: - self.err = FDCapture(2) + self.err = py.io.FDCapture(2) if patchsys: self.err.setasfile('stderr') @@ -86,16 +49,44 @@ err = errfile.read() return out, err -def callcapture(func, *args, **kwargs): - """ call the given function with args/kwargs - and return a (res, out, err) tuple where - out and err represent the output/error output - during function execution. - """ - so = OutErrCapture() - try: - res = func(*args, **kwargs) - finally: - out, err = so.reset() - return res, out, err +class StdCapture(Capture): + """ capture sys.stdout/sys.stderr (but not system level fd 1 and 2). + this captures only "In-Memory" and is currently intended to be + used by the unittest package to capture print-statements in tests. + """ + def __init__(self): + self.oldin = sys.stdin + self.oldout = sys.stdout + self.olderr = sys.stderr + sys.stdin = self.newin = DontReadFromInput() + sys.stdout = self.newout = StringIO() + sys.stderr = self.newerr = StringIO() + + def reset(self): + """ return captured output and restore sys.stdout/err.""" + x, y = self.done() + return x.read(), y.read() + + def done(self): + o,e = sys.stdout, sys.stderr + sys.stdin, sys.stdout, sys.stderr = ( + self.oldin, self.oldout, self.olderr) + del self.oldin, self.oldout, self.olderr + o, e = self.newout, self.newerr + o.seek(0) + e.seek(0) + return o,e + +class DontReadFromInput: + """Temporary stub class. Ideally when stdin is accessed, the + capturing should be turned off, with possibly all data captured + so far sent to the screen. This should be configurable, though, + because in automated test runs it is better to crash than + hang indefinitely. + """ + def read(self, *args): + raise IOError("reading from stdin while output is captured") + readline = read + readlines = read + __iter__ = read Modified: py/trunk/py/io/test/test_capture.py ============================================================================== --- py/trunk/py/io/test/test_capture.py (original) +++ py/trunk/py/io/test/test_capture.py Thu Feb 1 16:20:39 2007 @@ -59,7 +59,7 @@ class TestCapturing: def getcapture(self): - return py.io.OutErrCapture() + return py.io.StdCaptureFD() def test_capturing_simple(self): cap = self.getcapture() @@ -120,13 +120,13 @@ print >>py.std.sys.stderr, y return 42 - res, out, err = py.io.callcapture(func, 3, y=4) + res, out, err = py.io.StdCaptureFD.call(func, 3, y=4) assert res == 42 assert out.startswith("3") assert err.startswith("4") def test_just_out_capture(): - cap = py.io.OutErrCapture(out=True, err=False) + cap = py.io.StdCaptureFD(out=True, err=False) print >>sys.stdout, "hello" print >>sys.stderr, "world" out, err = cap.reset() @@ -134,7 +134,7 @@ assert not err def test_just_err_capture(): - cap = py.io.OutErrCapture(out=False, err=True) + cap = py.io.StdCaptureFD(out=False, err=True) print >>sys.stdout, "hello" print >>sys.stderr, "world" out, err = cap.reset() @@ -142,7 +142,7 @@ assert not out def test_capture_no_sys(): - cap = py.io.OutErrCapture(patchsys=False) + cap = py.io.StdCaptureFD(patchsys=False) print >>sys.stdout, "hello" print >>sys.stderr, "world" os.write(1, "1") Copied: py/trunk/py/io/test/test_simplecapture.py (from r37733, py/trunk/py/misc/testing/test_simplecapture.py) ============================================================================== --- py/trunk/py/misc/testing/test_simplecapture.py (original) +++ py/trunk/py/io/test/test_simplecapture.py Thu Feb 1 16:20:39 2007 @@ -1,29 +1,10 @@ import os, sys import py -from py.__.misc.simplecapture import SimpleOutErrCapture, callcapture -from py.__.misc.capture import Capture, FDCapture - -class TestFDCapture: - def test_basic(self): - tmpfile = py.std.os.tmpfile() - fd = tmpfile.fileno() - cap = FDCapture(fd) - os.write(fd, "hello") - f = cap.done() - s = f.read() - assert s == "hello" - - def test_stderr(self): - cap = FDCapture(2, 'stderr') - print >>sys.stderr, "hello" - f = cap.done() - s = f.read() - assert s == "hello\n" class TestCapturingOnSys: def getcapture(self): - return SimpleOutErrCapture() + return py.io.StdCapture() def test_capturing_simple(self): cap = self.getcapture() @@ -73,20 +54,14 @@ finally: cap.reset() -def test_callcapture(): +def test_callcapture_nofd(): def func(x, y): print x print >>py.std.sys.stderr, y return 42 - res, out, err = callcapture(func, 3, y=4) + res, out, err = py.io.StdCapture.call(func, 3, y=4) assert res == 42 assert out.startswith("3") assert err.startswith("4") -class TestCapturingOnFDs(TestCapturingOnSys): - def test_reading_stdin_while_captured_doesnt_hang(self): - py.test.skip("Hangs in py.test --session=R") - - def getcapture(self): - return Capture() Modified: py/trunk/py/log/testing/test_log.py ============================================================================== --- py/trunk/py/log/testing/test_log.py (original) +++ py/trunk/py/log/testing/test_log.py Thu Feb 1 16:20:39 2007 @@ -1,7 +1,8 @@ import py -from py.__.misc.simplecapture import callcapture import sys +callcapture = py.io.StdCapture.call + def setup_module(mod): mod.tempdir = py.test.ensuretemp("py.log-test") mod.logstate = py.log._getstate() Deleted: /py/trunk/py/misc/capture.py ============================================================================== --- /py/trunk/py/misc/capture.py Thu Feb 1 16:20:39 2007 +++ (empty file) @@ -1,49 +0,0 @@ - -import os, sys - -class FDCapture: - def __init__(self, targetfd, sysattr=None): - self.targetfd = targetfd - self.tmpfile = self.maketmpfile() - self._savefd = os.dup(targetfd) - os.dup2(self.tmpfile.fileno(), targetfd) - if sysattr is not None: - self._reset = (lambda oldval=getattr(sys, sysattr): - setattr(sys, sysattr, oldval)) - setattr(sys, sysattr, self.tmpfile) - - def done(self): - os.dup2(self._savefd, self.targetfd) - if hasattr(self, '_reset'): - self._reset() - del self._reset - os.close(self._savefd) - f = self.tmpfile - f.seek(0) - del self._savefd - del self.tmpfile - return f - - def maketmpfile(self): - f = os.tmpfile() - fd = f.fileno() - newfd = os.dup(fd) - newf = os.fdopen(newfd, 'w+b', 0) - f.close() - return newf - -class Capture: - def __init__(self): - self._out = FDCapture(1, 'stdout') - self._oldsysout = sys.stdout - sys.stdout = self._out.tmpfile - - self._err = FDCapture(2, 'stderr') - self._olderrout = sys.stderr - sys.stderr = self._err.tmpfile - - def reset(self): - outfile = self._out.done() - errfile = self._err.done() - return outfile.read(), errfile.read() - Deleted: /py/trunk/py/misc/simplecapture.py ============================================================================== --- /py/trunk/py/misc/simplecapture.py Thu Feb 1 16:20:39 2007 +++ (empty file) @@ -1,58 +0,0 @@ -""" - -capture stdout/stderr - -""" -import sys -try: from cStringIO import StringIO -except ImportError: from StringIO import StringIO - -class SimpleOutErrCapture: - """ capture sys.stdout/sys.stderr (but not system level fd 1 and 2). - - this captures only "In-Memory" and is currently intended to be - used by the unittest package to capture print-statements in tests. - """ - def __init__(self): - self.oldin = sys.stdin - self.oldout = sys.stdout - self.olderr = sys.stderr - sys.stdin = self.newin = DontReadFromInput() - sys.stdout = self.newout = StringIO() - sys.stderr = self.newerr = StringIO() - - def reset(self): - """ return captured output and restore sys.stdout/err.""" - x, y = self.done() - return x.read(), y.read() - - def done(self): - o,e = sys.stdout, sys.stderr - sys.stdin, sys.stdout, sys.stderr = ( - self.oldin, self.oldout, self.olderr) - del self.oldin, self.oldout, self.olderr - o, e = self.newout, self.newerr - o.seek(0) - e.seek(0) - return o,e - -class DontReadFromInput: - """Temporary stub class. Ideally when stdin is accessed, the - capturing should be turned off, with possibly all data captured - so far sent to the screen. This should be configurable, though, - because in automated test runs it is better to crash than - hang indefinitely. - """ - def read(self, *args): - raise IOError("reading from stdin while output is captured") - readline = read - readlines = read - __iter__ = read - -def callcapture(func, *args, **kwargs): - so = SimpleOutErrCapture() - try: - res = func(*args, **kwargs) - finally: - out, err = so.reset() - return res, out, err Deleted: /py/trunk/py/misc/testing/test_simplecapture.py ============================================================================== --- /py/trunk/py/misc/testing/test_simplecapture.py Thu Feb 1 16:20:39 2007 +++ (empty file) @@ -1,92 +0,0 @@ -import os, sys -import py -from py.__.misc.simplecapture import SimpleOutErrCapture, callcapture -from py.__.misc.capture import Capture, FDCapture - -class TestFDCapture: - def test_basic(self): - tmpfile = py.std.os.tmpfile() - fd = tmpfile.fileno() - cap = FDCapture(fd) - os.write(fd, "hello") - f = cap.done() - s = f.read() - assert s == "hello" - - def test_stderr(self): - cap = FDCapture(2, 'stderr') - print >>sys.stderr, "hello" - f = cap.done() - s = f.read() - assert s == "hello\n" - -class TestCapturingOnSys: - - def getcapture(self): - return SimpleOutErrCapture() - - def test_capturing_simple(self): - cap = self.getcapture() - print "hello world" - print >>sys.stderr, "hello error" - out, err = cap.reset() - assert out == "hello world\n" - assert err == "hello error\n" - - def test_capturing_twice_error(self): - cap = self.getcapture() - print "hello" - cap.reset() - py.test.raises(AttributeError, "cap.reset()") - - def test_capturing_modify_sysouterr_in_between(self): - oldout = sys.stdout - olderr = sys.stderr - cap = self.getcapture() - print "hello", - print >>sys.stderr, "world", - sys.stdout = py.std.StringIO.StringIO() - sys.stderr = py.std.StringIO.StringIO() - print "not seen" - print >>sys.stderr, "not seen" - out, err = cap.reset() - assert out == "hello" - assert err == "world" - assert sys.stdout == oldout - assert sys.stderr == olderr - - def test_capturing_error_recursive(self): - cap1 = self.getcapture() - print "cap1" - cap2 = self.getcapture() - print "cap2" - out2, err2 = cap2.reset() - py.test.raises(AttributeError, "cap2.reset()") - out1, err1 = cap1.reset() - assert out1 == "cap1\n" - assert out2 == "cap2\n" - - def test_reading_stdin_while_captured_doesnt_hang(self): - cap = self.getcapture() - try: - py.test.raises(IOError, raw_input) - finally: - cap.reset() - -def test_callcapture(): - def func(x, y): - print x - print >>py.std.sys.stderr, y - return 42 - - res, out, err = callcapture(func, 3, y=4) - assert res == 42 - assert out.startswith("3") - assert err.startswith("4") - -class TestCapturingOnFDs(TestCapturingOnSys): - def test_reading_stdin_while_captured_doesnt_hang(self): - py.test.skip("Hangs in py.test --session=R") - - def getcapture(self): - return Capture() Modified: py/trunk/py/test/collect.py ============================================================================== --- py/trunk/py/test/collect.py (original) +++ py/trunk/py/test/collect.py Thu Feb 1 16:20:39 2007 @@ -379,11 +379,7 @@ def startcapture(self): if not self.config.option.nocapture: assert not hasattr(self, '_capture') - self._capture = py.io.OutErrCapture() - # XXX integrate this into py.io / refactor - # execnet/py.test capturing mechanisms - #from py.__.misc.simplecapture import SimpleOutErrCapture - #self._capture = SimpleOutErrCapture() + self._capture = py.io.StdCaptureFD() def finishcapture(self): if hasattr(self, '_capture'): Modified: py/trunk/py/test/item.py ============================================================================== --- py/trunk/py/test/item.py (original) +++ py/trunk/py/test/item.py Thu Feb 1 16:20:39 2007 @@ -32,10 +32,7 @@ class Item(py.test.collect.Collector): def startcapture(self): if not self.config.option.nocapture: - # XXX refactor integrate capturing - self._capture = py.io.OutErrCapture() - #from py.__.misc.simplecapture import SimpleOutErrCapture - #self._capture = SimpleOutErrCapture() + self._capture = py.io.StdCaptureFD() def finishcapture(self): if hasattr(self, '_capture'): Modified: py/trunk/py/test/rsession/local.py ============================================================================== --- py/trunk/py/test/rsession/local.py (original) +++ py/trunk/py/test/rsession/local.py Thu Feb 1 16:20:39 2007 @@ -2,6 +2,7 @@ """ local-only operations """ +import py from py.__.test.rsession.executor import BoxExecutor, RunExecutor,\ ApigenExecutor from py.__.test.rsession import report @@ -9,10 +10,8 @@ # XXX copied from session.py def startcapture(session): - if not session.config.option.nocapture and not session.config.option.usepdb: - # XXX refactor integrate capturing - from py.__.misc.simplecapture import SimpleOutErrCapture - session._capture = SimpleOutErrCapture() + if not session.config.option.nocapture: + session._capture = py.io.StdCapture() def finishcapture(session): if hasattr(session, '_capture'): Modified: py/trunk/py/test/rsession/rsession.py ============================================================================== --- py/trunk/py/test/rsession/rsession.py (original) +++ py/trunk/py/test/rsession/rsession.py Thu Feb 1 16:20:39 2007 @@ -259,7 +259,7 @@ raise NotImplementedError("%s does not contain 'build' " "function" %(apigen,)) print >>sys.stderr, 'building documentation' - capture = py.io.OutErrCapture() + capture = py.io.StdCaptureFD() try: pkgdir = self.getpkgdir(self.config.args[0]) apigen.build(pkgdir, Modified: py/trunk/py/test/rsession/testing/test_reporter.py ============================================================================== --- py/trunk/py/test/rsession/testing/test_reporter.py (original) +++ py/trunk/py/test/rsession/testing/test_reporter.py Thu Feb 1 16:20:39 2007 @@ -74,7 +74,7 @@ for outcome in outcomes: r.report(report.ReceivedItemOutcome(ch, item, outcome)) - cap = py.io.OutErrCapture() + cap = py.io.StdCaptureFD() boxfun(config, item, outcomes) out, err = cap.reset() assert not err @@ -97,7 +97,7 @@ for outcome in outcomes: r.report(report.ReceivedItemOutcome(ch, funcitem, outcome)) - cap = py.io.OutErrCapture() + cap = py.io.StdCaptureFD() boxfun(self.pkgdir, config, moditem, funcitem, outcomes) out, err = cap.reset() assert not err @@ -125,7 +125,7 @@ r = self.reporter(config, hosts) list(rootcol.tryiter(reporterror=lambda x : AbstractSession.reporterror(r.report, x))) - cap = py.io.OutErrCapture() + cap = py.io.StdCaptureFD() boxfun() out, err = cap.reset() assert not err @@ -147,7 +147,7 @@ list(rootcol.tryiter(reporterror=lambda x : AbstractSession.reporterror(r.report, x))) r.report(report.TestFinished()) - cap = py.io.OutErrCapture() + cap = py.io.StdCaptureFD() boxfun() out, err = cap.reset() assert not err @@ -156,7 +156,7 @@ def _test_still_to_go(self): tmpdir = py.test.ensuretemp("stilltogo") tmpdir.ensure("__init__.py") - cap = py.io.OutErrCapture() + cap = py.io.StdCaptureFD() config = py.test.config._reparse([str(tmpdir)]) hosts = [HostInfo(i) for i in ["host1", "host2", "host3"]] r = self.reporter(config, hosts) Modified: py/trunk/py/test/testing/test_session.py ============================================================================== --- py/trunk/py/test/testing/test_session.py (original) +++ py/trunk/py/test/testing/test_session.py Thu Feb 1 16:20:39 2007 @@ -189,7 +189,7 @@ import py class Function(py.test.Function): def startcapture(self): - self._mycapture = py.io.OutErrCapture() + self._mycapture = py.io.StdCaptureFD() def finishcapture(self): self._testmycapture = self._mycapture.reset() Deleted: /py/trunk/py/test/todo-cleanup.txt ============================================================================== --- /py/trunk/py/test/todo-cleanup.txt Thu Feb 1 16:20:39 2007 +++ (empty file) @@ -1,10 +0,0 @@ - - -fix --looponfailing: currently in case of failures -all tests are re-run. the problem was introduced -while cleaning up session.main() calls ... -the setup of remote and local config objects -and collectors needs to be reviewed anyway -(and the terminalsession and rsession handling -of such config object should be unified with -respect to this configuration/failure communication) Modified: py/trunk/py/thread/testing/test_pool.py ============================================================================== --- py/trunk/py/thread/testing/test_pool.py (original) +++ py/trunk/py/thread/testing/test_pool.py Thu Feb 1 16:20:39 2007 @@ -77,7 +77,7 @@ pool.join(timeout=0.1) def test_pool_clean_shutdown(): - capture = py.io.OutErrCapture() + capture = py.io.StdCaptureFD() pool = WorkerPool() def f(): pass From hpk at codespeak.net Thu Feb 1 16:23:29 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 1 Feb 2007 16:23:29 +0100 (CET) Subject: [py-svn] r37742 - py/trunk/py/doc Message-ID: <20070201152329.A9446100A4@code0.codespeak.net> Author: hpk Date: Thu Feb 1 16:23:27 2007 New Revision: 37742 Modified: py/trunk/py/doc/index.txt Log: fix reference Modified: py/trunk/py/doc/index.txt ============================================================================== --- py/trunk/py/doc/index.txt (original) +++ py/trunk/py/doc/index.txt Thu Feb 1 16:23:27 2007 @@ -35,6 +35,7 @@ .. _`py.execnet`: execnet.html .. _`py.magic.greenlet`: greenlet.html .. _`py.log`: log.html +.. _`py.io`: io.html .. _`py.path`: path.html .. _`py.code`: code.html .. _`py.test`: test.html From hpk at codespeak.net Thu Feb 1 16:23:52 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 1 Feb 2007 16:23:52 +0100 (CET) Subject: [py-svn] r37743 - py/trunk/py/misc Message-ID: <20070201152352.58F3A10090@code0.codespeak.net> Author: hpk Date: Thu Feb 1 16:23:51 2007 New Revision: 37743 Removed: py/trunk/py/misc/stdoutcapture.py Log: ah, there was yet another version of capturing hiding Deleted: /py/trunk/py/misc/stdoutcapture.py ============================================================================== --- /py/trunk/py/misc/stdoutcapture.py Thu Feb 1 16:23:51 2007 +++ (empty file) @@ -1,73 +0,0 @@ -""" -A quick hack to capture stdout/stderr. -""" - -import os, sys - - -class Capture: - - def __init__(self, mixed_out_err = False): - "Start capture of the Unix-level stdout and stderr." - if (not hasattr(os, 'tmpfile') or - not hasattr(os, 'dup') or - not hasattr(os, 'dup2') or - not hasattr(os, 'fdopen')): - self.dummy = 1 - else: - self.dummy = 0 - # make new stdout/stderr files if needed - self.localoutfd = os.dup(1) - self.localerrfd = os.dup(2) - if hasattr(sys.stdout, 'fileno') and sys.stdout.fileno() == 1: - self.saved_stdout = sys.stdout - sys.stdout = os.fdopen(self.localoutfd, 'w', 1) - else: - self.saved_stdout = None - if hasattr(sys.stderr, 'fileno') and sys.stderr.fileno() == 2: - self.saved_stderr = sys.stderr - sys.stderr = os.fdopen(self.localerrfd, 'w', 0) - else: - self.saved_stderr = None - self.tmpout = os.tmpfile() - if mixed_out_err: - self.tmperr = self.tmpout - else: - self.tmperr = os.tmpfile() - os.dup2(self.tmpout.fileno(), 1) - os.dup2(self.tmperr.fileno(), 2) - - def done(self): - "End capture and return the captured text (stdoutfile, stderrfile)." - if self.dummy: - import cStringIO - return cStringIO.StringIO(), cStringIO.StringIO() - else: - os.dup2(self.localoutfd, 1) - os.dup2(self.localerrfd, 2) - if self.saved_stdout is not None: - f = sys.stdout - sys.stdout = self.saved_stdout - f.close() - else: - os.close(self.localoutfd) - if self.saved_stderr is not None: - f = sys.stderr - sys.stderr = self.saved_stderr - f.close() - else: - os.close(self.localerrfd) - self.tmpout.seek(0) - self.tmperr.seek(0) - return self.tmpout, self.tmperr - - -if __name__ == '__main__': - # test - c = Capture() - try: - os.system('echo hello') - finally: - fout, ferr = c.done() - print 'Output:', `fout.read()` - print 'Error:', `ferr.read()` From hpk at codespeak.net Thu Feb 1 16:35:30 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 1 Feb 2007 16:35:30 +0100 (CET) Subject: [py-svn] r37745 - py/trunk/py/io Message-ID: <20070201153530.06544100A4@code0.codespeak.net> Author: hpk Date: Thu Feb 1 16:35:29 2007 New Revision: 37745 Modified: py/trunk/py/io/stdcapture.py Log: fixing and adding to docstring Modified: py/trunk/py/io/stdcapture.py ============================================================================== --- py/trunk/py/io/stdcapture.py (original) +++ py/trunk/py/io/stdcapture.py Thu Feb 1 16:35:29 2007 @@ -52,8 +52,8 @@ class StdCapture(Capture): """ capture sys.stdout/sys.stderr (but not system level fd 1 and 2). - this captures only "In-Memory" and is currently intended to be - used by the unittest package to capture print-statements in tests. + This class allows to capture writes to sys.stdout|stderr "in-memory" + and will raise errors on tries to read from sys.stdin. """ def __init__(self): self.oldin = sys.stdin From hpk at codespeak.net Thu Feb 1 16:58:11 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 1 Feb 2007 16:58:11 +0100 (CET) Subject: [py-svn] r37751 - in py/trunk/py: c-extension/greenlet misc Message-ID: <20070201155811.9D5BB10079@code0.codespeak.net> Author: hpk Date: Thu Feb 1 16:58:10 2007 New Revision: 37751 Modified: py/trunk/py/c-extension/greenlet/test_generator.py py/trunk/py/misc/buildcmodule.py Log: fix two other places that used capturing (although the greenlet fix is not really related, but i first saw it now on win32) Modified: py/trunk/py/c-extension/greenlet/test_generator.py ============================================================================== --- py/trunk/py/c-extension/greenlet/test_generator.py (original) +++ py/trunk/py/c-extension/greenlet/test_generator.py Thu Feb 1 16:58:10 2007 @@ -1,7 +1,7 @@ import py try: from py.magic import greenlet -except RuntimeError, e: +except (ImportError, RuntimeError), e: py.test.skip(str(e)) Modified: py/trunk/py/misc/buildcmodule.py ============================================================================== --- py/trunk/py/misc/buildcmodule.py (original) +++ py/trunk/py/misc/buildcmodule.py Thu Feb 1 16:58:10 2007 @@ -11,7 +11,6 @@ import os, sys, imp from distutils.core import setup from distutils.extension import Extension - import stdoutcapture debug = 0 #try: @@ -36,7 +35,7 @@ if lib.check(): lib.remove() - c = stdoutcapture.Capture(mixed_out_err = True) + c = py.io.StdCaptureFD() try: try: saved_environ = os.environ.items() From guido at codespeak.net Thu Feb 1 16:58:43 2007 From: guido at codespeak.net (guido at codespeak.net) Date: Thu, 1 Feb 2007 16:58:43 +0100 (CET) Subject: [py-svn] r37752 - py/trunk/py/doc Message-ID: <20070201155843.739F41008D@code0.codespeak.net> Author: guido Date: Thu Feb 1 16:58:42 2007 New Revision: 37752 Modified: py/trunk/py/doc/path.txt Log: Added some more code examples. Modified: py/trunk/py/doc/path.txt ============================================================================== --- py/trunk/py/doc/path.txt (original) +++ py/trunk/py/doc/path.txt Thu Feb 1 16:58:42 2007 @@ -8,7 +8,8 @@ The 'py' lib provides a uniform high-level api to deal with filesystems and filesystem-like interfaces: :api:`py.path`. It aims to offer a central object to fs-like object trees (reading from and writing to files, adding -files/directories, examining the types and structure, etc.). +files/directories, examining the types and structure, etc.), and out-of-the-box +provides a number of implementations of this API. Path implementations provided by :api:`py.path` =============================================== @@ -34,6 +35,9 @@ True >>> foopath.read() 'bar' + >>> foofile = foopath.open() # return a 'real' file object + >>> foofile.read(1) + 'b' :api:`py.path.svnurl` and :api:`py.path.svnwc` ---------------------------------------------- @@ -74,32 +78,103 @@ path object within an application (e.g. from "local" to "svnwc"). The common set includes functions such as `path.read()` to read all data from a file, `path.write()` to write data, `path.listdir()` to get a list -of directory entries, and `path.check()` to check if a node exists +of directory entries, `path.check()` to check if a node exists and is of a particular type, `path.join()` to get to a (grand)child, `path.visit()` to recursively walk through a node's -children, etc. Only things that are not common on all filesystems, such -as handling metadata (e.g. the subversion "properties") require +children, etc. Only things that are not common on 'normal' filesystems (yet), +such as handling metadata (e.g. the Subversion "properties") require using specific APIs. Examples --------------------------------- +A quick 'cookbook' of small examples that will be useful 'in real life', +which also presents parts of the 'common' API, and shows some non-common +methods: + Searching `.txt` files +++++++++++++++++++++++++++++++++++++ -XXX example +Search for a particular string inside all files with a .txt extension in a +specific directory. +:: -Joining and checking path types -+++++++++++++++++++++++++++++++++++++ + >>> dirpath = temppath.ensure('testdir', dir=True) + >>> dirpath.join('textfile1.txt').write('foo bar baz') + >>> dirpath.join('textfile2.txt').write('frob bar spam eggs') + >>> subdir = dirpath.ensure('subdir', dir=True) + >>> subdir.join('textfile1.txt').write('foo baz') + >>> subdir.join('textfile2.txt').write('spam eggs spam foo bar spam') + >>> results = [] + >>> for fpath in dirpath.visit('*.txt'): + ... if 'bar' in fpath.read(): + ... results.append(fpath.relto(dirpath)) + >>> results + ['textfile1.txt', 'textfile2.txt', 'subdir/textfile2.txt'] + +Joining path types +++++++++++++++++++++ + +This example shows the :api:`py.path` features to deal with actual paths +(strings). Note that the filesystem is never touched, all operations are +performed on a string level (so the paths don't have to exist, either):: + + >>> p1 = py.path.local('/foo/bar') + >>> p2 = p1.join('baz/qux') + >>> p2.strpath + '/foo/bar/baz/qux' + >>> p2.relto(p1) + 'baz/qux' + >>> p3 = p1 / 'baz/qux' # the / operator allows joining, too + >>> p2 == p3 + True + +This should be possible on every implementation of :api:`py.path`, so +regardless of whether the implementation wraps a UNIX filesystem, a Windows +one, or a database or object tree, these functions should be available (each +with their own notion of path seperators and dealing with conversions, etc.). -XXX example +Checking path types ++++++++++++++++++++++ -setting svn-properties +Now we will show a bit about the powerful 'check()' method on paths, which +allows you to check whether a file exists, what type it is, etc.:: + + >>> file1 = temppath.join('file1') + >>> file1.check() # does it exist? + False + >>> file1 = file1.ensure(file=True) # 'touch' the file + >>> file1.check() + True + >>> file1.check(dir=True) # is it a dir? + False + >>> file1.check(file=True) # or a file? + True + >>> file1.check(ext='.txt') # check the extension + False + >>> textfile = temppath.ensure('text.txt', file=True) + >>> textfile.check(ext='.txt') + True + >>> file1.check(basename='file1') # we can use all the path's properties here + True + +Setting svn-properties +++++++++++++++++++++++++++++++++++++++ -XXX example +As an example of 'uncommon' methods, we'll show how to read and write +properties in an :api:`py.path.svnwc` instance:: + >>> wc.propget('foo') + '' + >>> wc.propset('foo', 'bar') + >>> wc.propget('foo') + 'bar' + >>> len(wc.status().prop_modified) # our own props + 1 + >>> msg = wc.revert() # roll back our changes + >>> len(wc.status().prop_modified) + 0 XXX more examples (look at API) +++++++++++++++++++++++++++++++++++++++ @@ -117,7 +192,6 @@ XXX note more here - Future plans ============ From guido at codespeak.net Thu Feb 1 21:10:53 2007 From: guido at codespeak.net (guido at codespeak.net) Date: Thu, 1 Feb 2007 21:10:53 +0100 (CET) Subject: [py-svn] r37765 - py/trunk/py/apigen Message-ID: <20070201201053.532C010074@code0.codespeak.net> Author: guido Date: Thu Feb 1 21:10:48 2007 New Revision: 37765 Modified: py/trunk/py/apigen/apigen.py py/trunk/py/apigen/htmlgen.py Log: Was still getting filenames for source files from code objects, so adding more defensiveness, and made that the 'capture' object is passed over to the builder instances (to help debugging, currently not used). Modified: py/trunk/py/apigen/apigen.py ============================================================================== --- py/trunk/py/apigen/apigen.py (original) +++ py/trunk/py/apigen/apigen.py Thu Feb 1 21:10:48 2007 @@ -29,8 +29,9 @@ all_names = dsa._get_names(filter=lambda x, y: True) namespace_tree = htmlgen.create_namespace_tree(all_names) - apb = htmlgen.ApiPageBuilder(targetdir, l, dsa, pkgdir, namespace_tree) - spb = htmlgen.SourcePageBuilder(targetdir, l, pkgdir) + apb = htmlgen.ApiPageBuilder(targetdir, l, dsa, pkgdir, namespace_tree, + capture) + spb = htmlgen.SourcePageBuilder(targetdir, l, pkgdir, capture) capture.err.writeorg('preparing namespace pages\n') ns_data = apb.prepare_namespace_pages() Modified: py/trunk/py/apigen/htmlgen.py ============================================================================== --- py/trunk/py/apigen/htmlgen.py (original) +++ py/trunk/py/apigen/htmlgen.py Thu Feb 1 21:10:48 2007 @@ -205,10 +205,11 @@ class SourcePageBuilder(AbstractPageBuilder): """ builds the html for a source docs page """ - def __init__(self, base, linker, projroot): + def __init__(self, base, linker, projroot, capture=None): self.base = base self.linker = linker self.projroot = projroot + self.capture = capture def build_navigation(self, fspath): nav = H.Navigation() @@ -348,14 +349,16 @@ class ApiPageBuilder(AbstractPageBuilder): """ builds the html for an api docs page """ - def __init__(self, base, linker, dsa, projroot, namespace_tree): + def __init__(self, base, linker, dsa, projroot, namespace_tree, + capture=None): self.base = base self.linker = linker self.dsa = dsa self.projroot = projroot self.projpath = py.path.local(projroot) self.namespace_tree = namespace_tree - + self.capture = capture + def build_callable_view(self, dotted_name): """ build the html for a class method """ # XXX we may want to have seperate @@ -728,7 +731,8 @@ # skip py.code.Source objects and source files outside of the # package is_code_source = self._reg_source.match(sourcefile) - if (not is_code_source and self.is_in_pkg(sourcefile)): + if (not is_code_source and self.is_in_pkg(sourcefile) and + py.path.local(sourcefile).check()): enc = source_html.get_module_encoding(sourcefile) href = self.linker.get_lazyhref(sourcefile) sourcelink = H.a(linktext, href=href) @@ -737,7 +741,8 @@ sourcelink = H.div(linktext) colored = enumerate_and_color(mangled, frame.firstlineno, enc) else: - sourcelink = H.div('source unknown') + sourcelink = H.div('source unknown (%s)' % (sourcefile,)) + colored = mangled[:] tbdiv.append(sourcelink) tbdiv.append(H.div(class_='code', *colored)) return tbdiv From hpk at codespeak.net Thu Feb 1 21:26:28 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 1 Feb 2007 21:26:28 +0100 (CET) Subject: [py-svn] r37766 - in py/trunk/py/io: . test Message-ID: <20070201202628.1197C1007F@code0.codespeak.net> Author: hpk Date: Thu Feb 1 21:26:27 2007 New Revision: 37766 Removed: py/trunk/py/io/test/test_simplecapture.py Modified: py/trunk/py/io/stdcapture.py py/trunk/py/io/test/test_capture.py Log: unifying non-FD and FD capturing some more (could be more, but at least the APIs start to feel the same) Modified: py/trunk/py/io/stdcapture.py ============================================================================== --- py/trunk/py/io/stdcapture.py (original) +++ py/trunk/py/io/stdcapture.py Thu Feb 1 21:26:27 2007 @@ -4,6 +4,8 @@ try: from cStringIO import StringIO except ImportError: from StringIO import StringIO +emptyfile = StringIO() + class Capture(object): def call(cls, func, *args, **kwargs): """ return a (res, out, err) tuple where @@ -24,13 +26,17 @@ """ capture Stdout and Stderr both on filedescriptor and sys.stdout/stderr level. """ - def __init__(self, out=True, err=True, patchsys=True): + def __init__(self, out=True, err=True, mixed=False, patchsys=True): if out: self.out = py.io.FDCapture(1) if patchsys: self.out.setasfile('stdout') if err: - self.err = py.io.FDCapture(2) + if mixed and out: + tmpfile = self.out.tmpfile + else: + tmpfile = None + self.err = py.io.FDCapture(2, tmpfile=tmpfile) if patchsys: self.err.setasfile('stderr') @@ -40,14 +46,12 @@ returns a tuple of file objects (out, err) for the captured data """ - out = err = "" + outfile = errfile = emptyfile if hasattr(self, 'out'): outfile = self.out.done() - out = outfile.read() if hasattr(self, 'err'): errfile = self.err.done() - err = errfile.read() - return out, err + return outfile.read(), errfile.read() class StdCapture(Capture): """ capture sys.stdout/sys.stderr (but not system level fd 1 and 2). @@ -55,13 +59,21 @@ This class allows to capture writes to sys.stdout|stderr "in-memory" and will raise errors on tries to read from sys.stdin. """ - def __init__(self): + def __init__(self, out=True, err=True, mixed=False): + self._out = out + self._err = err + if out: + self.oldout = sys.stdout + sys.stdout = self.newout = StringIO() + if err: + self.olderr = sys.stderr + if out and mixed: + newerr = self.newout + else: + newerr = StringIO() + sys.stderr = self.newerr = newerr self.oldin = sys.stdin - self.oldout = sys.stdout - self.olderr = sys.stderr sys.stdin = self.newin = DontReadFromInput() - sys.stdout = self.newout = StringIO() - sys.stderr = self.newerr = StringIO() def reset(self): """ return captured output and restore sys.stdout/err.""" @@ -70,13 +82,25 @@ def done(self): o,e = sys.stdout, sys.stderr - sys.stdin, sys.stdout, sys.stderr = ( - self.oldin, self.oldout, self.olderr) - del self.oldin, self.oldout, self.olderr - o, e = self.newout, self.newerr - o.seek(0) - e.seek(0) - return o,e + outfile = errfile = emptyfile + if self._out: + try: + sys.stdout = self.oldout + except AttributeError: + raise IOError("stdout capturing already reset") + del self.oldout + outfile = self.newout + outfile.seek(0) + if self._err: + try: + sys.stderr = self.olderr + except AttributeError: + raise IOError("stderr capturing already reset") + del self.olderr + errfile = self.newerr + errfile.seek(0) + sys.stdin = self.oldin + return outfile, errfile class DontReadFromInput: """Temporary stub class. Ideally when stdin is accessed, the Modified: py/trunk/py/io/test/test_capture.py ============================================================================== --- py/trunk/py/io/test/test_capture.py (original) +++ py/trunk/py/io/test/test_capture.py Thu Feb 1 21:26:27 2007 @@ -57,9 +57,9 @@ tmpfp.close() f = cap.done() -class TestCapturing: - def getcapture(self): - return py.io.StdCaptureFD() +class TestStdCapture: + def getcapture(self, **kw): + return py.io.StdCapture(**kw) def test_capturing_simple(self): cap = self.getcapture() @@ -69,6 +69,15 @@ assert out == "hello world\n" assert err == "hello error\n" + def test_capturing_mixed(self): + cap = self.getcapture(mixed=True) + print "hello", + print >>sys.stderr, "world", + print >>sys.stdout, ".", + out, err = cap.reset() + assert out.strip() == "hello world ." + assert not err + def test_capturing_twice_error(self): cap = self.getcapture() print "hello" @@ -101,6 +110,26 @@ out1, err1 = cap1.reset() assert out1 == "cap1\n" assert out2 == "cap2\n" + + def test_just_out_capture(self): + cap = self.getcapture(out=True, err=False) + print >>sys.stdout, "hello" + print >>sys.stderr, "world" + out, err = cap.reset() + assert out == "hello\n" + assert not err + + def test_just_err_capture(self): + cap = self.getcapture(out=False, err=True) + print >>sys.stdout, "hello" + print >>sys.stderr, "world" + out, err = cap.reset() + assert err == "world\n" + assert not out + +class TestStdCaptureFD(TestStdCapture): + def getcapture(self, **kw): + return py.io.StdCaptureFD(**kw) def test_intermingling(self): cap = self.getcapture() @@ -114,32 +143,16 @@ assert out == "123" assert err == "abc" -def test_callcapture(): - def func(x, y): - print x - print >>py.std.sys.stderr, y - return 42 - - res, out, err = py.io.StdCaptureFD.call(func, 3, y=4) - assert res == 42 - assert out.startswith("3") - assert err.startswith("4") - -def test_just_out_capture(): - cap = py.io.StdCaptureFD(out=True, err=False) - print >>sys.stdout, "hello" - print >>sys.stderr, "world" - out, err = cap.reset() - assert out == "hello\n" - assert not err - -def test_just_err_capture(): - cap = py.io.StdCaptureFD(out=False, err=True) - print >>sys.stdout, "hello" - print >>sys.stderr, "world" - out, err = cap.reset() - assert err == "world\n" - assert not out + def test_callcapture(self): + def func(x, y): + print x + print >>py.std.sys.stderr, y + return 42 + + res, out, err = py.io.StdCaptureFD.call(func, 3, y=4) + assert res == 42 + assert out.startswith("3") + assert err.startswith("4") def test_capture_no_sys(): cap = py.io.StdCaptureFD(patchsys=False) @@ -151,6 +164,15 @@ assert out == "1" assert err == "2" -#class TestCapturingOnFDs(TestCapturingOnSys): -# def getcapture(self): -# return Capture() +def test_callcapture_nofd(): + def func(x, y): + os.write(1, "hello") + os.write(2, "hello") + print x + print >>py.std.sys.stderr, y + return 42 + + res, out, err = py.io.StdCapture.call(func, 3, y=4) + assert res == 42 + assert out.startswith("3") + assert err.startswith("4") Deleted: /py/trunk/py/io/test/test_simplecapture.py ============================================================================== --- /py/trunk/py/io/test/test_simplecapture.py Thu Feb 1 21:26:27 2007 +++ (empty file) @@ -1,67 +0,0 @@ -import os, sys -import py - -class TestCapturingOnSys: - - def getcapture(self): - return py.io.StdCapture() - - def test_capturing_simple(self): - cap = self.getcapture() - print "hello world" - print >>sys.stderr, "hello error" - out, err = cap.reset() - assert out == "hello world\n" - assert err == "hello error\n" - - def test_capturing_twice_error(self): - cap = self.getcapture() - print "hello" - cap.reset() - py.test.raises(AttributeError, "cap.reset()") - - def test_capturing_modify_sysouterr_in_between(self): - oldout = sys.stdout - olderr = sys.stderr - cap = self.getcapture() - print "hello", - print >>sys.stderr, "world", - sys.stdout = py.std.StringIO.StringIO() - sys.stderr = py.std.StringIO.StringIO() - print "not seen" - print >>sys.stderr, "not seen" - out, err = cap.reset() - assert out == "hello" - assert err == "world" - assert sys.stdout == oldout - assert sys.stderr == olderr - - def test_capturing_error_recursive(self): - cap1 = self.getcapture() - print "cap1" - cap2 = self.getcapture() - print "cap2" - out2, err2 = cap2.reset() - py.test.raises(AttributeError, "cap2.reset()") - out1, err1 = cap1.reset() - assert out1 == "cap1\n" - assert out2 == "cap2\n" - - def test_reading_stdin_while_captured_doesnt_hang(self): - cap = self.getcapture() - try: - py.test.raises(IOError, raw_input) - finally: - cap.reset() - -def test_callcapture_nofd(): - def func(x, y): - print x - print >>py.std.sys.stderr, y - return 42 - - res, out, err = py.io.StdCapture.call(func, 3, y=4) - assert res == 42 - assert out.startswith("3") - assert err.startswith("4") - From guido at codespeak.net Thu Feb 1 21:28:22 2007 From: guido at codespeak.net (guido at codespeak.net) Date: Thu, 1 Feb 2007 21:28:22 +0100 (CET) Subject: [py-svn] r37767 - py/trunk/py/doc Message-ID: <20070201202822.8AB111008D@code0.codespeak.net> Author: guido Date: Thu Feb 1 21:28:21 2007 New Revision: 37767 Modified: py/trunk/py/doc/path.txt Log: Fixed Windows issues in doctests. Modified: py/trunk/py/doc/path.txt ============================================================================== --- py/trunk/py/doc/path.txt (original) +++ py/trunk/py/doc/path.txt Thu Feb 1 21:28:21 2007 @@ -109,9 +109,9 @@ >>> results = [] >>> for fpath in dirpath.visit('*.txt'): ... if 'bar' in fpath.read(): - ... results.append(fpath.relto(dirpath)) + ... results.append(fpath.basename) >>> results - ['textfile1.txt', 'textfile2.txt', 'subdir/textfile2.txt'] + ['textfile1.txt', 'textfile2.txt', 'textfile2.txt'] Joining path types ++++++++++++++++++++ @@ -122,9 +122,10 @@ >>> p1 = py.path.local('/foo/bar') >>> p2 = p1.join('baz/qux') - >>> p2.strpath - '/foo/bar/baz/qux' - >>> p2.relto(p1) + >>> p2 == py.path.local('/foo/bar/baz/qux') + True + >>> sep = py.path.local.sep + >>> p2.relto(p1).replace(sep, '/') # os-specific path sep in the string 'baz/qux' >>> p3 = p1 / 'baz/qux' # the / operator allows joining, too >>> p2 == p3 From hpk at codespeak.net Thu Feb 1 22:16:12 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 1 Feb 2007 22:16:12 +0100 (CET) Subject: [py-svn] r37769 - py/trunk/py/doc Message-ID: <20070201211612.7B0731007F@code0.codespeak.net> Author: hpk Date: Thu Feb 1 22:16:10 2007 New Revision: 37769 Modified: py/trunk/py/doc/misc.txt Log: minor clarifi Modified: py/trunk/py/doc/misc.txt ============================================================================== --- py/trunk/py/doc/misc.txt (original) +++ py/trunk/py/doc/misc.txt Thu Feb 1 22:16:10 2007 @@ -176,8 +176,8 @@ -Python version compatibility helpers -===================================== +Cross-Python Version compatibility helpers +============================================= sources: From hpk at codespeak.net Thu Feb 1 22:18:42 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 1 Feb 2007 22:18:42 +0100 (CET) Subject: [py-svn] r37770 - py/trunk/py/path/svn Message-ID: <20070201211842.CA5F11007F@code0.codespeak.net> Author: hpk Date: Thu Feb 1 22:18:40 2007 New Revision: 37770 Removed: py/trunk/py/path/svn/xsvnbinding.py Log: remove very old try at doing svn bindings Deleted: /py/trunk/py/path/svn/xsvnbinding.py ============================================================================== --- /py/trunk/py/path/svn/xsvnbinding.py Thu Feb 1 22:18:40 2007 +++ (empty file) @@ -1,110 +0,0 @@ -""" - -module defining subversion path objects. - -SvnBindingPath implementation using svn-swig-py-bindings - -""" -# import svn swig-c-bindings - -if 0: - #import svn.client, svn.core, svn.util, atexit - #svn.util.apr_initialize() - #atexit.register(svn.util.apr_terminate) - - #import os, sys, time, re - #from common import * - #from svncommon import _cache, _repocache, SvnPathBase - - class SvnBindingPath(SvnPathBase): - pool = svn.util.svn_pool_create(None) - - def _make_rev_t(self): - rev = svn.core.svn_opt_revision_t() - if self.rev is not None: - rev.kind = svn.core.svn_opt_revision_number - rev.value.number = self.rev - return rev - - def _make_cctx(self): - cctx = svn.client.svn_client_ctx_t() - provider = svn.client.svn_client_get_simple_provider(self.pool) - cctx.auth_baton = svn.util.svn_auth_open([provider], self.pool) - return cctx - - def open(self, mode='r'): - assert 'w' not in mode and 'a' not in mode - assert self.exists() # svn cat returns an empty file otherwise - return os.popen("svn cat -r %s '%s'" % (self.rev, self.strpath)) - # XXX we don't know how to make our own stream - #from svn import client, util, core - #url = self.strpath - #rev = self._make_rev_t() - #cctx = self._make_cctx() - #stream = core.svn_stream_create(None, self.pool) - #client.svn_client_cat(stream, url, rev, cctx, self.pool) - #return stream.get_value() - - def _propget(self, name): - url = self.strpath - rev = self._make_rev_t() - cctx = self._make_cctx() - - table = svn.client.svn_client_propget(name, url, rev, 0, cctx, self.pool) - return str(table.values()[0]) - - def _proplist(self): - url = self.strpath - rev = self._make_rev_t() - cctx = self._make_cctx() - table = svn.client.svn_client_proplist(url, rev, 0, cctx, self.pool) - - if not table: - return {} - content = table[0][1] - for name, value in content.items(): - content[name] = str(value) - return content - - def exists(self): - try: - self.proplist() - except RuntimeError, e: - if e.args[0].lower().find('unknown node')!=-1: - return 0 - raise - return 1 - - def _listdir_nameinfo(self): - """ return a tuple of paths on 'self' as a directory """ - url = self.strpath - rev = self._make_rev_t() - cctx = self._make_cctx() - try: - dic = svn.client.svn_client_ls(url, rev, 0, cctx, self.pool) - except RuntimeError, e: - raise IOError(e.args) - - nameinfo_seq = map(lambda x: (x[0], InfoSvnBinding(x[1])), dic.items()) - return nameinfo_seq - - class InfoSvnBinding: - def __init__(self, _info): - self.size = _info.size - self.time = _info.time - self.last_author = _info.last_author - self.created_rev = _info.created_rev - self.has_props = _info.has_props - - if _info.kind == svn.core.svn_node_dir: - self.kind = 'dir' - elif _info.kind == svn.core.svn_node_file: - self.kind = 'file' - else: - raise ValueError, "unknown kind of svn object" - - self.mtime = self.time / 1000000 - def __eq__(self, other): - return self.__dict__ == other.__dict__ - - SvnPath = SvnBindingPath From hpk at codespeak.net Thu Feb 1 22:52:44 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 1 Feb 2007 22:52:44 +0100 (CET) Subject: [py-svn] r37772 - in py/trunk/py/io: . test Message-ID: <20070201215244.8F28610088@code0.codespeak.net> Author: hpk Date: Thu Feb 1 22:52:42 2007 New Revision: 37772 Modified: py/trunk/py/io/stdcapture.py py/trunk/py/io/test/test_capture.py Log: have both capturings have the same done/reset semantics (should also fix a buildcmodule related problem, e.g. for greenlets) Modified: py/trunk/py/io/stdcapture.py ============================================================================== --- py/trunk/py/io/stdcapture.py (original) +++ py/trunk/py/io/stdcapture.py Thu Feb 1 22:52:42 2007 @@ -22,6 +22,16 @@ return res, out, err call = classmethod(call) + def reset(self): + """ reset sys.stdout and sys.stderr + + returns a tuple of file objects (out, err) for the captured + data + """ + outfile, errfile = self.done() + return outfile.read(), errfile.read() + + class StdCaptureFD(Capture): """ capture Stdout and Stderr both on filedescriptor and sys.stdout/stderr level. @@ -40,18 +50,14 @@ if patchsys: self.err.setasfile('stderr') - def reset(self): - """ reset sys.stdout and sys.stderr - - returns a tuple of file objects (out, err) for the captured - data - """ + def done(self): + """ return (outfile, errfile) and stop capturing. """ outfile = errfile = emptyfile if hasattr(self, 'out'): outfile = self.out.done() if hasattr(self, 'err'): errfile = self.err.done() - return outfile.read(), errfile.read() + return outfile, errfile class StdCapture(Capture): """ capture sys.stdout/sys.stderr (but not system level fd 1 and 2). @@ -76,11 +82,12 @@ sys.stdin = self.newin = DontReadFromInput() def reset(self): - """ return captured output and restore sys.stdout/err.""" + """ return captured output as strings and restore sys.stdout/err.""" x, y = self.done() return x.read(), y.read() def done(self): + """ return (outfile, errfile) and stop capturing. """ o,e = sys.stdout, sys.stderr outfile = errfile = emptyfile if self._out: Modified: py/trunk/py/io/test/test_capture.py ============================================================================== --- py/trunk/py/io/test/test_capture.py (original) +++ py/trunk/py/io/test/test_capture.py Thu Feb 1 22:52:42 2007 @@ -61,7 +61,15 @@ def getcapture(self, **kw): return py.io.StdCapture(**kw) - def test_capturing_simple(self): + def test_capturing_done_simple(self): + cap = self.getcapture() + print "hello world" + print >>sys.stderr, "hello error" + outfile, errfile = cap.done() + assert outfile.read() == "hello world\n" + assert errfile.read() == "hello error\n" + + def test_capturing_reset_simple(self): cap = self.getcapture() print "hello world" print >>sys.stderr, "hello error" From cfbolz at codespeak.net Thu Feb 1 23:27:52 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 1 Feb 2007 23:27:52 +0100 (CET) Subject: [py-svn] r37773 - py/trunk/py/doc Message-ID: <20070201222752.216231008B@code0.codespeak.net> Author: cfbolz Date: Thu Feb 1 23:27:51 2007 New Revision: 37773 Modified: py/trunk/py/doc/test.txt Log: typo Modified: py/trunk/py/doc/test.txt ============================================================================== --- py/trunk/py/doc/test.txt (original) +++ py/trunk/py/doc/test.txt Thu Feb 1 23:27:51 2007 @@ -654,7 +654,7 @@ * python (2.3 or 2.4 should work) * unix like machine (reliance on ``os.fork``) -Hot to use it +How to use it ----------------------- When you issue ``py.test -d`` then your computer becomes From guido at codespeak.net Thu Feb 1 23:30:53 2007 From: guido at codespeak.net (guido at codespeak.net) Date: Thu, 1 Feb 2007 23:30:53 +0100 (CET) Subject: [py-svn] r37774 - py/trunk/py/doc Message-ID: <20070201223053.652DC1008B@code0.codespeak.net> Author: guido Date: Thu Feb 1 23:30:51 2007 New Revision: 37774 Modified: py/trunk/py/doc/conftest.py py/trunk/py/doc/test_conftest.py Log: Adding support for checking generated API links (for link roles). Modified: py/trunk/py/doc/conftest.py ============================================================================== --- py/trunk/py/doc/conftest.py (original) +++ py/trunk/py/doc/conftest.py Thu Feb 1 23:30:51 2007 @@ -15,7 +15,7 @@ ) _initialized = False -def checkdocutils(): +def checkdocutils(path): global _initialized try: import docutils @@ -23,8 +23,8 @@ py.test.skip("docutils not importable") if not _initialized: from py.__.rest import directive - directive.register_linkrole('api', resolve_linkrole) - directive.register_linkrole('source', resolve_linkrole) + directive.register_linkrole('api', get_resolve_linkrole(path)) + directive.register_linkrole('source', get_resolve_linkrole(path)) _initialized = True def restcheck(path): @@ -32,7 +32,7 @@ if hasattr(path, 'localpath'): localpath = path.localpath _checkskip(localpath) - checkdocutils() + checkdocutils(localpath) import docutils.utils try: @@ -234,24 +234,34 @@ return self.ReSTChecker(p, parent=self) Directory = DocDirectory -def resolve_linkrole(name, text): - if name == 'api': - if text == 'py': - return 'py', '../../apigen/api/index.html' - assert text.startswith('py.'), ( - 'api link "%s" does not point to the py package') % (text,) - dotted_name = text - if dotted_name.find('(') > -1: - dotted_name = dotted_name[:text.find('(')] - dotted_name = '.'.join(dotted_name.split('.')[1:]) # remove pkg root - return text, '../../apigen/api/%s.html' % (dotted_name,) - elif name == 'source': - assert text.startswith('py/'), ('source link "%s" does not point ' - 'to the py package') % (text,) - relpath = '/'.join(text.split('/')[1:]) - if relpath.endswith('/') or not relpath: - relpath += 'index.html' - else: - relpath += '.html' - return text, '../../apigen/source/%s' % (relpath,) - +def get_resolve_linkrole(checkpath=None): + # XXX yuck... + def resolve_linkrole(name, text): + if name == 'api': + if text == 'py': + ret = ('py', '../../apigen/api/index.html') + else: + assert text.startswith('py.'), ( + 'api link "%s" does not point to the py package') % (text,) + dotted_name = text + if dotted_name.find('(') > -1: + dotted_name = dotted_name[:text.find('(')] + # remove pkg root + dotted_name = '.'.join(dotted_name.split('.')[1:]) + ret = (text, '../../apigen/api/%s.html' % (dotted_name,)) + elif name == 'source': + assert text.startswith('py/'), ('source link "%s" does not point ' + 'to the py package') % (text,) + relpath = '/'.join(text.split('/')[1:]) + if relpath.endswith('/') or not relpath: + relpath += 'index.html' + else: + relpath += '.html' + ret = (text, '../../apigen/source/%s' % (relpath,)) + if checkpath: + if not py.path.local(checkpath).join(ret[1]).check(): + raise AssertionError( + '%s linkrole: %s points to non-existant path %s' % ( + name, ret[0], ret[1])) + return ret + return resolve_linkrole Modified: py/trunk/py/doc/test_conftest.py ============================================================================== --- py/trunk/py/doc/test_conftest.py (original) +++ py/trunk/py/doc/test_conftest.py Thu Feb 1 23:30:51 2007 @@ -70,7 +70,8 @@ assert len(l+l2) == 3 def test_resolve_linkrole(): - from py.__.doc.conftest import resolve_linkrole + from py.__.doc.conftest import get_resolve_linkrole + resolve_linkrole = get_resolve_linkrole(None) assert resolve_linkrole('api', 'py.foo.bar') == ( 'py.foo.bar', '../../apigen/api/foo.bar.html') assert resolve_linkrole('api', 'py.foo.bar()') == ( @@ -86,3 +87,14 @@ 'py/', '../../apigen/source/index.html') py.test.raises(AssertionError, 'resolve_linkrole("source", "/foo/bar/")') +def test_resolve_linkrole_relpath(): + from py.__.doc.conftest import get_resolve_linkrole + pypath = tmpdir.join('py') + docpath = pypath.join('doc') + apipath = tmpdir.join('apigen/api') + apipath.ensure('foo.bar.html') + resolve_linkrole = get_resolve_linkrole(docpath) + + assert resolve_linkrole('api', 'py.foo.bar') + py.test.raises(AssertionError, "resolve_linkrole('api', 'py.foo.baz')") + From guido at codespeak.net Fri Feb 2 00:12:53 2007 From: guido at codespeak.net (guido at codespeak.net) Date: Fri, 2 Feb 2007 00:12:53 +0100 (CET) Subject: [py-svn] r37776 - py/trunk/py/doc Message-ID: <20070201231253.827821008A@code0.codespeak.net> Author: guido Date: Fri Feb 2 00:12:49 2007 New Revision: 37776 Modified: py/trunk/py/doc/conftest.py py/trunk/py/doc/misc.txt py/trunk/py/doc/path.txt Log: Fixed some problems with the relative link checker (was using the wrong base path for checks), fixed links in the documents. Modified: py/trunk/py/doc/conftest.py ============================================================================== --- py/trunk/py/doc/conftest.py (original) +++ py/trunk/py/doc/conftest.py Fri Feb 2 00:12:49 2007 @@ -14,25 +14,27 @@ ) ) -_initialized = False -def checkdocutils(path): - global _initialized +def checkdocutils(): try: import docutils except ImportError: py.test.skip("docutils not importable") - if not _initialized: - from py.__.rest import directive - directive.register_linkrole('api', get_resolve_linkrole(path)) - directive.register_linkrole('source', get_resolve_linkrole(path)) - _initialized = True + +def initrestdirectives(path): + from py.__.rest import directive + dirpath = path.dirpath() + # XXX note that this gets called for every test, because the path is + # different every test... + directive.register_linkrole('api', get_resolve_linkrole(dirpath)) + directive.register_linkrole('source', get_resolve_linkrole(dirpath)) def restcheck(path): localpath = path if hasattr(path, 'localpath'): localpath = path.localpath _checkskip(localpath) - checkdocutils(localpath) + checkdocutils() + initrestdirectives(localpath) import docutils.utils try: @@ -262,6 +264,7 @@ if not py.path.local(checkpath).join(ret[1]).check(): raise AssertionError( '%s linkrole: %s points to non-existant path %s' % ( - name, ret[0], ret[1])) + name, ret[0], py.path.local(checkpath).join(ret[1]))) return ret return resolve_linkrole + Modified: py/trunk/py/doc/misc.txt ============================================================================== --- py/trunk/py/doc/misc.txt (original) +++ py/trunk/py/doc/misc.txt Fri Feb 2 00:12:49 2007 @@ -93,7 +93,8 @@ Currently, the py lib offers two ways to interact with system executables. :api:`py.process.cmdexec()` invokes the shell in order to execute a string. The other -one, :api:`py.path.local.sysexec()` lets you directly execute a binary. +one, :api:`py.path.local`'s 'sysexec()' method lets you +directly execute a binary. Both approaches will raise an exception in case of a return- code other than 0 and otherwise return the stdout-output @@ -217,3 +218,4 @@ module) :api:`py.builtin.BaseException` is just ``Exception`` before Python 2.5. + Modified: py/trunk/py/doc/path.txt ============================================================================== --- py/trunk/py/doc/path.txt (original) +++ py/trunk/py/doc/path.txt Fri Feb 2 00:12:49 2007 @@ -209,10 +209,10 @@ of platform-dependencies as much as possible). There is some experimental small approach -(:source:`py/path/gateway`) aiming at having +(:source:`py/path/gateway/`) aiming at having a convenient Remote Path implementation and some considerations about future -works in the according :source:`py/path/gateway/todo.txt` +works in the according :source:`py/path/gateway/TODO.txt` There are various hacks out there to have Memory-Filesystems and even path objects From fijal at codespeak.net Fri Feb 2 00:41:01 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Feb 2007 00:41:01 +0100 (CET) Subject: [py-svn] r37779 - in py/trunk/py/execnet: . testing Message-ID: <20070201234101.CBE0110075@code0.codespeak.net> Author: fijal Date: Fri Feb 2 00:40:49 2007 New Revision: 37779 Modified: py/trunk/py/execnet/rsync.py py/trunk/py/execnet/rsync_remote.py py/trunk/py/execnet/testing/test_rsync.py Log: Fix and a test for disappearing files. Modified: py/trunk/py/execnet/rsync.py ============================================================================== --- py/trunk/py/execnet/rsync.py (original) +++ py/trunk/py/execnet/rsync.py Fri Feb 2 00:40:49 2007 @@ -85,26 +85,33 @@ elif command == "send": modified_rel_path, checksum = data modifiedpath = os.path.join(self.sourcedir, *modified_rel_path) - f = open(modifiedpath, 'rb') - data = f.read() + try: + f = open(modifiedpath, 'rb') + data = f.read() + except IOError: + data = None # provide info to progress callback function modified_rel_path = "/".join(modified_rel_path) - self.paths[modified_rel_path] = len(data) + if data is not None: + self.paths[modified_rel_path] = len(data) + else: + self.paths[modified_rel_path] = 0 if channel not in self.to_send: self.to_send[channel] = [] self.to_send[channel].append(modified_rel_path) - f.close() - if checksum is not None and checksum == md5.md5(data).digest(): - data = None # not really modified - else: - # ! there is a reason for the interning: - # sharing multiple copies of the file's data - data = intern(data) - print '%s <= %s' % ( - channel.gateway._getremoteaddress(), - modified_rel_path) + if data is not None: + f.close() + if checksum is not None and checksum == md5.md5(data).digest(): + data = None # not really modified + else: + # ! there is a reason for the interning: + # sharing multiple copies of the file's data + data = intern(data) + print '%s <= %s' % ( + channel.gateway._getremoteaddress(), + modified_rel_path) channel.send(data) del data else: @@ -118,7 +125,11 @@ self.links.append(("link", basename, linkpoint)) def _send_directory_structure(self, path): - st = os.lstat(path) + try: + st = os.lstat(path) + except OSError: + self._broadcast((0, 0)) + return if stat.S_ISREG(st.st_mode): # regular file: send a timestamp/size pair self._broadcast((st.st_mtime, st.st_size)) Modified: py/trunk/py/execnet/rsync_remote.py ============================================================================== --- py/trunk/py/execnet/rsync_remote.py (original) +++ py/trunk/py/execnet/rsync_remote.py Fri Feb 2 00:40:49 2007 @@ -65,7 +65,10 @@ f = open(path, 'wb') f.write(data) f.close() - os.utime(path, (time, time)) + try: + os.utime(path, (time, time)) + except OSError: + pass del data channel.send(("links", None)) Modified: py/trunk/py/execnet/testing/test_rsync.py ============================================================================== --- py/trunk/py/execnet/testing/test_rsync.py (original) +++ py/trunk/py/execnet/testing/test_rsync.py Fri Feb 2 00:40:49 2007 @@ -78,3 +78,24 @@ assert total == {("list", 110):True, ("ack", 100):True, ("ack", 10):True} +def test_file_disappearing(): + base = py.test.ensuretemp("file_disappearing") + dest = base.join("dest") + source = base.join("source") + source.ensure("ex").write("a" * 100) + source.ensure("ex2").write("a" * 100) + + class DRsync(RSync): + def filter(self, x): + if x.endswith("ex2"): + self.x = 1 + source.join("ex2").remove() + return True + + rsync = DRsync() + rsync.add_target(gw, dest) + rsync.send(source) + assert rsync.x == 1 + assert len(dest.listdir()) == 1 + assert len(source.listdir()) == 1 + From hpk at codespeak.net Fri Feb 2 00:58:10 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 2 Feb 2007 00:58:10 +0100 (CET) Subject: [py-svn] r37780 - in py/trunk/py/execnet: . testing Message-ID: <20070201235810.3D16410078@code0.codespeak.net> Author: hpk Date: Fri Feb 2 00:57:55 2007 New Revision: 37780 Modified: py/trunk/py/execnet/gateway.py py/trunk/py/execnet/register.py py/trunk/py/execnet/rsync.py py/trunk/py/execnet/testing/test_gateway.py Log: simplifying code a bit (but test_confusion* for Ssh still fails) Modified: py/trunk/py/execnet/gateway.py ============================================================================== --- py/trunk/py/execnet/gateway.py (original) +++ py/trunk/py/execnet/gateway.py Fri Feb 2 00:57:55 2007 @@ -31,12 +31,12 @@ sysex = (KeyboardInterrupt, SystemExit) class Gateway(object): - num_worker_threads = 2 ThreadOut = ThreadOut + remoteaddress = "" def __init__(self, io, startcount=2, maxthreads=None): global registered_cleanup - self._execpool = WorkerPool() + self._execpool = WorkerPool(maxthreads=maxthreads) ## self.running = True self.io = io self._outgoing = Queue.Queue() @@ -50,7 +50,7 @@ sender = self.thread_sender) def __repr__(self): - addr = self._getremoteaddress() + addr = self.remoteaddress if addr: addr = '[%s]' % (addr,) else: @@ -67,9 +67,6 @@ return "<%s%s %s/%s (%s active channels)>" %( self.__class__.__name__, addr, r, s, i) - def _getremoteaddress(self): - return None - ## def _local_trystopexec(self): ## self._execpool.shutdown() Modified: py/trunk/py/execnet/register.py ============================================================================== --- py/trunk/py/execnet/register.py (original) +++ py/trunk/py/execnet/register.py Fri Feb 2 00:57:55 2007 @@ -136,9 +136,7 @@ sock.connect((host, port)) io = inputoutput.SocketIO(sock) super(SocketGateway, self).__init__(io=io) - - def _getremoteaddress(self): - return '%s:%d' % (self.host, self.port) + self.remoteaddress = '%s:%d' % (self.host, self.port) def remote_install(cls, gateway, hostport=None): """ return a connected socket gateway through the @@ -167,7 +165,7 @@ class SshGateway(PopenCmdGateway): def __init__(self, sshaddress, remotepython='python', identity=None): - self.sshaddress = sshaddress + self.remoteaddress = sshaddress remotecmd = '%s -u -c "exec input()"' % (remotepython,) cmdline = [sshaddress, remotecmd] # XXX Unix style quoting @@ -179,9 +177,6 @@ cmdline.insert(0, cmd) super(SshGateway, self).__init__(' '.join(cmdline)) - def _getremoteaddress(self): - return self.sshaddress - class ExecGateway(PopenGateway): def remote_exec_sync_stdcapture(self, lines, callback): # hack: turn the content of the cell into Modified: py/trunk/py/execnet/rsync.py ============================================================================== --- py/trunk/py/execnet/rsync.py (original) +++ py/trunk/py/execnet/rsync.py Fri Feb 2 00:57:55 2007 @@ -110,7 +110,7 @@ # sharing multiple copies of the file's data data = intern(data) print '%s <= %s' % ( - channel.gateway._getremoteaddress(), + channel.gateway.remoteaddress, modified_rel_path) channel.send(data) del data Modified: py/trunk/py/execnet/testing/test_gateway.py ============================================================================== --- py/trunk/py/execnet/testing/test_gateway.py (original) +++ py/trunk/py/execnet/testing/test_gateway.py Fri Feb 2 00:57:55 2007 @@ -62,8 +62,8 @@ def setup_class(cls): cls.gw = py.execnet.PopenGateway() -## def teardown_class(cls): -## cls.gw.exit() + def teardown_class(cls): + cls.gw.exit() class BasicRemoteExecution: def test_correct_setup(self): @@ -455,7 +455,7 @@ cls.gw = py.execnet.SshGateway(option.sshtarget) def test_sshaddress(self): - assert self.gw.sshaddress == option.sshtarget + assert self.gw.remoteaddress == option.sshtarget def test_failed_connexion(self): gw = py.execnet.SshGateway('nowhere.codespeak.net') From guido at codespeak.net Fri Feb 2 00:59:02 2007 From: guido at codespeak.net (guido at codespeak.net) Date: Fri, 2 Feb 2007 00:59:02 +0100 (CET) Subject: [py-svn] r37781 - py/trunk/py/apigen Message-ID: <20070201235902.81B5E1007E@code0.codespeak.net> Author: guido Date: Fri Feb 2 00:58:57 2007 New Revision: 37781 Modified: py/trunk/py/apigen/apigen.py Log: Added 'execnet.channel.Channel' object to the to-be-documented items. Modified: py/trunk/py/apigen/apigen.py ============================================================================== --- py/trunk/py/apigen/apigen.py (original) +++ py/trunk/py/apigen/apigen.py Fri Feb 2 00:58:57 2007 @@ -15,7 +15,9 @@ def get_documentable_items(pkgdir): sys.path.insert(0, str(pkgdir.dirpath())) rootmod = __import__(pkgdir.basename) - return 'py', pkg_to_dict(rootmod) + d = pkg_to_dict(rootmod) + d['execnet.Channel'] = py.__.execnet.channel.Channel + return 'py', d def build(pkgdir, dsa, capture): l = linker.Linker() From guido at codespeak.net Fri Feb 2 01:01:06 2007 From: guido at codespeak.net (guido at codespeak.net) Date: Fri, 2 Feb 2007 01:01:06 +0100 (CET) Subject: [py-svn] r37782 - py/trunk/py/doc Message-ID: <20070202000106.6B5DB10077@code0.codespeak.net> Author: guido Date: Fri Feb 2 01:00:50 2007 New Revision: 37782 Modified: py/trunk/py/doc/conftest.py py/trunk/py/doc/test_conftest.py Log: Changed the checking in resolve_linkrole() so that instead of the apigen results (the directory with HTML files) it uses the py lib object tree and source tree to find out whether links are valid. Modified: py/trunk/py/doc/conftest.py ============================================================================== --- py/trunk/py/doc/conftest.py (original) +++ py/trunk/py/doc/conftest.py Fri Feb 2 01:00:50 2007 @@ -14,19 +14,18 @@ ) ) +_initialized = False def checkdocutils(): + global _initialized try: import docutils except ImportError: py.test.skip("docutils not importable") - -def initrestdirectives(path): - from py.__.rest import directive - dirpath = path.dirpath() - # XXX note that this gets called for every test, because the path is - # different every test... - directive.register_linkrole('api', get_resolve_linkrole(dirpath)) - directive.register_linkrole('source', get_resolve_linkrole(dirpath)) + if not _initialized: + from py.__.rest import directive + directive.register_linkrole('api', resolve_linkrole) + directive.register_linkrole('source', resolve_linkrole) + _initialized = True def restcheck(path): localpath = path @@ -34,7 +33,6 @@ localpath = path.localpath _checkskip(localpath) checkdocutils() - initrestdirectives(localpath) import docutils.utils try: @@ -236,35 +234,42 @@ return self.ReSTChecker(p, parent=self) Directory = DocDirectory -def get_resolve_linkrole(checkpath=None): - # XXX yuck... - def resolve_linkrole(name, text): - if name == 'api': - if text == 'py': - ret = ('py', '../../apigen/api/index.html') - else: - assert text.startswith('py.'), ( - 'api link "%s" does not point to the py package') % (text,) - dotted_name = text - if dotted_name.find('(') > -1: - dotted_name = dotted_name[:text.find('(')] - # remove pkg root - dotted_name = '.'.join(dotted_name.split('.')[1:]) - ret = (text, '../../apigen/api/%s.html' % (dotted_name,)) - elif name == 'source': - assert text.startswith('py/'), ('source link "%s" does not point ' - 'to the py package') % (text,) - relpath = '/'.join(text.split('/')[1:]) - if relpath.endswith('/') or not relpath: - relpath += 'index.html' - else: - relpath += '.html' - ret = (text, '../../apigen/source/%s' % (relpath,)) - if checkpath: - if not py.path.local(checkpath).join(ret[1]).check(): - raise AssertionError( - '%s linkrole: %s points to non-existant path %s' % ( - name, ret[0], py.path.local(checkpath).join(ret[1]))) - return ret - return resolve_linkrole +def resolve_linkrole(name, text, check=True): + if name == 'api': + if text == 'py': + return ('py', '../../apigen/api/index.html') + else: + assert text.startswith('py.'), ( + 'api link "%s" does not point to the py package') % (text,) + dotted_name = text + if dotted_name.find('(') > -1: + dotted_name = dotted_name[:text.find('(')] + # remove pkg root + path = dotted_name.split('.')[1:] + dotted_name = '.'.join(path) + obj = py + if check: + for chunk in path: + try: + obj = getattr(obj, chunk) + except AttributeError: + raise AssertionError( + 'problem with linkrole :api:`%s`: can not resolve ' + 'dotted name %s' % (text, dotted_name,)) + return (text, '../../apigen/api/%s.html' % (dotted_name,)) + elif name == 'source': + assert text.startswith('py/'), ('source link "%s" does not point ' + 'to the py package') % (text,) + relpath = '/'.join(text.split('/')[1:]) + if check: + pkgroot = py.__package__.getpath() + abspath = pkgroot.join(relpath) + assert pkgroot.join(relpath).check(), ( + 'problem with linkrole :source:`%s`: ' + 'path %s does not exist' % (text, relpath)) + if relpath.endswith('/') or not relpath: + relpath += 'index.html' + else: + relpath += '.html' + return (text, '../../apigen/source/%s' % (relpath,)) Modified: py/trunk/py/doc/test_conftest.py ============================================================================== --- py/trunk/py/doc/test_conftest.py (original) +++ py/trunk/py/doc/test_conftest.py Fri Feb 2 01:00:50 2007 @@ -70,31 +70,30 @@ assert len(l+l2) == 3 def test_resolve_linkrole(): - from py.__.doc.conftest import get_resolve_linkrole - resolve_linkrole = get_resolve_linkrole(None) - assert resolve_linkrole('api', 'py.foo.bar') == ( + from py.__.doc.conftest import resolve_linkrole + assert resolve_linkrole('api', 'py.foo.bar', False) == ( 'py.foo.bar', '../../apigen/api/foo.bar.html') - assert resolve_linkrole('api', 'py.foo.bar()') == ( + assert resolve_linkrole('api', 'py.foo.bar()', False) == ( 'py.foo.bar()', '../../apigen/api/foo.bar.html') - assert resolve_linkrole('api', 'py') == ( + assert resolve_linkrole('api', 'py', False) == ( 'py', '../../apigen/api/index.html') py.test.raises(AssertionError, 'resolve_linkrole("api", "foo.bar")') - assert resolve_linkrole('source', 'py/foo/bar.py') == ( + assert resolve_linkrole('source', 'py/foo/bar.py', False) == ( 'py/foo/bar.py', '../../apigen/source/foo/bar.py.html') - assert resolve_linkrole('source', 'py/foo/') == ( + assert resolve_linkrole('source', 'py/foo/', False) == ( 'py/foo/', '../../apigen/source/foo/index.html') - assert resolve_linkrole('source', 'py/') == ( + assert resolve_linkrole('source', 'py/', False) == ( 'py/', '../../apigen/source/index.html') py.test.raises(AssertionError, 'resolve_linkrole("source", "/foo/bar/")') -def test_resolve_linkrole_relpath(): - from py.__.doc.conftest import get_resolve_linkrole - pypath = tmpdir.join('py') - docpath = pypath.join('doc') - apipath = tmpdir.join('apigen/api') - apipath.ensure('foo.bar.html') - resolve_linkrole = get_resolve_linkrole(docpath) - - assert resolve_linkrole('api', 'py.foo.bar') +def test_resolve_linkrole_check_api(): + from py.__.doc.conftest import resolve_linkrole + assert resolve_linkrole('api', 'py.test.ensuretemp') py.test.raises(AssertionError, "resolve_linkrole('api', 'py.foo.baz')") +def test_resolve_linkrole_check_source(): + from py.__.doc.conftest import resolve_linkrole + assert resolve_linkrole('source', 'py/path/common.py') + py.test.raises(AssertionError, + "resolve_linkrole('source', 'py/foo/bar.py')") + From hpk at codespeak.net Fri Feb 2 01:32:27 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 2 Feb 2007 01:32:27 +0100 (CET) Subject: [py-svn] r37783 - py/trunk/py/execnet/testing Message-ID: <20070202003227.9CD3910077@code0.codespeak.net> Author: hpk Date: Fri Feb 2 01:32:24 2007 New Revision: 37783 Modified: py/trunk/py/execnet/testing/test_gateway.py Log: puh, it took me quite a while to find out why ssh-stdout-confusion tests involving "os.write(1, ...)" were passing on earlier revisions: in 35888 a test name got fixed, but when you specify "-S somehost" then this will hang, so i am skipping the test for now, this never worked. Modified: py/trunk/py/execnet/testing/test_gateway.py ============================================================================== --- py/trunk/py/execnet/testing/test_gateway.py (original) +++ py/trunk/py/execnet/testing/test_gateway.py Fri Feb 2 01:32:24 2007 @@ -454,6 +454,9 @@ py.test.skip("no known ssh target, use -S to set one") cls.gw = py.execnet.SshGateway(option.sshtarget) + def test_confusion_from_os_write_stdout(self): + py.test.skip("writing to FD 1 with SshGateways not supported yet") + def test_sshaddress(self): assert self.gw.remoteaddress == option.sshtarget From hpk at codespeak.net Fri Feb 2 01:34:41 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 2 Feb 2007 01:34:41 +0100 (CET) Subject: [py-svn] r37784 - py/trunk/py/execnet Message-ID: <20070202003441.D89641007E@code0.codespeak.net> Author: hpk Date: Fri Feb 2 01:34:40 2007 New Revision: 37784 Modified: py/trunk/py/execnet/gateway.py Log: privatizing some attributes Modified: py/trunk/py/execnet/gateway.py ============================================================================== --- py/trunk/py/execnet/gateway.py (original) +++ py/trunk/py/execnet/gateway.py Fri Feb 2 01:34:40 2007 @@ -31,7 +31,7 @@ sysex = (KeyboardInterrupt, SystemExit) class Gateway(object): - ThreadOut = ThreadOut + _ThreadOut = ThreadOut remoteaddress = "" def __init__(self, io, startcount=2, maxthreads=None): @@ -46,8 +46,8 @@ atexit.register(cleanup_atexit) registered_cleanup = True _active_sendqueues[self._outgoing] = True - self.pool = NamedThreadPool(receiver = self.thread_receiver, - sender = self.thread_sender) + self.pool = NamedThreadPool(receiver = self._thread_receiver, + sender = self._thread_sender) def __repr__(self): addr = self.remoteaddress @@ -92,7 +92,7 @@ excinfo[1]) self._trace(errortext) - def thread_receiver(self): + def _thread_receiver(self): """ thread to read and handle Messages half-sync-half-async. """ try: from sys import exc_info @@ -113,7 +113,7 @@ self.channelfactory._finished_receiving() self._trace('leaving %r' % threading.currentThread()) - def thread_sender(self): + def _thread_sender(self): """ thread to send Messages over the wire. """ try: from sys import exc_info @@ -141,7 +141,7 @@ for name, id in ('stdout', outid), ('stderr', errid): if id: channel = self.channelfactory.new(outid) - out = ThreadOut(sys, name) + out = self._ThreadOut(sys, name) out.setwritefunc(channel.send) l.append((out, channel)) def close(): @@ -150,7 +150,7 @@ channel.close() return close - def thread_executor(self, channel, (source, outid, errid)): + def _thread_executor(self, channel, (source, outid, errid)): """ worker thread to execute source objects from the execution queue. """ from sys import exc_info try: @@ -177,7 +177,7 @@ def _local_schedulexec(self, channel, sourcetask): self._trace("dispatching exec") - self._execpool.dispatch(self.thread_executor, channel, sourcetask) + self._execpool.dispatch(self._thread_executor, channel, sourcetask) def _newredirectchannelid(self, callback): if callback is None: @@ -231,7 +231,7 @@ channel = self.remote_exec(""" import sys outchannel = channel.receive() - outchannel.gateway.ThreadOut(sys, %r).setdefaultwriter(outchannel.send) + outchannel.gateway._ThreadOut(sys, %r).setdefaultwriter(outchannel.send) """ % name) channel.send(outchannel) clist.append(channel) @@ -243,7 +243,7 @@ if out: c = self.remote_exec(""" import sys - channel.gateway.ThreadOut(sys, %r).resetdefault() + channel.gateway._ThreadOut(sys, %r).resetdefault() """ % name) c.waitclose(1.0) return Handle() From hpk at codespeak.net Fri Feb 2 02:02:57 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 2 Feb 2007 02:02:57 +0100 (CET) Subject: [py-svn] r37785 - in py/trunk/py/execnet: . testing Message-ID: <20070202010257.BF85110080@code0.codespeak.net> Author: hpk Date: Fri Feb 2 02:02:55 2007 New Revision: 37785 Modified: py/trunk/py/execnet/channel.py py/trunk/py/execnet/gateway.py py/trunk/py/execnet/message.py py/trunk/py/execnet/register.py py/trunk/py/execnet/testing/test_gateway.py Log: privatizing some more attributes Modified: py/trunk/py/execnet/channel.py ============================================================================== --- py/trunk/py/execnet/channel.py (original) +++ py/trunk/py/execnet/channel.py Fri Feb 2 02:02:55 2007 @@ -37,10 +37,10 @@ def setcallback(self, callback, endmarker=NO_ENDMARKER_WANTED): queue = self._items - lock = self.gateway.channelfactory._receivelock + lock = self.gateway._channelfactory._receivelock lock.acquire() try: - _callbacks = self.gateway.channelfactory._callbacks + _callbacks = self.gateway._channelfactory._callbacks dictvalue = (callback, endmarker) if _callbacks.setdefault(self.id, dictvalue) != dictvalue: raise IOError("%r has callback already registered" %(self,)) @@ -58,7 +58,7 @@ callback(olditem) if self._closed or self._receiveclosed.isSet(): # no need to keep a callback - self.gateway.channelfactory._close_callback(self.id) + self.gateway._channelfactory._close_callback(self.id) finally: lock.release() @@ -129,7 +129,7 @@ queue = self._items if queue is not None: queue.put(ENDMARKER) - self.gateway.channelfactory._no_longer_opened(self.id) + self.gateway._channelfactory._no_longer_opened(self.id) def waitclose(self, timeout=None): """ wait until this channel is closed (or the remote side Modified: py/trunk/py/execnet/gateway.py ============================================================================== --- py/trunk/py/execnet/gateway.py (original) +++ py/trunk/py/execnet/gateway.py Fri Feb 2 02:02:55 2007 @@ -10,7 +10,7 @@ # other modules) execute not only on the local side but # also on any gateway's remote side. On such remote sides # we cannot assume the py library to be there and -# InstallableGateway.remote_bootstrap_gateway() (located +# InstallableGateway._remote_bootstrap_gateway() (located # in register.py) will take care to send source fragments # to the other side. Yes, it is fragile but we have a # few tests that try to catch when we mess up. @@ -38,15 +38,15 @@ global registered_cleanup self._execpool = WorkerPool(maxthreads=maxthreads) ## self.running = True - self.io = io + self._io = io self._outgoing = Queue.Queue() - self.channelfactory = ChannelFactory(self, startcount) + self._channelfactory = ChannelFactory(self, startcount) ## self._exitlock = threading.Lock() if not registered_cleanup: atexit.register(cleanup_atexit) registered_cleanup = True _active_sendqueues[self._outgoing] = True - self.pool = NamedThreadPool(receiver = self._thread_receiver, + self._pool = NamedThreadPool(receiver = self._thread_receiver, sender = self._thread_sender) def __repr__(self): @@ -56,11 +56,11 @@ else: addr = '' try: - r = (len(self.pool.getstarted('receiver')) + r = (len(self._pool.getstarted('receiver')) and "receiving" or "not receiving") - s = (len(self.pool.getstarted('sender')) + s = (len(self._pool.getstarted('sender')) and "sending" or "not sending") - i = len(self.channelfactory.channels()) + i = len(self._channelfactory.channels()) except AttributeError: r = s = "uninitialized" i = "no" @@ -98,7 +98,7 @@ from sys import exc_info while 1: try: - msg = Message.readfrom(self.io) + msg = Message.readfrom(self._io) self._trace("received <- %r" % msg) msg.received(self) except sysex: @@ -110,7 +110,7 @@ break finally: self._outgoing.put(None) - self.channelfactory._finished_receiving() + self._channelfactory._finished_receiving() self._trace('leaving %r' % threading.currentThread()) def _thread_sender(self): @@ -121,9 +121,9 @@ msg = self._outgoing.get() try: if msg is None: - self.io.close_write() + self._io.close_write() break - msg.writeto(self.io) + msg.writeto(self._io) except: excinfo = exc_info() self._traceex(excinfo) @@ -140,7 +140,7 @@ l = [] for name, id in ('stdout', outid), ('stderr', errid): if id: - channel = self.channelfactory.new(outid) + channel = self._channelfactory.new(outid) out = self._ThreadOut(sys, name) out.setwritefunc(channel.send) l.append((out, channel)) @@ -196,7 +196,7 @@ # def newchannel(self): """ return new channel object. """ - return self.channelfactory.new() + return self._channelfactory.new() def remote_exec(self, source, stdout=None, stderr=None): """ return channel object for communicating with the asynchronously @@ -218,7 +218,7 @@ (source, outid, errid))) return channel - def remote_redirect(self, stdout=None, stderr=None): + def _remote_redirect(self, stdout=None, stderr=None): """ return a handle representing a redirection of a remote end's stdout to a local file object. with handle.close() the redirection will be reverted. @@ -262,7 +262,7 @@ ## try: ## if self.running: ## self.running = False -## if not self.pool.getstarted('sender'): +## if not self._pool.getstarted('sender'): ## raise IOError("sender thread not alive anymore!") ## self._outgoing.put(None) ## self._trace("exit procedure triggered, pid %d " % (os.getpid(),)) @@ -279,7 +279,7 @@ def join(self, joinexec=True): current = threading.currentThread() - for x in self.pool.getstarted(): + for x in self._pool.getstarted(): if x != current: self._trace("joining %s" % x) x.join() Modified: py/trunk/py/execnet/message.py ============================================================================== --- py/trunk/py/execnet/message.py (original) +++ py/trunk/py/execnet/message.py Fri Feb 2 02:02:55 2007 @@ -62,32 +62,32 @@ class CHANNEL_OPEN(Message): def received(self, gateway): - channel = gateway.channelfactory.new(self.channelid) + channel = gateway._channelfactory.new(self.channelid) gateway._local_schedulexec(channel=channel, sourcetask=self.data) class CHANNEL_NEW(Message): def received(self, gateway): """ receive a remotely created new (sub)channel. """ newid = self.data - newchannel = gateway.channelfactory.new(newid) - gateway.channelfactory._local_receive(self.channelid, newchannel) + newchannel = gateway._channelfactory.new(newid) + gateway._channelfactory._local_receive(self.channelid, newchannel) class CHANNEL_DATA(Message): def received(self, gateway): - gateway.channelfactory._local_receive(self.channelid, self.data) + gateway._channelfactory._local_receive(self.channelid, self.data) class CHANNEL_CLOSE(Message): def received(self, gateway): - gateway.channelfactory._local_close(self.channelid) + gateway._channelfactory._local_close(self.channelid) class CHANNEL_CLOSE_ERROR(Message): def received(self, gateway): - remote_error = gateway.channelfactory.RemoteError(self.data) - gateway.channelfactory._local_close(self.channelid, remote_error) + remote_error = gateway._channelfactory.RemoteError(self.data) + gateway._channelfactory._local_close(self.channelid, remote_error) class CHANNEL_LAST_MESSAGE(Message): def received(self, gateway): - gateway.channelfactory._local_last_message(self.channelid) + gateway._channelfactory._local_last_message(self.channelid) classes = [x for x in locals().values() if hasa