import py, os, stat class RSync(object): def __init__(self, **options): for name in options: assert name in ('delete',) self.options = options self.channels = [] def filter(self, path): return True def add_target(self, gateway, destdir): channel = gateway.remote_exec(REMOTE_SOURCE) channel.send((str(destdir), self.options)) self.channels.append(channel) def send(self, sourcedir): sourcedir = str(sourcedir) # send directory structure and file timestamps/sizes self._send_directory_structure(sourcedir) # send modified file in a round-robin fashion to clients # XXX good enough for now :-( while self.channels: for channel in self.channels: try: modified_rel_path = channel.receive() except EOFError: self.channels.remove(channel) continue modifiedpath = os.path.join(sourcedir, *modified_rel_path) f = open(modifiedpath, 'rb') data = f.read() f.close() channel.send(data) del data def _broadcast(self, msg): for channel in self.channels: channel.send(msg) def _send_directory_structure(self, path): st = os.lstat(path) if stat.S_ISREG(st.st_mode): # regular file: send a timestamp/size pair self._broadcast((st.st_mtime, st.st_size)) elif stat.S_ISDIR(st.st_mode): # dir: send a list of entries names = [] subpaths = [] for name in os.listdir(path): p = os.path.join(path, name) if self.filter(p): names.append(name) subpaths.append(p) self._broadcast(names) for p in subpaths: self._send_directory_structure(p) elif stat.S_ISLNK(st.st_mode): pass # ignore symlinks for now else: raise ValueError, "cannot sync %r" % (path,) REMOTE_SOURCE = """ import os, stat, shutil destdir, options = channel.receive() modifiedfiles = [] def remove(path): assert path.startswith(destdir) try: os.unlink(path) except OSError: # assume it's a dir shutil.rmtree(path) def receive_directory_structure(path, relcomponents): try: st = os.lstat(path) except OSError: st = None msg = channel.receive() if isinstance(msg, list): if st and not stat.S_ISDIR(st.st_mode): os.unlink(path) st = None if not st: os.mkdir(path) entrynames = {} for entryname in msg: receive_directory_structure(os.path.join(path, entryname), relcomponents + [entryname]) entrynames[entryname] = True if options.get('delete'): for othername in os.listdir(path): if othername not in entrynames: otherpath = os.path.join(path, othername) remove(otherpath) else: if st and stat.S_ISREG(st.st_mode): mystamp = (st.st_mtime, st.st_size) else: if st: remove(path) mystamp = None if mystamp != msg: channel.send(relcomponents) modifiedfiles.append(path) receive_directory_structure(destdir, []) for path in modifiedfiles: data = channel.receive() f = open(path, 'wb') f.write(data) f.close() del data """ # ____________________________________________________________ def setup_module(mod): mod.gw = py.execnet.PopenGateway() mod.gw2 = py.execnet.PopenGateway() def teardown_module(mod): mod.gw.exit() mod.gw2.exit() def test_dirsync(): base = py.test.ensuretemp('dirsync') dest = base.join('dest') dest2 = base.join('dest2') source = base.mkdir('source') for s in ('content1', 'content2-a-bit-longer'): source.ensure('subdir', 'file1').write(s) rsync = RSync() rsync.add_target(gw, dest) rsync.add_target(gw2, dest2) rsync.send(source) assert dest.join('subdir').check(dir=1) assert dest.join('subdir', 'file1').check(file=1) assert dest.join('subdir', 'file1').read() == s assert dest2.join('subdir').check(dir=1) assert dest2.join('subdir', 'file1').check(file=1) assert dest2.join('subdir', 'file1').read() == s source.join('subdir').remove('file1') rsync = RSync() rsync.add_target(gw2, dest2) rsync.add_target(gw, dest) rsync.send(source) assert dest.join('subdir', 'file1').check(file=1) assert dest2.join('subdir', 'file1').check(file=1) rsync = RSync(delete=True) rsync.add_target(gw2, dest2) rsync.add_target(gw, dest) rsync.send(source) assert not dest.join('subdir', 'file1').check() assert not dest2.join('subdir', 'file1').check()