#! /usr/bin/env python """fdb.py ... Command-line interface to a db file index. For now its location is determined by the environment variable FILESDBDIR, which should initially point to an empty directory. Commands: link filename [filename...] Add or update an index entry for each of the files. This command makes links, without copying the data into the db. check filename [filename...] Look up the entry of each the files, without modifying the db. store filename [filename...] Store a copy of the files into the db. Files with identical content are always stored only once. Use '-' to read from stdin. find md5 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx find sha xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Print the entry for the file with the given MD5 or SHA checksum. The keyword 'md5' or 'sha' can be omitted (as the checksums don't have the same length anyway). cat md5 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx cat sha xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Cat the data of the file with the given MD5 or SHA checksum. The keyword 'md5' or 'sha' can be omitted. """ import sys, os import md5, sha from filesdb import FilesDB def cmd_link(fdb, *filenames): for filename in filenames: entry = fdb.add_link(filename) print str(entry) def cmd_check(fdb, *filenames): error = False for filename in filenames: try: entry = fdb.get_entry(filename) except KeyError: print >> sys.stderr, 'not found:'.ljust(52), filename error = True else: print str(entry) if error: sys.exit(1) def cmd_store(fdb, *filenames): if filenames == ('-',): entry = fdb.import_data(sys.stdin.read()) print str(entry) else: for filename in filenames: entry = fdb.import_file(filename) print str(entry) def parse_checksum(*args): if not args: error("not enough arguments") if len(args) > 2: error("too many arguments") if len(args[-1]) == 2*md5.digest_size: if len(args) == 2 and args[0].lower() != 'md5': error("invalid checksum") return 'md5', args[-1].decode('hex') elif len(args[-1]) == 2*sha.digest_size: if len(args) == 2 and args[0].lower() != 'sha': error("invalid checksum") return 'sha', args[-1].decode('hex') else: error("invalid checksum") def cmd_find(fdb, *args): kind, sum = parse_checksum(*args) try: entry = getattr(fdb, 'find_' + kind)(sum) except KeyError: print >> sys.stderr, 'not found' sys.exit(1) else: print str(entry) def cmd_cat(fdb, *args): kind, sum = parse_checksum(*args) try: entry = getattr(fdb, 'find_' + kind)(sum) except KeyError: error('not found') else: f = entry.open() while True: block = f.read(32768) if not block: break sys.stdout.write(block) f.close() def cmd_publish(fdb, *filenames): from py.execnet import SshGateway from pubfile import PubFile if not filenames: filenames = [] for basename in os.listdir(fdb.dbpath): if basename.endswith('.pub'): filenames.append(os.path.join(fdb.dbpath, basename)) if not filenames: error("no .pub file in $FILESDBDIR") for filename in filenames: assert filename.endswith('.pub') p = PubFile(filename) target = p['Metaserver'] hostname, remotepath = target.split(':') print 'Publishing %s to %s...' % (os.path.basename(filename), target) gw = SshGateway(hostname) channel = gw.remote_exec(""" import os tmpext = '~' path, rootname, pubtext = channel.receive() filename = os.path.join(path, rootname + '.pub') tmp = filename + tmpext f = open(tmp, 'w') f.write(pubtext) f.close() os.rename(tmp, filename) filename = os.path.join(path, rootname + '.sha') tmp = filename + tmpext f = open(tmp, 'wb') for blob in channel: if blob is None: break f.write(blob) f.close() os.rename(tmp, filename) channel.send('ok') """) f = open(filename) pubtext = f.read() f.close() root, ext = os.path.splitext(os.path.basename(filename)) channel.send((remotepath, root, pubtext)) blob = [] for key in fdb.shatable: if len(blob) == 512: channel.send(''.join(blob)) blob = [] blob.append(key) blob.append('>>IndexDB') channel.send(''.join(blob)) channel.send(None) answer = channel.receive() assert answer == 'ok' print 'done.' def help(): print >> sys.stderr, __doc__ sys.exit(2) def error(msg): print >> sys.stderr, '%s: %s' % (sys.argv[0], msg) sys.exit(1) def main(): if len(sys.argv) <= 1: help() cmd = sys.argv[1].lstrip('-') if cmd == 'help': help() dbpath = os.environ.get('FILESDBDIR') if not dbpath: error('$FILESDBDIR not set') if not os.path.isdir(dbpath): error('$FILESDBDIR=%s: not a dir' % (dbpath,)) func = globals().get('cmd_' + cmd) if not func: error('Unknown command: %r' % (cmd,)) func(FilesDB(dbpath), *sys.argv[2:]) if __name__ == '__main__': main()