Codeville-0.8.0/0000775000076400007640000000000010645744720012766 5ustar rcohenrcohenCodeville-0.8.0/Codeville/0000775000076400007640000000000010645744720014674 5ustar rcohenrcohenCodeville-0.8.0/Codeville/old/0000775000076400007640000000000010645744720015452 5ustar rcohenrcohenCodeville-0.8.0/Codeville/old/__init__.py0000664000076400007640000000002310340212676017546 0ustar rcohenrcohenversion = '0.1.10' Codeville-0.8.0/Codeville/old/history.py0000664000076400007640000011077010340212676017523 0ustar rcohenrcohen# Written by Bram Cohen and Ross Cohen # see LICENSE.txt for license information from Codeville.bencode import bdecode, bencode import binascii from Codeville.db import db from merge import find_conflict, find_conflict_multiple_safe, MergeError, replay from os import path from sets import Set from sha import sha import struct from time import localtime, strftime from xml.sax import saxutils import zlib roothandle = '\xdfp\xce\xbaNt5\xf9\xa1\xabd\xf2\xc4\x87\xc5\xd1\x0bI\x8d\xb4' rootnode = '\xb3L\xac\x1f\x98B\x15\\\x8c\t0&\xd7m\xecK\xdd\n\x81\xc4' class HistoryError(Exception): pass def dmerge(da, db): dc = {}.fromkeys(da) dc.update({}.fromkeys(db)) return dc.keys() def rename_conflict_check(linfo, rinfo): if linfo['name'] == rinfo['name']: try: if linfo['parent'] == rinfo['parent']: return ('local', dmerge(linfo['rename point'], rinfo['rename point'])) except KeyError: assert linfo['rename point'] == rinfo['rename point'] == [rootnode] return ('local', linfo['rename point']) for i in rinfo['rename point']: if i in linfo['points']: return ('local', linfo['rename point']) for i in linfo['rename point']: if i in rinfo['points']: return ('remote', rinfo['rename point']) return ('conflict', None) def _name_use_count(co, state, point, func, txn): cursor = co.allnamesdb.cursor(txn=txn) lookup = state['parent'] + state['name'] try: key, value = cursor.set(lookup) except (db.DBNotFoundError, TypeError): return [] named = [] while key == lookup: vinfo = func(co, value, point, txn) if vinfo is not None: if not vinfo.has_key('delete') and vinfo['parent'] == state['parent'] and vinfo['name'] == state['name']: named.append(value) # next_dup() is broken foo = cursor.next() if foo is None: break key, value = foo cursor.close() return named def name_use_count(co, state, point, txn): return _name_use_count(co, state, point, __handle_name_at_point, txn) def _children_count(co, handle, point, func, txn): cursor = co.allnamesdb.cursor(txn=txn) try: foo = cursor.set_range(handle) except (db.DBNotFoundError, TypeError): foo = None children = {} while foo is not None: key, value = foo parent = key[:20] if parent != handle: break vinfo = func(co, value, point, txn) if vinfo is not None: if not vinfo.has_key('delete') and vinfo['parent'] == handle: children[value] = None foo = cursor.next() cursor.close() return children.keys() def children_count(co, handle, point, txn): return _children_count(co, handle, point, __handle_name_at_point, txn) def parent_loop_check(co, handle, point, txn): hseen = {} while handle != roothandle: if hseen.has_key(handle): return handle hseen[handle] = 1 change = handle_last_modified(co, co.names, handle, point, txn) if change is None: return handle handle = _handle_name_at_point(co, handle, change, txn)['parent'] return None def is_ancestor(co, ancestor, point, txn): if point is None: return 0 return _is_ancestor(co, ancestor, [point], txn) def _is_ancestor(co, ancestor, points, txn): #ainfo = bdecode(co.branchmapdb.get(ancestor, txn=txn)) ainfo = db_get(co, co.branchmapdb, ancestor, txn) points = points[:] state = {} while len(points): pnext = points.pop() if pnext == ancestor: return 1 if state.has_key(pnext): continue state[pnext] = 1 #pinfo = bdecode(co.branchmapdb.get(pnext, txn=txn)) pinfo = db_get(co, co.branchmapdb, pnext, txn) if pinfo['generation'] <= ainfo['generation']: continue points.extend(pinfo['precursors']) return 0 def read_diff(co, handle, point, txn): hinfo = bdecode(co.contents.dagdb.get(handle + point, txn=txn)) hfile = open(path.join(co.cpath, binascii.hexlify(handle)), 'rb') diff = _read_diff(hinfo, hfile) hfile.close() return diff def _read_diff(index, hfile): if not index.has_key('handle'): return None try: hfile.seek(index['handle']['offset']) except IOError: print index raise return hfile.read(index['handle']['length']) def write_diff(co, handle, diff, txn): cdagdb = co.contents.dagdb try: fend = struct.unpack(' c_%s [style=%s]' % (short_id(co, pre), sid, style) style = "dashed" points.extend([point for point in info['precursors']]) print '}' def pretty_print_dag(co, handle, heads): print 'digraph {' points = [] for point in heads: head = handle_last_modified(co, co.contents, handle, point, None) if head is not None: points.append(head) cdagdb = co.contents.dagdb cache = {} while len(points): point = points.pop() if cache.has_key(point): continue cache[point] = 1 sid = short_id(co, point) print 'c_%s [label="%s"]' % (sid, sid) info = bdecode(cdagdb.get(handle + point)) for pre, foo in info['precursors']: print 'c_%s -> c_%s' % (sid, short_id(co, pre)) points.extend([point for point, index in info['precursors']]) print '}' def _update_mini_dag(co, changedbs, helper, handles, cset, txn): indexdb = changedbs.indexdb dagdb = changedbs.dagdb pres = cset['precursors'] point = cset['point'] #bminfo = bdecode(co.branchmapdb.get(point, txn=txn)) bminfo = db_get(co, co.branchmapdb, point, txn) bnum = struct.pack('>II', bminfo['branch'], bminfo['branchnum']) clean_merges = [] for handle in handles: precursors = simplify_precursors(co, handle, changedbs, pres, txn)[0] mdinfo = {'handle': {}} if cset['handles'].has_key(handle): mdinfo['handle'] = helper(co, handle, point, cset['handles'][handle], txn) if mdinfo['handle'] == {}: del mdinfo['handle'] if len(precursors) > 1: clean_merges.append(handle) mdinfo['precursors'] = precursors if precursors == []: assert cset['handles'][handle].has_key('add') dagdb.put(handle + point, bencode(mdinfo), txn=txn) indexdb.put(handle + bnum, point, txn=txn) if len(clean_merges) > 0: changedbs.mergedb.put(point, bencode(clean_merges), txn=txn) return def simplify_precursors(co, handle, changedbs, pres, txn): # map big DAG precursors to little DAG dagdb = changedbs.dagdb precursors, indices = [], [] for i in xrange(len(pres)): last = handle_last_modified(co, changedbs, handle, pres[i], txn) if last is None: continue # XXX: this is correct, but breaks old history if 0: pinfo = bdecode(dagdb.get(handle + last, txn=txn)) if not pinfo.has_key('handle') and len(pinfo['precursors']) == 1: last = handle_last_modified(co, changedbs, handle, pinfo['precursors'][0][0], txn) if last is None: continue precursors.append(last) indices.append(i) # second pass to eliminate precursors which are ancestors of others retval = [] for i in xrange(len(precursors)): pre = precursors.pop(0) index = indices.pop(0) if _is_ancestor(co, pre, precursors, txn): continue precursors.append(pre) retval.append((pre, index)) return retval, len(pres) def _update_helper_content(co, handle, point, hinfo, txn): oinfo = {} if hinfo.has_key('hash'): if co.contents.dagdb.has_key(handle + point, txn): dinfo = bdecode(co.contents.dagdb.get(handle + point, txn=txn)) oinfo['offset'] = dinfo['handle']['offset'] oinfo['length'] = dinfo['handle']['length'] else: # XXX: ugly in general oinfo['offset'] = -1 if hinfo.has_key('add'): oinfo['add'] = 1 elif hinfo.has_key('delete'): oinfo['delete'] = 1 return oinfo def _update_helper_name(co, handle, point, hinfo, txn): oinfo = {} if hinfo.has_key('name'): oinfo['name'] = hinfo['name'] try: oinfo['parent'] = hinfo['parent'] co.allnamesdb.put(hinfo['parent'] + hinfo['name'], handle, flags=db.DB_NODUPDATA, txn=txn) except KeyError: assert handle == roothandle assert hinfo.has_key('add') except db.DBKeyExistError: pass if hinfo.has_key('add'): oinfo['add'] = 1 elif hinfo.has_key('delete'): oinfo['delete'] = 1 return oinfo def changes_in_branch(co, lpoints, bpoints, txn, cache=None): points = bpoints[:] seen, changes = {}, [] while len(points): pnext = points.pop() if seen.has_key(pnext): continue seen[pnext] = 1 if _is_ancestor(co, pnext, lpoints, txn): continue if cache: if cache.has_key(pnext): pinfo = cache[pnext] else: pinfo = bdecode(co.lcrepo.get(pnext, txn=txn)) cache[pnext] = pinfo else: pinfo = bdecode(co.lcrepo.get(pnext, txn=txn)) points.extend(pinfo['precursors']) changes.append(pnext) return changes def handles_in_branch(co, lpoints, bpoints, txn, cache=None, deleted_modified=False): points = bpoints[:] seen, named, modified, deleted = {}, {}, {}, {} while len(points): pnext = points.pop() if seen.has_key(pnext): continue seen[pnext] = 1 if _is_ancestor(co, pnext, lpoints, txn): continue if cache: if cache.has_key(pnext): pinfo = cache[pnext] else: pinfo = bdecode(co.lcrepo.get(pnext, txn=txn)) cache[pnext] = pinfo else: pinfo = bdecode(co.lcrepo.get(pnext, txn=txn)) for handle, hinfo in pinfo['handles'].items(): if hinfo.has_key('name'): named[handle] = 1 if hinfo.has_key('hash'): modified[handle] = 1 if hinfo.has_key('delete'): named[handle] = 1 if not deleted_modified: deleted[handle] = 1 # XXX: afaik, this is only an issue for ambiguous clean merges, which # don't happen with name operations. requires more thought. if co.contents.mergedb.has_key(pnext, txn): clean_merges = bdecode(co.contents.mergedb.get(pnext, txn=txn)) for handle in clean_merges: modified[handle] = 1 # XXX: check for deletes? points.extend(pinfo['precursors']) for handle in deleted.keys(): if modified.has_key(handle): del modified[handle] return (named.keys(), modified.keys()) def handle_last_modified(co, changedbs, handle, change, txn): indexdb = changedbs.indexdb dagdb = changedbs.dagdb if dagdb.has_key(handle + change, txn): return change #hinfo = bdecode(co.branchmapdb.get(change, txn=txn)) hinfo = db_get(co, co.branchmapdb, change, txn) bbranch, bnum = int(hinfo['branch']), hinfo['branchnum'] retval = None cursor = indexdb.cursor(txn=txn) while 1: branchpoint = struct.pack('>20sII', handle, bbranch, bnum) try: foo = cursor.set_range(branchpoint) except db.DBNotFoundError: foo = None if foo is None: foo = cursor.last() if foo is None: break key, value = foo else: key, value = foo if key != branchpoint: try: foo = cursor.prev_nodup() except db.DBNotFound: foo = None if foo is None: break key, value = foo rhandle, rbranch, rnum = struct.unpack('>20sII', key) if rhandle == handle and rbranch == bbranch: retval = value break #pinfo = bdecode(co.branchdb.get(bbranch, txn=txn)) pinfo = db_get(co, co.branchdb, bbranch, txn) try: bbranch, bnum = int(pinfo['parent']), pinfo['parentnum'] except KeyError: #_print_indexdb(co, indexdb, handle, txn) break cursor.close() return retval def _print_indexdb(co, indexdb, handle, txn): print 'indexdb' cursor = indexdb.cursor(txn=txn) foo = cursor.set_range(handle) while foo != None: key, value = foo nhandle, branch, num = struct.unpack('<20sII', key) if handle != nhandle: break print "%d, %d" % (branch, num) foo = cursor.next() cursor.close() def handle_name_at_point(co, handle, point, txn, dochecks=0): change = handle_last_modified(co, co.names, handle, point, txn) if change is None: return None co.name_cache = {} return _handle_name_at_point(co, handle, change, txn, dochecks=dochecks) def __handle_name_at_point(co, handle, point, txn, dochecks=0): key = handle + point if not dochecks and co.name_cache.has_key(key): return co.name_cache[key] change = handle_last_modified(co, co.names, handle, point, txn) if change is None: return None return _handle_name_at_point(co, handle, change, txn, dochecks=dochecks) def _handle_name_from_precursors(precursors, resolved): state = {} for pre in precursors: if state == {}: state = pre.copy() continue if pre.has_key('delete'): return pre outcome, rename_point = rename_conflict_check(state, pre) if not resolved and outcome == 'conflict': raise HistoryError, 'double name conflict' elif outcome == 'remote': state['name'] = pre['name'] state['parent'] = pre['parent'] state['rename point'] = rename_point state['points'] = dmerge(state['points'], pre['points']) return state def _handle_name_at_point(co, handle, point, txn, dochecks=0): def walk_precursors(cset, dochecks): precursors, points = [], [point] for pre, index in cset['precursors']: foo = _handle_name_at_point(co, handle, pre, txn, dochecks=dochecks) if foo is None: continue points = dmerge(points, foo['points']) precursors.append(foo) return precursors, points cset = bdecode(co.names.dagdb.get(handle + point, txn=txn)) if not cset.has_key('handle'): precursors, points = walk_precursors(cset, dochecks) state = _handle_name_from_precursors(precursors, 0) elif cset['handle'].has_key('delete'): precursors, points = walk_precursors(cset, dochecks) state = _handle_name_from_precursors(precursors, 1) state['delete'] = 1 else: precursors, points = walk_precursors(cset, dochecks) state = {} state['name'] = cset['handle']['name'] try: state['parent'] = cset['handle']['parent'] except KeyError: assert handle == roothandle assert cset['handle'].has_key('add') state['rename point'] = [point] state['points'] = points if dochecks == 0: return state co.name_cache[handle + point] = state if state['name'] == '' and handle != roothandle: raise HistoryError, 'illegal name' if state['name'] == '.' or state['name'] == '..': raise HistoryError, 'illegal name' if state.has_key('delete'): if len(children_count(co, handle, point, txn)): raise HistoryError, 'non-empty directory can\'t be deleted' return state staticinfo = db_get(co, co.staticdb, handle, txn) if staticinfo['type'] == 'dir': try: if parent_loop_check(co, state['parent'], point, txn): raise HistoryError, 'parent loop' except KeyError: pass try: #parentinfo = bdecode(co.staticdb.get(state['parent'], txn=txn)) parentinfo = db_get(co, co.staticdb, state['parent'], txn) if parentinfo['type'] != 'dir': raise HistoryError, 'parent not a directory' parentstate = __handle_name_at_point(co, state['parent'], point, txn) if parentstate is None: raise HistoryError, 'parent not in repository' if parentstate.has_key('delete'): raise HistoryError, 'file committed with deleted parent' if len(name_use_count(co, state, point, txn)) != 1: raise HistoryError, 'name already in use' if state['name'] == 'CVILLE': raise HistoryError, 'illegal name' except KeyError: assert handle == roothandle return state def fullpath_at_point(co, handle, point, txn=None): name = None while handle != roothandle: pinfo = handle_name_at_point(co, handle, point, txn) if pinfo is None: return None if name is None: name = pinfo['name'] else: name = path.join(pinfo['name'], name) handle = pinfo['parent'] return name def _mini_dag_refcount(co, handle, point, txn, cache=None, info_cache=None): assert info_cache is not None if cache is None: cache = {} points = [point] while len(points): point = points.pop() if cache.has_key(point): cache[point]['refcount'] += 1 continue cache[point] = {'refcount': 1} pinfo = bdecode(co.contents.dagdb.get(handle + point, txn=txn)) info_cache[point] = pinfo for p, i in pinfo['precursors']: points.append(p) return cache def handle_contents_at_point(co, handle, point, txn, dcache=None): if dcache is None: dcache = {} #staticinfo = bdecode(co.staticdb.get(handle, txn=txn)) staticinfo = db_get(co, co.staticdb, handle, txn) if staticinfo['type'] != 'file': raise ValueError, 'no contents for non-file' change = handle_last_modified(co, co.contents, handle, point, txn) if change is None: return None hcache = {} cache = _mini_dag_refcount(co, handle, change, txn, info_cache=hcache) hfile = open(path.join(co.cpath, binascii.hexlify(handle)), 'rb') points = [change] while len(points): point = points[-1] # we may have already done this one if cache.has_key(point) and cache[point].has_key('info'): points.pop() continue # cache this, since we visit most nodes twice if hcache.has_key(point): hinfo = hcache[point] else: hinfo = bdecode(co.contents.dagdb.get(handle + point, txn=txn)) hcache[point] = hinfo # check if we've got the precursors dirty = False for pre, foo in hinfo['precursors']: if not cache[pre].has_key('info'): dirty = True points.append(pre) if dirty: continue points.pop() # read the diff if dcache.has_key(point): diff = dcache[point] else: diff = _read_diff(hinfo, hfile) if diff is not None: diff = bdecode(zlib.decompress(diff)) # put together the precursor list and decrement refcounts precursors = [] for pre, foo in hinfo['precursors']: precursors.append(cache[pre]['info']) cache[pre]['refcount'] -= 1 if cache[pre]['refcount'] == 0: del cache[pre] # finally, get the contents cache[point]['info'] = _handle_contents_at_point(point, hinfo, precursors, diff) hfile.close() cache[change]['info']['type'] = staticinfo['type'] return cache[change]['info'] def _handle_contents_at_point(point, hinfo, precursors, diff): state = {} points = [] for pre in precursors: points = dmerge(points, pre['points']) state['points'] = points matches = [] if diff is not None: for pre, index in hinfo['precursors']: matches.append(diff['matches'][index]) if diff.has_key('add'): if precursors != []: raise HistoryError, 'handle already exists' elif precursors == []: raise HistoryError, 'cannot modify non-existent file' if diff.has_key('delete'): state['delete'] = diff['delete'] if not state.has_key('delete'): fpre = [] for pre in precursors: fpre.append((pre['lines'], pre['line points'], pre['points'])) if diff is not None: try: lines, line_points = replay(fpre, matches, diff['newlines'], point) except MergeError, msg: raise HistoryError, 'merge error: ' + str(msg) except KeyError: raise HistoryError, 'malformed change' points.append(point) else: lines, line_points, points = find_conflict_multiple_safe(fpre) if lines is None: # XXX: this is a pretty gross hack if len(fpre) == 2: s0 = Set(fpre[0][2]) s1 = Set(fpre[1][2]) if s0 == s1: raise HistoryError, 'merge error' elif s0.issubset(s1): lines, line_points, points = fpre[1] elif s0.issuperset(s1): lines, line_points, points = fpre[0] else: raise HistoryError, 'merge error' else: raise HistoryError, 'merge error' state['lines'] = lines state['line points'] = line_points return state def print_file_with_points(pre): def _print_points(line_points): ps = [] for line_point in line_points: ps.append(binascii.hexlify(line_point)) return ', '.join(ps) lines, line_points, points = pre out = [_print_points(line_points[0])] for i in xrange(len(lines)): out.append(lines[i]) out.append(_print_points(line_points[i+1])) return '\n'.join(out) from merge import _find_conflict def print_conflict(co, fpre): p1, p2 = fpre[0], fpre[1] olines, oline_points = _find_conflict(fpre[0][0], fpre[0][1], fpre[0][2], fpre[1][0], fpre[1][1], fpre[1][2]) ls = [] offset = [0, 0] for i in xrange(len(olines)): l = olines[i] if type(l) is str: offset[0] += 1 offset[1] += 1 continue print '@@ -%d,%d +%d,%d @@' % (offset[0], len(l[0]), offset[0], len(l[1])) offset[0] += len(l[0]) offset[1] += len(l[1]) lp = oline_points[i] ls.append('<<<<<<< local') ps = ', '.join([short_id(co, p) for p in lp[0][0]]) ls.append(ps) for j in xrange(len(l[0])): ls.append(l[0][j]) ps = ', '.join([short_id(co, p) for p in lp[0][j+1]]) ls.append(ps) ls.append('=======') ps = ', '.join([short_id(co, p) for p in lp[1][0]]) ls.append(ps) for j in xrange(len(l[1])): ls.append(l[1][j]) ps = ', '.join([short_id(co, p) for p in lp[1][j+1]]) ls.append(ps) ls.append('>>>>>>> remote') return '\n'.join(ls) def rebuild_from_points(co, points, txn): co.changesdb.truncate(txn) co.branchdb.truncate(txn) co.branchmapdb.truncate(txn) co.names.indexdb.truncate(txn) co.names.dagdb.truncate(txn) co.names.mergedb.truncate(txn) co.contents.indexdb.truncate(txn) co.contents.mergedb.truncate(txn) # we don't truncate the cdagdb because it contains the offsets and lengths # for the diffs in the files, which we can't recreate. the sync below will # read those parameters out and rewrite the cdagdb, anyway. co.linforepo.put('branchmax', bencode(0), txn=txn) cdagdb = co.contents.dagdb for key, value in cdagdb.items(txn): if len(key) != 40: continue if not bdecode(value).has_key('handle'): cdagdb.delete(key, txn=txn) for point in points: sync_history(co, point, txn) def clean_merge_point(info): if info['handles'] != {}: return 0 if len(info['precursors']) != 2: return 0 if info.has_key('comment'): return 0 return 1 def short_id(co, point): apoint = binascii.hexlify(point) length = 3 key = apoint[:length] cursor = co.changesdb.cursor() while key.startswith(apoint[:length]) and length < len(apoint): length += 1 try: key, value = cursor.set_range(apoint[:length]) except (db.DBNotFoundError, TypeError): break if key != apoint: continue foo = cursor.next() if foo is None: break key, value = foo cursor.close() return apoint[:length] class ChangeNotKnown(Exception): pass class ChangeNotUnique(Exception): pass class ChangeBadRepository(Exception): pass def long_id(co, point): if point.startswith('cdv://'): key = repo_head(co, tuple_to_server(server_to_tuple(point))) if key is None: raise ChangeBadRepository, point return key cursor = co.changesdb.cursor() try: key, value = cursor.set_range(point) except (db.DBNotFoundError, TypeError): cursor.close() raise ChangeNotKnown, point if not key.startswith(point): cursor.close() raise ChangeNotKnown, point try: key2, value = cursor.next() except (db.DBNotFoundError, TypeError): cursor.close() return binascii.unhexlify(key) if key2.startswith(point): keys = [key] while key2.startswith(point): keys.append(key2) try: key2, value = cursor.next() except (db.DBNotFoundError, TypeError): break cursor.close() raise ChangeNotUnique, (point, keys) cursor.close() return binascii.unhexlify(key) def repo_head(co, repo): if repo is None: return None repo = tuple_to_server(server_to_tuple(repo)) return co.linforepo.get(repo) def server_to_tuple(server_string): if server_string.startswith('cdv://'): server_string = server_string[6:] temp = server_string.split('/', 1) if len(temp) != 2 or temp[1] == '': repo = None else: repo = temp[1] if repo[-1] == '/': repo = repo[:-1] temp = temp[0].split(':', 1) if len(temp) == 1: port = 6601 else: try: port = int(temp[1]) except ValueError: return None server_string = temp[0] return (temp[0], port, repo) def tuple_to_server(tuple): return 'cdv://%s:%d/%s' % tuple def print_change(co, change, changeset): escape = saxutils.escape output = [] output.append("") output.append("" + binascii.hexlify(change) + "") output.append("" + short_id(co, change) + "") output.append("" + escape(changeset['user']) + "") try: output.append("" + escape(changeset['comment']) + "") except KeyError: pass output.append("" + str(changeset['time']) + "") output.append("" + strftime('%a %b %d %H:%M:%S %Y %Z', localtime(changeset['time'])) + "") adds, deletes, renames, mods = {}, {}, {}, {} for handle, value in changeset['handles'].items(): if value.has_key('add'): adds[handle] = value['add'] else: if value.has_key('delete'): deletes[handle] = 1 if value.has_key('name'): renames[handle] = 1 if value.has_key('hash'): mods[handle] = 1 output.append("") for handle, info in adds.items(): output.append("") output.append("" + escape(fullpath_at_point(co, handle, change)) + "") output.append("" + info['type'] + "") output.append("") output.append("") output.append("") for handle in deletes.keys(): output.append("") output.append("" + escape(fullpath_at_point(co, handle, change)) + "") output.append("") output.append("") output.append("") for handle in renames.keys(): output.append("") output.append("" + escape(fullpath_at_point(co, handle, change)) + "") output.append("") output.append("") output.append("") for handle in mods.keys(): output.append("") output.append("" + escape(fullpath_at_point(co, handle, change)) + "") output.append("") output.append("") output.append("") return output def dump_changeinfo(co, change, repo=None): output = [] changeset = bdecode(co.lcrepo.get(change)) if not clean_merge_point(changeset): raise ValueError output.append("") if repo: output.append("" + saxutils.escape(repo) + "") output.append("" + binascii.hexlify(change) + "") output.append("" + short_id(co, change) + "") output.append("" + saxutils.escape(changeset['user']) + "") output.append("" + str(changeset['time']) + "") for change in changes_in_branch(co, [changeset['precursors'][0]], changeset['precursors'][1:], None): changeset = bdecode(co.lcrepo.get(change)) if not clean_merge_point(changeset): output.extend(print_change(co, change, changeset)) output.append("\n") return '\n'.join(output) def db_get(co, cdb, key, txn): try: cache = co.db_cache[db] except KeyError: cache = co.db_cache[db] = {} if cache.has_key(key): return cache[key] cache[key] = bdecode(cdb.get(key, txn=txn)) #try: # return cache[key] #except KeyError: # cache[key] = bdecode(cdb.get(key, txn=txn)) return cache[key] def db_put(co, cdb, key, value, txn): cdb.put(key, bencode(value), txn=txn) try: cache = co.db_cache[db] except KeyError: cache = co.db_cache[db] = {} cache[key] = value try: import psyco psyco.bind(_is_ancestor, 0) except ImportError: pass Codeville-0.8.0/Codeville/old/merge.py0000664000076400007640000006601510340212676017123 0ustar rcohenrcohen# Written by Bram Cohen # see LICENSE.txt for license information from sets import Set def find_conflict(local, local_line_points, local_points, remote, remote_line_points, remote_points): return _find_conflict(local, local_line_points, local_points, remote, remote_line_points, remote_points)[0] def find_conflict_safe(local, local_line_points, local_points, remote, remote_line_points, remote_points): return find_conflict_multiple_safe([(local, local_line_points, local_points), (remote, remote_line_points, remote_points)])[:2] def find_conflict_multiple_safe(precursors): olds = precursors[:] lines, line_points, points = olds.pop(0) while olds: nlines, nlinepoints, npoints = olds.pop(0) lines, line_points = _find_conflict(lines, line_points, points, nlines, nlinepoints, npoints) points = _union(points, npoints) if None in line_points: return (None, None, None) return lines, line_points, points def _is_applied(ll, pdict): for i in ll: for j in i: if j in pdict: break else: return 0 return 1 def _find_conflict(local, local_line_points, local_points, remote, remote_line_points, remote_points): assert len(local_line_points) == len(local) + 1 assert len(remote_line_points) == len(remote) + 1 lpdict = Set(local_points) for i in local_line_points: for point in i: assert point in lpdict rpdict = Set(remote_points) for i in remote_line_points: for point in i: assert point in rpdict result = [] result_line_points = [] lastl, lastr = 0, 0 for (lpos, rpos, matchlen) in _find_matches(local, remote) + [(len(local), len(remote), 0)]: rapplied = _is_applied(remote_line_points[lastr:rpos + 1], lpdict) lapplied = _is_applied(local_line_points[lastl:lpos + 1], rpdict) if rapplied and not lapplied: result.extend(local[lastl:lpos + matchlen]) result_line_points.extend(local_line_points[lastl:lpos + 1]) elif lapplied and not rapplied: result.extend(remote[lastr:rpos + matchlen]) result_line_points.extend(remote_line_points[lastr:rpos + 1]) elif (lastl, lastr) == (len(local), len(remote)): result_line_points.append(_union(local_line_points[-1], remote_line_points[-1])) elif (lpos, rpos) == (0, 0): result.extend(local[:matchlen]) result_line_points.append(_union(local_line_points[0], remote_line_points[0])) else: result.append((local[lastl:lpos], remote[lastr:rpos])) result.extend(local[lpos:lpos + matchlen]) result_line_points.append((local_line_points[lastl:lpos + 1], remote_line_points[lastr:rpos + 1])) result_line_points.append(None) for i in xrange(1, matchlen): result_line_points.append(_union(local_line_points[lpos + i], remote_line_points[rpos + i])) lastl, lastr = lpos + matchlen, rpos + matchlen return (result, result_line_points) def _union(la, lb): lc = la[:] for i in lb: if i not in lc: lc.append(i) return lc def test_conflict(): # conflict at begin ## local zero ### local wins assert _find_conflict(['aaa', 'bbb'], [['b'], ['a'], ['a']], ['a', 'b', 'c'], ['x', 'aaa', 'bbb'], [['a'], ['c'], ['a'], ['a']], ['a', 'c']) == (['aaa', 'bbb'], [['b'], ['a'], ['a']]) ### remote wins #### remote violate at begin assert _find_conflict(['aaa', 'bbb'], [['a'], ['a'], ['a']], ['a'], ['x', 'y', 'aaa', 'bbb'], [['b'], ['a'], ['a'], ['a'], ['a']], ['a', 'b']) == (['x', 'y', 'aaa', 'bbb'], [['b'], ['a'], ['a'], ['a'], ['a']]) #### remote violate at end assert _find_conflict(['aaa', 'bbb'], [['a'], ['a'], ['a']], ['a'], ['x', 'y', 'aaa', 'bbb'], [['a'], ['a'], ['b'], ['a'], ['a']], ['a', 'b']) == (['x', 'y', 'aaa', 'bbb'], [['a'], ['a'], ['b'], ['a'], ['a']]) ### neither win #### remote violate at begin assert _find_conflict(['aaa', 'bbb'], [['c'], ['a'], ['a']], ['a', 'c'], ['x', 'y', 'aaa', 'bbb'], [['b'], ['a'], ['a'], ['a'], ['a']], ['a', 'b']) == ([([], ['x', 'y']), 'aaa', 'bbb'], [([['c']], [['b'], ['a'], ['a']]), None, ['a'], ['a']]) #### remote violate at end assert _find_conflict(['aaa', 'bbb'], [['c'], ['a'], ['a']], ['a', 'c'], ['x', 'y', 'aaa', 'bbb'], [['a'], ['a'], ['b'], ['a'], ['a']], ['a', 'b']) == ([([], ['x', 'y']), 'aaa', 'bbb'], [([['c']], [['a'], ['a'], ['b']]), None, ['a'], ['a']]) ## remote zero ### local wins assert _find_conflict(['x', 'aaa', 'bbb'], [['a'], ['b'], ['a'], ['a']], ['a', 'b'], ['aaa', 'bbb'], [['a'], ['a'], ['a']], ['a']) == (['x', 'aaa', 'bbb'], [['a'], ['b'], ['a'], ['a']]) ### remote wins assert _find_conflict(['x', 'aaa', 'bbb'], [['a'], ['b'], ['a'], ['a']], ['a', 'b'], ['aaa', 'bbb'], [['c'], ['a'], ['a']], ['a', 'b', 'c']) == (['aaa', 'bbb'], [['c'], ['a'], ['a']]) ### neither win #### local violate at begin assert _find_conflict(['x', 'y', 'aaa', 'bbb'], [['b'], ['a'], ['a'], ['a'], ['a']], ['a', 'b'], ['aaa', 'bbb'], [['c'], ['a'], ['a']], ['a', 'c']) == ([(['x', 'y'], []), 'aaa', 'bbb'], [([['b'], ['a'], ['a']], [['c']]), None, ['a'], ['a']]) #### local violate at end assert _find_conflict(['x', 'y', 'aaa', 'bbb'], [['a'], ['a'], ['b'], ['a'], ['a']], ['a', 'b'], ['aaa', 'bbb'], [['c'], ['a'], ['a']], ['a', 'c']) == ([(['x', 'y'], []), 'aaa', 'bbb'], [([['a'], ['a'], ['b']], [['c']]), None, ['a'], ['a']]) ## neither zero ### local wins assert _find_conflict(['x', 'y', 'aaa', 'bbb'], [['a'], ['b'], ['a'], ['a'], ['a']], ['a', 'b'], ['z', 'w', 'aaa', 'bbb'], [['a'], ['a'], ['a'], ['a'], ['a']], ['a']) == (['x', 'y', 'aaa', 'bbb'], [['a'], ['b'], ['a'], ['a'], ['a']]) ### remote wins #### violates at beginning assert _find_conflict(['x', 'y', 'aaa', 'bbb'], [['a'], ['a'], ['a'], ['a'], ['a']], ['a'], ['z', 'w', 'aaa', 'bbb'], [['b'], ['a'], ['a'], ['a'], ['a']], ['a', 'b']) == (['z', 'w', 'aaa', 'bbb'], [['b'], ['a'], ['a'], ['a'], ['a']]) #### violates at end assert _find_conflict(['x', 'y', 'aaa', 'bbb'], [['a'], ['a'], ['a'], ['a'], ['a']], ['a'], ['z', 'w', 'aaa', 'bbb'], [['a'], ['a'], ['b'], ['a'], ['a']], ['a', 'b']) == (['z', 'w', 'aaa', 'bbb'], [['a'], ['a'], ['b'], ['a'], ['a']]) ### neither win #### violates at begin assert _find_conflict(['x', 'y', 'aaa', 'bbb'], [['c'], ['a'], ['a'], ['a'], ['a']], ['a', 'c'], ['z', 'w', 'aaa', 'bbb'], [['b'], ['a'], ['a'], ['a'], ['a']], ['a', 'b']) == ([(['x', 'y'], ['z', 'w']), 'aaa', 'bbb'], [([['c'], ['a'], ['a']], [['b'], ['a'], ['a']]), None, ['a'], ['a']]) #### violates at end assert _find_conflict(['x', 'y', 'aaa', 'bbb'], [['a'], ['a'], ['c'], ['a'], ['a']], ['a', 'c'], ['z', 'w', 'aaa', 'bbb'], [['a'], ['a'], ['b'], ['a'], ['a']], ['a', 'b']) == ([(['x', 'y'], ['z', 'w']), 'aaa', 'bbb'], [([['a'], ['a'], ['c']], [['a'], ['a'], ['b']]), None, ['a'], ['a']]) # conflict in middle ## local zero ### local wins assert _find_conflict(['aaa', 'bbb', 'ccc', 'ddd'], [['a'], ['a'], ['b'], ['a'], ['a']], ['a', 'b', 'c'], ['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['c'], ['c'], ['a'], ['a']], ['a', 'c']) == (['aaa', 'bbb', 'ccc', 'ddd'], [['a'], ['a'], ['b'], ['a'], ['a']]) ### remote wins #### violate at begin assert _find_conflict(['aaa', 'bbb', 'ccc', 'ddd'], [['a'], ['a'], ['b'], ['a'], ['a']], ['a', 'b'], ['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['c'], ['a'], ['a'], ['a']], ['a', 'b', 'c']) == (['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['c'], ['a'], ['a'], ['a']]) #### violate at end assert _find_conflict(['aaa', 'bbb', 'ccc', 'ddd'], [['a'], ['a'], ['b'], ['a'], ['a']], ['a', 'b'], ['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['a'], ['c'], ['a'], ['a']], ['a', 'b', 'c']) == (['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['a'], ['c'], ['a'], ['a']]) ### neither win #### remote violate at begin assert _find_conflict(['aaa', 'bbb', 'ccc', 'ddd'], [['a'], ['a'], ['b'], ['a'], ['a']], ['a', 'b'], ['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['c'], ['a'], ['a'], ['a']], ['a', 'c']) == (['aaa', 'bbb', ([], ['x']), 'ccc', 'ddd'], [['a'], ['a'], ([['b']], [['c'], ['a']]), None, ['a'], ['a']]) #### remote violate at end assert _find_conflict(['aaa', 'bbb', 'ccc', 'ddd'], [['a'], ['a'], ['b'], ['a'], ['a']], ['a', 'b'], ['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['a'], ['c'], ['a'], ['a']], ['a', 'c']) == (['aaa', 'bbb', ([], ['x']), 'ccc', 'ddd'], [['a'], ['a'], ([['b']], [['a'], ['c']]), None, ['a'], ['a']]) ## remote zero ### local wins assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['x'], ['x'], ['a'], ['a']], ['a', 'x', 'c'], ['aaa', 'bbb', 'ccc', 'ddd'], [['a'], ['a'], ['c'], ['a'], ['a']], ['a', 'c']) == (['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['x'], ['x'], ['a'], ['a']]) ### remote wins assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['x'], ['x'], ['a'], ['a']], ['a', 'x'], ['aaa', 'bbb', 'ccc', 'ddd'], [['a'], ['a'], ['c'], ['a'], ['a']], ['a', 'x', 'c']) == (['aaa', 'bbb', 'ccc', 'ddd'], [['a'], ['a'], ['c'], ['a'], ['a']]) ### neither win #### local violate at begin assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['x'], ['a'], ['a'], ['a']], ['a', 'x'], ['aaa', 'bbb', 'ccc', 'ddd'], [['a'], ['a'], ['c'], ['a'], ['a']], ['a', 'c']) == (['aaa', 'bbb', (['x'], []), 'ccc', 'ddd'], [['a'], ['a'], ([['x'], ['a']], [['c']]), None, ['a'], ['a']]) #### local violate at end assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['a'], ['x'], ['a'], ['a']], ['a', 'x'], ['aaa', 'bbb', 'ccc', 'ddd'], [['a'], ['a'], ['c'], ['a'], ['a']], ['a', 'c']) == (['aaa', 'bbb', (['x'], []), 'ccc', 'ddd'], [['a'], ['a'], ([['a'], ['x']], [['c']]), None, ['a'], ['a']]) ## neither zero ### local wins assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['x'], ['x'], ['a'], ['a']], ['a', 'x', 'y'], ['aaa', 'bbb', 'y', 'ccc', 'ddd'], [['a'], ['a'], ['y'], ['y'], ['a'], ['a']], ['a', 'y']) == (['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['x'], ['x'], ['a'], ['a']]) ### remote wins #### violate at begin assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['x'], ['x'], ['a'], ['a']], ['a', 'x'], ['aaa', 'bbb', 'y', 'ccc', 'ddd'], [['a'], ['a'], ['y'], ['a'], ['a'], ['a']], ['a', 'x', 'y']) == (['aaa', 'bbb', 'y', 'ccc', 'ddd'], [['a'], ['a'], ['y'], ['a'], ['a'], ['a']]) #### violate at end assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['x'], ['x'], ['a'], ['a']], ['a', 'x'], ['aaa', 'bbb', 'y', 'ccc', 'ddd'], [['a'], ['a'], ['a'], ['y'], ['a'], ['a']], ['a', 'x', 'y']) == (['aaa', 'bbb', 'y', 'ccc', 'ddd'], [['a'], ['a'], ['a'], ['y'], ['a'], ['a']]) ### neither win #### violates at begin assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['x'], ['a'], ['a'], ['a']], ['a', 'x'], ['aaa', 'bbb', 'y', 'ccc', 'ddd'], [['a'], ['a'], ['y'], ['a'], ['a'], ['a']], ['a', 'y']) == (['aaa', 'bbb', (['x'], ['y']), 'ccc', 'ddd'], [['a'], ['a'], ([['x'], ['a']], [['y'], ['a']]), None, ['a'], ['a']]) #### violates at end assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], [['a'], ['a'], ['a'], ['x'], ['a'], ['a']], ['a', 'x'], ['aaa', 'bbb', 'y', 'ccc', 'ddd'], [['a'], ['a'], ['a'], ['y'], ['a'], ['a']], ['a', 'y']) == (['aaa', 'bbb', (['x'], ['y']), 'ccc', 'ddd'], [['a'], ['a'], ([['a'], ['x']], [['a'], ['y']]), None, ['a'], ['a']]) # conflict at end ## local zero ### local wins assert _find_conflict(['aaa', 'bbb'], [['a'], ['a'], ['b']], ['a', 'b', 'x'], ['aaa', 'bbb', 'x'], [['a'], ['a'], ['x'], ['x']], ['a', 'x']) == (['aaa', 'bbb'], [['a'], ['a'], ['b']]) ### remote wins #### violate at begin assert _find_conflict(['aaa', 'bbb'], [['a'], ['a'], ['b']], ['a', 'b'], ['aaa', 'bbb', 'x'], [['a'], ['a'], ['x'], ['a']], ['a', 'b', 'x']) == (['aaa', 'bbb', 'x'], [['a'], ['a'], ['x'], ['a']]) #### violate at end assert _find_conflict(['aaa', 'bbb'], [['a'], ['a'], ['b']], ['a', 'b'], ['aaa', 'bbb', 'x'], [['a'], ['a'], ['a'], ['x']], ['a', 'b', 'x']) == (['aaa', 'bbb', 'x'], [['a'], ['a'], ['a'], ['x']]) ### neither win #### remote violate at begin assert _find_conflict(['aaa', 'bbb'], [['a'], ['a'], ['b']], ['a', 'b'], ['aaa', 'bbb', 'x'], [['a'], ['a'], ['x'], ['a']], ['a', 'x']) == (['aaa', 'bbb', ([], ['x'])], [['a'], ['a'], ([['b']], [['x'], ['a']]), None]) #### remote violate at end assert _find_conflict(['aaa', 'bbb'], [['a'], ['a'], ['b']], ['a', 'b'], ['aaa', 'bbb', 'x'], [['a'], ['a'], ['a'], ['x']], ['a', 'x']) == (['aaa', 'bbb', ([], ['x'])], [['a'], ['a'], ([['b']], [['a'], ['x']]), None]) ## remote zero ### local wins assert _find_conflict(['aaa', 'bbb', 'x'], [['a'], ['a'], ['x'], ['x']], ['a', 'b', 'x'], ['aaa', 'bbb'], [['a'], ['a'], ['b']], ['a', 'b']) == (['aaa', 'bbb', 'x'], [['a'], ['a'], ['x'], ['x']]) ### remote wins assert _find_conflict(['aaa', 'bbb', 'x'], [['a'], ['a'], ['x'], ['x']], ['a', 'x'], ['aaa', 'bbb'], [['a'], ['a'], ['b']], ['a', 'b', 'x']) == (['aaa', 'bbb'], [['a'], ['a'], ['b']]) ### neither win #### local violate at begin assert _find_conflict(['aaa', 'bbb', 'x'], [['a'], ['a'], ['x'], ['a']], ['a', 'x'], ['aaa', 'bbb'], [['a'], ['a'], ['b']], ['a', 'b']) == (['aaa', 'bbb', (['x'], [])], [['a'], ['a'], ([['x'], ['a']], [['b']]), None]) #### local violate at end assert _find_conflict(['aaa', 'bbb', 'x'], [['a'], ['a'], ['a'], ['x']], ['a', 'x'], ['aaa', 'bbb'], [['a'], ['a'], ['b']], ['a', 'b']) == (['aaa', 'bbb', (['x'], [])], [['a'], ['a'], ([['a'], ['x']], [['b']]), None]) ## neither zero ### local wins assert _find_conflict(['aaa', 'bbb', 'x'], [['a'], ['a'], ['x'], ['x']], ['a', 'x', 'y'], ['aaa', 'bbb', 'y'], [['a'], ['a'], ['y'], ['y']], ['a', 'y']) == (['aaa', 'bbb', 'x'], [['a'], ['a'], ['x'], ['x']]) ### remote wins assert _find_conflict(['aaa', 'bbb', 'x'], [['a'], ['a'], ['x'], ['x']], ['a', 'x'], ['aaa', 'bbb', 'y'], [['a'], ['a'], ['y'], ['y']], ['a', 'x', 'y']) == (['aaa', 'bbb', 'y'], [['a'], ['a'], ['y'], ['y']]) ### neither win #### violates at begin assert _find_conflict(['aaa', 'bbb', 'x'], [['a'], ['a'], ['x'], ['a']], ['a', 'x'], ['aaa', 'bbb', 'y'], [['a'], ['a'], ['y'], ['a']], ['a', 'y']) == (['aaa', 'bbb', (['x'], ['y'])], [['a'], ['a'], ([['x'], ['a']], [['y'], ['a']]), None]) #### violates at end assert _find_conflict(['aaa', 'bbb', 'x'], [['a'], ['a'], ['a'], ['x']], ['a', 'x'], ['aaa', 'bbb', 'y'], [['a'], ['a'], ['a'], ['y']], ['a', 'y']) == (['aaa', 'bbb', (['x'], ['y'])], [['a'], ['a'], ([['a'], ['x']], [['a'], ['y']]), None]) # whole file conflict ## local zero ### local wins assert _find_conflict([], [['a']], ['a', 'b'], ['x'], [['b'], ['b']], ['b']) == ([], [['a']]) ### remote wins #### violate at begin assert _find_conflict([], [['a']], ['a'], ['x'], [['b'], ['a']], ['a', 'b']) == (['x'], [['b'], ['a']]) #### violate at end assert _find_conflict([], [['a']], ['a'], ['x'], [['a'], ['b']], ['a', 'b']) == (['x'], [['a'], ['b']]) ### neither win #### remote violate at begin assert _find_conflict([], [['a']], ['a', 'c'], ['x'], [['b'], ['c']], ['b', 'c']) == ([([], ['x'])], [([['a']], [['b'], ['c']]), None]) #### remote violate at end assert _find_conflict([], [['a']], ['a', 'c'], ['x'], [['c'], ['b']], ['b', 'c']) == ([([], ['x'])], [([['a']], [['c'], ['b']]), None]) ## remote zero ### local wins assert _find_conflict(['a'], [['a'], ['a']], ['a', 'b'], [], [['b']], ['b']) == (['a'], [['a'], ['a']]) ### remote wins assert _find_conflict(['a'], [['a'], ['a']], ['a'], [], [['b']], ['a', 'b']) == ([], [['b']]) ### neither win #### local violate at begin assert _find_conflict(['a'], [['a'], ['c']], ['a', 'c'], [], [['b']], ['b', 'c']) == ([(['a'], [])], [([['a'], ['c']], [['b']]), None]) #### local violate at end assert _find_conflict(['a'], [['c'], ['a']], ['a', 'c'], [], [['b']], ['b', 'c']) == ([(['a'], [])], [([['c'], ['a']], [['b']]), None]) ## neither zero ### local wins assert _find_conflict(['a'], [['a'], ['a']], ['a', 'b'], ['b'], [['b'], ['b']], ['b']) == (['a'], [['a'], ['a']]) ### remote wins assert _find_conflict(['a'], [['a'], ['a']], ['a'], ['b'], [['b'], ['b']], ['a', 'b']) == (['b'], [['b'], ['b']]) ### neither win #### violate at begin assert _find_conflict(['a'], [['a'], ['c']], ['a', 'c'], ['b'], [['b'], ['c']], ['c', 'b']) == ([(['a'], ['b'])], [([['a'], ['c']], [['b'], ['c']]), None]) #### violate at end assert _find_conflict(['a'], [['c'], ['a']], ['a', 'c'], ['b'], [['c'], ['b']], ['c', 'b']) == ([(['a'], ['b'])], [([['c'], ['a']], [['c'], ['b']]), None]) # union counts for either assert _find_conflict(['aaa', 'bbb'], [['a'], ['a', 'b'], ['a']], ['a', 'b'], ['ccc', 'ddd'], [['a'], ['c'], ['a']], ['a', 'c']) == (['ccc', 'ddd'], [['a'], ['c'], ['a']]) # coincidental matches ## begin assert _find_conflict(['aaa', 'bbb'], [['b'], ['a'], ['a']], ['a', 'b'], ['aaa', 'bbb'], [['c'], ['a'], ['a']], ['a', 'c']) == (['aaa', 'bbb'], [['b', 'c'], ['a'], ['a']]) ## middle assert _find_conflict(['aaa', 'bbb'], [['a'], ['b'], ['a']], ['a', 'b'], ['aaa', 'bbb'], [['a'], ['c'], ['a']], ['a', 'c']) == (['aaa', 'bbb'], [['a'], ['b', 'c'], ['a']]) ## end assert _find_conflict(['aaa', 'bbb'], [['a'], ['a'], ['b']], ['a', 'b'], ['aaa', 'bbb'], [['a'], ['a'], ['c']], ['a', 'c']) == (['aaa', 'bbb'], [['a'], ['a'], ['b', 'c']]) # multiple conflicts, different resolutions assert _find_conflict(['x', 'aaa', 'bbb', 'y', 'ccc', 'ddd', 'z'], [['x'], ['x'], ['a'], ['y'], ['y'], ['a'], ['z'], ['z']], ['a', 'x', 'y', 'z', 'p'], ['p', 'aaa', 'bbb', 'q', 'ccc', 'ddd', 'r'], [['p'], ['p'], ['a'], ['q'], ['q'], ['a'], ['r'], ['r']], ['a', 'p', 'q', 'r', 'z']) == (['x', 'aaa', 'bbb', (['y'], ['q']), 'ccc', 'ddd', 'r'], [['x'], ['x'], ['a'], ([['y'], ['y']], [['q'], ['q']]), None, ['a'], ['r'], ['r']]) def _find_matches(local, remote): if local == remote: return [(0, 0, len(local))] bdict = {} for i in xrange(0, len(remote)): bdict.setdefault(remote[i], []).append(i) return _fm(local, 0, len(local), remote, 0, len(remote), bdict) def _fm(linesa, abegin, aend, linesb, bbegin, bend, bdict): bestlen = 0 for i in xrange(abegin, aend): if i + bestlen >= aend: break for j in bdict.get(linesa[i], []): if j < bbegin: continue if j + bestlen >= bend: break if i > abegin and j > bbegin and linesa[i - 1] == linesb[j - 1]: continue if linesa[i + bestlen] != linesb[j + bestlen]: continue count = 0 k = 0 while i + k < aend and j + k < bend and linesa[i + k] == linesb[j + k]: if count < 2 and len(linesa[i + k].strip()) > 2: count += 1 k += 1 if k > bestlen and count >= 2: besta = i bestb = j bestlen = k if i + bestlen >= aend: break if bestlen == 0: return [] return _fm(linesa, abegin, besta, linesb, bbegin, bestb, bdict) + [(besta, bestb, bestlen)] + _fm(linesa, besta + bestlen, aend, linesb, bestb + bestlen, bend, bdict) def test_find_matches(): assert _find_matches(['x', 'aaa', 'y'], ['z', 'aaa', 'y']) == [] assert _find_matches(['a'], ['a']) == [(0, 0, 1)] assert _find_matches(['aaa', 'aaa', 'ppp'], ['aaa', 'aaa', 'qqq']) == [(0, 0, 2)] assert _find_matches(['ppp', 'aaa', 'aaa'], ['qqq', 'aaa', 'aaa']) == [(1, 1, 2)] assert _find_matches(['ppp', 'aaa', 'aaa', 'ppp'], ['qqq', 'aaa', 'aaa', 'qqq']) == [(1, 1, 2)] assert _find_matches(['aaa', 'aaa', 'bbb', 'ccc', 'aaa', 'aaa', 'aaa'], ['aaa', 'aaa', 'bbb', 'aaa', 'aaa', 'aaa', 'ccc']) == [(0, 0, 3), (4, 3, 3)] assert _find_matches(['aaa', 'aaa', 'bbb', 'aaa', 'aaa', 'aaa', 'ccc'], ['aaa', 'aaa', 'bbb', 'ccc', 'aaa', 'aaa', 'aaa']) == [(0, 0, 3), (3, 4, 3)] assert _find_matches(['aaa', 'aaa', 'aaa', 'bbb', 'ccc', 'aaa', 'aaa', 'aaa'], ['bbb', 'aaa', 'aaa', 'aaa', 'ccc', 'aaa', 'aaa', 'aaa']) == [(0, 1, 3), (4, 4, 4)] assert _find_matches(['bbb', 'aaa', 'aaa', 'aaa', 'ccc', 'aaa', 'aaa', 'aaa'], ['aaa', 'aaa', 'aaa', 'bbb', 'ccc', 'aaa', 'aaa', 'aaa']) == [(1, 0, 3), (4, 4, 4)] assert _find_matches(['aaa', 'aaa', 'bbb', 'bbb'], ['bbb', 'bbb', 'aaa', 'aaa']) == [(0, 2, 2)] assert _find_matches(['aaa', 'aaa', 'bbb', 'bbb', 'bbb'], ['bbb', 'bbb', 'bbb', 'aaa', 'aaa']) == [(2, 0, 3)] def find_resolution(precursors, resolution): all = [] for i in xrange(len(precursors)): for (matchold, matchnew, matchlen) in _find_matches(precursors[i][0], resolution): all.append((matchlen, i, matchold, matchnew)) all.sort() ms = [[] for i in xrange(len(precursors))] result = [[] for i in xrange(len(resolution) + 1)] for (matchlen, i, matchold, matchnew) in all: pre = precursors[i][0] pre_line_points = precursors[i][1] (begina, beginb, mlen) = (matchold + 1, matchnew + 1, matchlen - 1) if (matchold, matchnew) == (0, 0): (begina, beginb, mlen) = (0, 0, mlen + 1) if (matchold + matchlen, matchnew + matchlen) == (len(pre), len(resolution)): mlen += 1 for j in xrange(mlen): for v in pre_line_points[begina + j]: if v not in result[beginb + j]: result[beginb + j].append(v) ms[i].append((matchold, matchnew, matchlen)) #for x in result: # x.sort() for x in ms: x.sort() hits = [False] * len(resolution) for i in xrange(len(ms)): for (matchold, matchnew, matchlen) in ms[i]: for j in xrange(matchlen): hits[matchnew + j] = True newlines = [] pos = 0 while pos < len(hits): if hits[pos]: pos += 1 else: n = [] j = pos while j < len(hits) and not hits[j]: n.append(resolution[j]) j += 1 newlines.append((pos, n)) pos = j return (result, ms, newlines) def test_resolve(): assert find_resolution([(['aaa', 'bbb', 'ccc'], [['x'], ['y'], ['z'], ['w']]), (['aaa', 'bbb', 'ccc'], [['p'], ['q'], ['z'], ['w']])], ['aaa', 'bbb', 'ccc'])[0] == [['p', 'x'], ['q', 'y'], ['z'], ['w']] assert find_resolution([(['aaa', 'bbb', 'qqq'], [['a'], ['b'], ['c'], ['d']]), (['qqq', 'bbb', 'ccc'], [['e'], ['f'], ['g'], ['h']])], ['aaa', 'bbb', 'ccc'])[0] == [['a'], ['b'], ['g'], ['h']] assert find_resolution([(['aaa'], [['a'], ['a']])], ['bbb'])[0] == [[], []] assert find_resolution([(['aaa'], [['x'], ['x']]), (['aaa'], [['x'], ['x']])], ['aaa'])[1] == [[(0, 0, 1)], []] assert find_resolution([(['aaa'], [['x'], ['x']]), (['aaa'], [['x'], ['y']])], ['aaa'])[1] == [[(0, 0, 1)], [(0, 0, 1)]] assert find_resolution([(['aaa', 'aaa'], [['x'], ['x'], ['x']])], ['aaa', 'aaa', 'bbb', 'ccc'])[2] == [(2, ['bbb', 'ccc'])] assert find_resolution([(['aaa', 'aaa'], [['x'], ['x'], ['x']])], ['bbb', 'ccc', 'aaa', 'aaa'])[2] == [(0, ['bbb', 'ccc'])] assert find_resolution([(['aaa', 'aaa'], [['x'], ['x'], ['x']])], ['bbb', 'ccc'])[2] == [(0, ['bbb', 'ccc'])] def replay(precursors, matches, newlines, current_point): resultlength = 0 for i in matches: if not len(i): continue matchold, matchnew, matchlen = i[-1] if matchnew + matchlen > resultlength: resultlength = matchnew + matchlen try: foo = newlines[-1][0] + len(newlines[-1][1]) if foo > resultlength: resultlength = foo except IndexError: pass r = [None] * resultlength rpoints = [None] * (resultlength + 1) def addrp(pos, vs, rpoints = rpoints): if rpoints[pos] is None: rpoints[pos] = vs[:] return rp = rpoints[pos] for v in vs: if v not in rp: rp.append(v) return delpoints = [0] for i in xrange(len(matches)): try: if matches[i][0][0] == 0 and matches[i][0][1] == 0: addrp(0, precursors[i][1][0]) except IndexError: pass try: matchold, matchnew, matchlen = matches[i][-1] if matchnew + matchlen == resultlength and matchold + matchlen == len(precursors[i][0]): addrp(resultlength, precursors[i][1][-1]) except IndexError: pass for (matchold, matchnew, matchlen) in matches[i]: foo = precursors[i][0] if matchold + matchlen > len(foo): raise MergeError, 'match too long' for j in xrange(matchlen): pos = matchnew + j v = foo[matchold + j] if r[pos] is not None and r[pos] != v: raise MergeError, 'conflicting values' r[pos] = v delpoints.append(matchnew + matchlen) foo = precursors[i][1] for j in xrange(1, matchlen): addrp(matchnew + j, foo[matchold + j]) foo = [current_point] for (begin, lines) in newlines: for i in xrange(len(lines)): if r[begin + i] is not None: raise MergeError, 'match covers newline' r[begin + i] = lines[i] for i in xrange(len(lines) + 1): addrp(begin + i, foo) for index in delpoints: if rpoints[index] is None: addrp(index, foo) if None in r: raise MergeError, 'unfilled line' if None in rpoints: raise MergeError, 'unset line point' return (r, rpoints) class MergeError(StandardError): pass def test_replay(): assert replay([(['aaa', 'bbb'], [['a'], ['b'], ['c']])], [[(0, 0, 2)]], [(2, ['ccc', 'ddd'])], 'z') == (['aaa', 'bbb', 'ccc', 'ddd'], [['a'], ['b'], ['z'], ['z'], ['z']]) assert replay([(['aaa', 'bbb'], [['a'], ['b'], ['c']])], [[(0, 2, 2)]], [(0, ['ccc', 'ddd'])], 'z') == (['ccc', 'ddd', 'aaa', 'bbb'], [['z'], ['z'], ['z'], ['b'], ['c']]) assert replay([(['aaa', 'bbb', 'ccc', 'ddd'], [['a'], ['b'], ['c'], ['d'], ['e']]), (['bbb', 'ccc', 'ddd', 'eee'], [['f'], ['c'], ['g'], ['h'], ['i']])], [[(0, 0, 4)], [(0, 1, 4)]], [], 'z') == (['aaa', 'bbb', 'ccc', 'ddd', 'eee'], [['a'], ['b'], ['c'], ['d', 'g'], ['h'], ['i']]) assert replay([(['aaa', 'bbb', 'ccc', 'ddd', 'eee'], [['a'], ['a'], ['a'], ['a'], ['a'], ['a']])], [[(0, 0, 2), (3, 2, 2)]], [], 'z') == (['aaa', 'bbb', 'ddd', 'eee'], [['a'], ['a'], ['z'], ['a'], ['a']]) assert replay([(['aaa', 'bbb', 'ccc', 'ddd'], [['a'], ['a'], ['a'], ['a'], ['a']])], [[(2, 0, 2)]], [], 'z') == (['ccc', 'ddd'], [['z'], ['a'], ['a']]) assert replay([(['aaa', 'bbb', 'ccc', 'ddd'], [['a'], ['a'], ['a'], ['a'], ['a']])], [[(0, 0, 2)]], [], 'z') == (['aaa', 'bbb'], [['a'], ['a'], ['z']]) try: import psyco psyco.bind(replay) psyco.bind(_find_matches) except ImportError: pass Codeville-0.8.0/Codeville/DFS.py0000664000076400007640000000205010340212674015645 0ustar rcohenrcohenclass DFS: def __init__(self, dep_func, dep_args): self._ordering = [] self._seen_nodes = {} self._dep_func = dep_func self._dep_args = dep_args return def _node_seen(self, node, others): self._stack.pop() if self._seen_nodes[node] == 0: self._ordering.append(node) self._seen_nodes[node] = 1 if len(others) > 0: self._stack.append((others[0], others[1:])) return def search(self, node): self._stack = stack = [(node, [])] while len(stack): node, others = stack[-1] if self._seen_nodes.has_key(node): self._node_seen(node, others) continue self._seen_nodes[node] = 0 deps = self._dep_func(node, self._dep_args) if len(deps) > 0: stack.append((deps[0], deps[1:])) continue self._node_seen(node, others) return def result(self): return self._ordering Codeville-0.8.0/Codeville/RawServer.py0000664000076400007640000004254310412020343017152 0ustar rcohenrcohen# Written by Bram Cohen # see LICENSE.txt for license information from bisect import insort import socket from cStringIO import StringIO from traceback import print_exc from errno import EWOULDBLOCK, EINTR try: from select import poll, error, POLLIN, POLLOUT, POLLERR, POLLHUP timemult = 1000 except ImportError: from selectpoll import poll, error, POLLIN, POLLOUT, POLLERR, POLLHUP timemult = 1 from threading import Thread, Event from time import time, sleep import sys from random import randrange from traceback import print_stack all = POLLIN | POLLOUT class SingleSocket: def __init__(self, raw_server, sock, handler): self.raw_server = raw_server self.socket = sock self.handler = handler self.buffer = [] self.last_hit = time() self.fileno = sock.fileno() self.connected = False def get_ip(self): try: return self.socket.getpeername()[0] except socket.error: return 'no connection' def close(self, unregister=True): # print 'RawServer close: ' + str(self.fileno) # print_stack() sock = self.socket self.socket = None self.buffer = [] if unregister: self.raw_server.unregister(self.fileno) sock.close() def shutdown(self, val): self.socket.shutdown(val) def is_flushed(self): return len(self.buffer) == 0 def write(self, s): assert self.socket is not None self.buffer.append(s) if self.connected and len(self.buffer) == 1: self.try_write() return not len(self.buffer) def try_write(self): if self.connected: try: while self.buffer != []: amount = self.socket.send(self.buffer[0]) if amount != len(self.buffer[0]): if amount != 0: self.buffer[0] = self.buffer[0][amount:] break del self.buffer[0] except socket.error, e: code, msg = e if code != EWOULDBLOCK: self.raw_server.dead_from_write.append(self) return if self.buffer == []: self.raw_server.poll.register(self.socket, POLLIN) else: self.raw_server.poll.register(self.socket, all) class RawServer: def __init__(self, doneflag, timeout_check_interval, timeout): self.timeout_check_interval = timeout_check_interval self.timeout = timeout self.poll = poll() # {socket: SingleSocket} self.single_sockets = {} self.dead_from_write = [] self.doneflag = doneflag self.funcs = [] self.externally_added = [] self.server = None self.add_task(self.scan_for_timeouts, timeout_check_interval) def add_task(self, func, delay): insort(self.funcs, (time() + delay, func)) def external_add_task(self, func, delay = 0): self.externally_added.append((func, delay)) def scan_for_timeouts(self): self.add_task(self.scan_for_timeouts, self.timeout_check_interval) t = time() - self.timeout tokill = [] for s in self.single_sockets.values(): if s.last_hit < t: tokill.append(s) for k in tokill: if k.socket is not None: self._close_socket(k) def bind(self, port, bind = '', reuse = False): server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if reuse: server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server.setblocking(0) server.bind((bind, port)) server.listen(5) self.poll.register(server, POLLIN) self.server = server def start_connection(self, dns, handler = None): if handler is None: handler = self.handler sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setblocking(0) try: sock.connect_ex(dns) except socket.error: raise except Exception, e: raise socket.error(str(e)) self.poll.register(sock, POLLIN|POLLOUT|POLLERR) s = SingleSocket(self, sock, handler) self.single_sockets[sock.fileno()] = s return s def handle_events(self, events): for sock, event in events: if self.server is not None and sock == self.server.fileno(): if event & (POLLHUP | POLLERR) != 0: self.poll.unregister(self.server) self.server.close() self.errorfunc('lost server socket') else: try: newsock, addr = self.server.accept() newsock.setblocking(0) nss = SingleSocket(self, newsock, self.handler) self.single_sockets[newsock.fileno()] = nss self.poll.register(newsock, POLLIN) self.handler.external_connection_made(nss) except socket.error: sleep(1) else: s = self.single_sockets.get(sock) if s is None: continue if (event & (POLLHUP|POLLERR)) != 0: if s.connected: self._close_socket(s, msg='Remote end closed connection') else: self._close_socket(s, msg='Connection refused') continue s.connected = True if (event & POLLIN) != 0: try: s.last_hit = time() data = s.socket.recv(100000) if data == '': self._close_socket(s, msg='Remote end closed connection') else: s.handler.data_came_in(s, data) except socket.error, e: code, msg = e if code != EWOULDBLOCK: self._close_socket(s, msg=msg) continue if (event & POLLOUT) != 0 and s.socket is not None and not s.is_flushed(): s.try_write() if s.is_flushed(): s.handler.connection_flushed(s) def pop_external(self): try: while True: (a, b) = self.externally_added.pop() self.add_task(a, b) except IndexError: pass def listen_forever(self, handler): self.handler = handler try: while not self.doneflag.isSet(): try: self.pop_external() if len(self.funcs) == 0: period = 2 ** 30 else: period = self.funcs[0][0] - time() if period < 0: period = 0 events = self.poll.poll(period * timemult) if self.doneflag.isSet(): return while len(self.funcs) > 0 and self.funcs[0][0] <= time(): garbage, func = self.funcs[0] del self.funcs[0] try: func() except KeyboardInterrupt: print_exc() return self._close_dead() self.handle_events(events) if self.doneflag.isSet(): return self._close_dead() except error: if self.doneflag.isSet(): return #except KeyboardInterrupt: # print_exc() # return # except: # data = StringIO() # print_exc(file = data) # self.errorfunc(data.getvalue()) finally: # for ss in self.single_sockets.values(): # ss.close() if self.server is not None: self.poll.unregister(self.server) self.server.close() def unregister(self, fd): del self.single_sockets[fd] self.poll.unregister(fd) return def _close_dead(self): while len(self.dead_from_write) > 0: old = self.dead_from_write self.dead_from_write = [] for s in old: if s.socket is not None: self._close_socket(s) def _close_socket(self, s, msg=None): sock = s.socket.fileno() self.poll.unregister(sock) del self.single_sockets[sock] s.socket.close() s.socket = None s.handler.connection_lost(s, msg) # everything below is for testing class DummyHandler: def __init__(self): self.external_made = [] self.data_in = [] self.lost = [] def external_connection_made(self, s): self.external_made.append(s) def data_came_in(self, s, data): self.data_in.append((s, data)) def connection_lost(self, s, msg): self.lost.append(s) def connection_flushed(self, s): pass def sl(rs, handler, port): rs.bind(port) Thread(target = rs.listen_forever, args = [handler]).start() def loop(rs): x = [] def r(rs = rs, x = x): rs.add_task(x[0], .1) x.append(r) rs.add_task(r, .1) beginport = 5000 + randrange(10000) def test_starting_side_close(): try: fa = Event() fb = Event() da = DummyHandler() sa = RawServer(fa, 100, 100) loop(sa) sl(sa, da, beginport) db = DummyHandler() sb = RawServer(fb, 100, 100) loop(sb) sl(sb, db, beginport + 1) sleep(.5) ca = sa.start_connection(('127.0.0.1', beginport + 1)) sleep(1) assert da.external_made == [] assert da.data_in == [] assert da.lost == [] assert len(db.external_made) == 1 cb = db.external_made[0] del db.external_made[:] assert db.data_in == [] assert db.lost == [] ca.write('aaa') cb.write('bbb') sleep(1) assert da.external_made == [] assert da.data_in == [(ca, 'bbb')] del da.data_in[:] assert da.lost == [] assert db.external_made == [] assert db.data_in == [(cb, 'aaa')] del db.data_in[:] assert db.lost == [] ca.write('ccc') cb.write('ddd') sleep(1) assert da.external_made == [] assert da.data_in == [(ca, 'ddd')] del da.data_in[:] assert da.lost == [] assert db.external_made == [] assert db.data_in == [(cb, 'ccc')] del db.data_in[:] assert db.lost == [] ca.close() sleep(1) assert da.external_made == [] assert da.data_in == [] assert da.lost == [] assert db.external_made == [] assert db.data_in == [] assert db.lost == [cb] del db.lost[:] finally: fa.set() fb.set() def test_receiving_side_close(): try: da = DummyHandler() fa = Event() sa = RawServer(fa, 100, 100) loop(sa) sl(sa, da, beginport + 2) db = DummyHandler() fb = Event() sb = RawServer(fb, 100, 100) loop(sb) sl(sb, db, beginport + 3) sleep(.5) ca = sa.start_connection(('127.0.0.1', beginport + 3)) sleep(1) assert da.external_made == [] assert da.data_in == [] assert da.lost == [] assert len(db.external_made) == 1 cb = db.external_made[0] del db.external_made[:] assert db.data_in == [] assert db.lost == [] ca.write('aaa') cb.write('bbb') sleep(1) assert da.external_made == [] assert da.data_in == [(ca, 'bbb')] del da.data_in[:] assert da.lost == [] assert db.external_made == [] assert db.data_in == [(cb, 'aaa')] del db.data_in[:] assert db.lost == [] ca.write('ccc') cb.write('ddd') sleep(1) assert da.external_made == [] assert da.data_in == [(ca, 'ddd')] del da.data_in[:] assert da.lost == [] assert db.external_made == [] assert db.data_in == [(cb, 'ccc')] del db.data_in[:] assert db.lost == [] cb.close() sleep(1) assert da.external_made == [] assert da.data_in == [] assert da.lost == [ca] del da.lost[:] assert db.external_made == [] assert db.data_in == [] assert db.lost == [] finally: fa.set() fb.set() def test_connection_refused(): try: da = DummyHandler() fa = Event() sa = RawServer(fa, 100, 100) loop(sa) sl(sa, da, beginport + 6) sleep(.5) ca = sa.start_connection(('127.0.0.1', beginport + 15)) sleep(1) assert da.external_made == [] assert da.data_in == [] assert da.lost == [ca] del da.lost[:] finally: fa.set() def test_both_close(): try: da = DummyHandler() fa = Event() sa = RawServer(fa, 100, 100) loop(sa) sl(sa, da, beginport + 4) sleep(1) db = DummyHandler() fb = Event() sb = RawServer(fb, 100, 100) loop(sb) sl(sb, db, beginport + 5) sleep(.5) ca = sa.start_connection(('127.0.0.1', beginport + 5)) sleep(1) assert da.external_made == [] assert da.data_in == [] assert da.lost == [] assert len(db.external_made) == 1 cb = db.external_made[0] del db.external_made[:] assert db.data_in == [] assert db.lost == [] ca.write('aaa') cb.write('bbb') sleep(1) assert da.external_made == [] assert da.data_in == [(ca, 'bbb')] del da.data_in[:] assert da.lost == [] assert db.external_made == [] assert db.data_in == [(cb, 'aaa')] del db.data_in[:] assert db.lost == [] ca.write('ccc') cb.write('ddd') sleep(1) assert da.external_made == [] assert da.data_in == [(ca, 'ddd')] del da.data_in[:] assert da.lost == [] assert db.external_made == [] assert db.data_in == [(cb, 'ccc')] del db.data_in[:] assert db.lost == [] sa.unregister(ca.fileno) sb.unregister(cb.fileno) ca.close(unregister=False) cb.close(unregister=False) sleep(1) assert da.external_made == [] assert da.data_in == [] assert da.lost == [] assert db.external_made == [] assert db.data_in == [] assert db.lost == [] finally: fa.set() fb.set() def test_normal(): l = [] f = Event() s = RawServer(f, 100, 100) loop(s) sl(s, DummyHandler(), beginport + 7) s.add_task(lambda l = l: l.append('b'), 2) s.add_task(lambda l = l: l.append('a'), 1) s.add_task(lambda l = l: l.append('d'), 4) sleep(1.5) s.add_task(lambda l = l: l.append('c'), 1.5) sleep(3) assert l == ['a', 'b', 'c', 'd'] f.set() def test_catch_exception(): l = [] f = Event() s = RawServer(f, 100, 100, False) loop(s) sl(s, DummyHandler(), beginport + 9) s.add_task(lambda l = l: l.append('b'), 2) s.add_task(lambda: 4/0, 1) sleep(3) assert l == ['b'] f.set() def test_closes_if_not_hit(): try: da = DummyHandler() fa = Event() sa = RawServer(fa, 2, 2) loop(sa) sl(sa, da, beginport + 14) sleep(1) db = DummyHandler() fb = Event() sb = RawServer(fb, 100, 100) loop(sb) sl(sb, db, beginport + 13) sleep(.5) sa.start_connection(('127.0.0.1', beginport + 13)) sleep(1) assert da.external_made == [] assert da.data_in == [] assert da.lost == [] assert len(db.external_made) == 1 del db.external_made[:] assert db.data_in == [] assert db.lost == [] sleep(3.1) assert len(da.lost) == 1 assert len(db.lost) == 1 finally: fa.set() fb.set() def test_does_not_close_if_hit(): try: fa = Event() fb = Event() da = DummyHandler() sa = RawServer(fa, 2, 2) loop(sa) sl(sa, da, beginport + 12) sleep(1) db = DummyHandler() sb = RawServer(fb, 100, 100) loop(sb) sl(sb, db, beginport + 13) sleep(.5) sa.start_connection(('127.0.0.1', beginport + 13)) sleep(1) assert da.external_made == [] assert da.data_in == [] assert da.lost == [] assert len(db.external_made) == 1 cb = db.external_made[0] del db.external_made[:] assert db.data_in == [] assert db.lost == [] cb.write('bbb') sleep(.5) assert da.lost == [] assert db.lost == [] finally: fa.set() fb.set() Codeville-0.8.0/Codeville/SRP.py0000664000076400007640000001323110340212674015700 0ustar rcohenrcohen"""Secure Remote Passwords. This is slightly different from the standard implementation (with regard to the definition of 'u', the authentication hash, and the fact that the database is a pickle). Also the default random number generator is not cryptographically strong. It may be good enough to password protect your MUD, if not your bank account. Note that the passwd database should not be world readable, or it will be vulnerable to a dictionary attack (like the standard Unix password file). See the SRP distribution at http://srp.stanford.edu/srp/ for more information.""" from entropy import random_long, random_string, string_to_long, long_to_string import hmac import sha # Some constants defining the sizes of various entities. saltlen = 16 # bytes tlen = 128 # bits ablen = 128 # bits # The prime field to work in, and the base to use. Note that this must be # common to both client and host. (Alternatively, the host can send these # values to the client, who should then verify that they are safe.) # The first number is a prime p of the form 2q + 1, where q is also prime. # The second number is a generator in the field GF(p). pf = (137656596376486790043182744734961384933899167257744121335064027192370741112305920493080254690601316526576747330553110881621319493425219214435734356437905637147670206858858966652975541347966997276817657605917471296442404150473520316654025988200256062845025470327802138620845134916799507318209468806715548156999L, 8623462398472349872L) # New exceptions we raise. class ImproperKeyValue(Exception): pass def hash(s): """Hash a value with some hashing algorithm.""" if type(s) != type(''): s = long_to_string(s) return sha.new(s).digest() def private_key(u, s, p): """Given the username, salt, and cleartext password, return the private key, which is the long integer form of the hashed arguments.""" h = hash(s + hash(u + p)) x = string_to_long(h) return x def _create_new_verifier(u, p, pf): """Given a username, cleartext password, and a prime field, pick a random salt and calculate the verifier. The salt, verifier tuple is returned.""" s = random_string(saltlen) n, g = pf v = pow(g, private_key(u, s, p), n) return (s, v) def new_passwd(user, password): salt, verifier = _create_new_verifier(user, password, pf) return (salt, verifier) # This is the authentication protocol. There are two parts, the client and # the host. These functions are called from the client side. def client_begin(): # Here we could optionally query the host for the pfid and salt, or # indeed the pf itself plus salt. We'd have to verify that n and g # are valid in the latter case, and we need a local copy anyway in the # former. pfid = 0 n, g = pf # Pick a random number and send it to the host, who responds with # the user's salt and more random numbers. Note that in the standard # SRP implementation, u is derived from B. a = random_long(ablen) A = pow(g, a, n) return (A, a, g, n) def client_key(user, passphrase, s, B, u, keys, key_func=private_key): A, a, g, n = keys # We don't trust the host. Perhaps the host is being spoofed. if B <= 0 or n <= B: raise ImproperKeyValue # Calculate the shared, secret session key. x = key_func(user, s, passphrase) v = 3 * pow(g, x, n) t = B if t < v: t = t + n S = pow(t - v, a + u * x, n) K = hash(S) # Compute the authentication proof. # This verifies that we do indeed know the same session key, # implying that we knew the correct password (even though the host # doesn't know the password!) m = _client_authenticator(K, n, g, user, s, A, B, u) return (K, m) # The next function is called from the host side. def host_begin(user, A, s, v): """Look the user up in the passwd database, calculate our version of the session key, and return it along with a keyed hash of the values used in the calculation as proof. The client must match this proof.""" n, g = pf # We don't trust the client, who might be trying to send bogus data in # order to break the protocol. if A <= 0 or n <= A: raise ImproperKeyValue # Pick our random public keys. B = 0 while B == 0: b = random_long(ablen) B = ((3*v) + pow(g, b, n)) % n u = pow(g, random_long(tlen), n) # Calculate the (private, shared secret) session key. t = (A * pow(v, u, n)) % n if t <= 1 or t + 1 == n: raise ImproperKeyValue # WeakKeyValue -- could be our fault so retry S = pow(t, b, n) K = hash(S) # Create the proof using a keyed hash. m = _client_authenticator(K, n, g, user, s, A, B, u) return (B, u, K, m) # These two functions calculate the "proofs": keyed hashes of values used # in the computation of the key. def _client_authenticator(K, n, g, user, s, A, B, u): A = long_to_string(A, 128) B = long_to_string(B, 128) u = long_to_string(u, 128) return hmac.new(K, hash(n) + hash(g) + hash(user) + s + A + B + u, sha) def host_authenticator(K, A, m): A = long_to_string(A, 128) return hmac.new(K, A + m, sha) def test_SRP(): s, v = new_passwd('user', 'password') keys = client_begin() B, u, Khost, mhost = host_begin('user', keys[0], s, v) Kclient, mclient = client_key('user', 'password', s, B, u, keys) assert Khost == Kclient assert mhost.digest() == mclient.digest() Kclient, mclient = client_key('user', 'bad password', s, B, u, keys) assert Khost != Kclient assert mhost.digest() != mclient.digest() Kclient, mclient = client_key('user', 'password', '12345', B, u, keys) assert Khost != Kclient assert mhost.digest() != mclient.digest() Codeville-0.8.0/Codeville/__init__.py0000664000076400007640000000002210645744653017004 0ustar rcohenrcohenversion = '0.8.0' Codeville-0.8.0/Codeville/agent.py0000664000076400007640000002020110340212674016325 0ustar rcohenrcohen# Written by Ross Cohen # see LICENSE.txt for license information from bencode import bdecode, bencode from crypt import crypt from errno import EINTR import os try: import select except ImportError: import selectpoll as select import sha import socket import SRP from cStringIO import StringIO import struct AUTH_UNUSED = 0 AUTH_SOCK = 1 AUTH_CONNECTION = 2 class Agent: def __init__(self): self.auth_path = None self.auth_file = None self.poll_obj = select.poll() self.hstatus = {} self.identities = {} self.allow_shutdown = 0 return def listen_sock(self, auth_path, auth_file): if self.auth_file is not None: print 'Auth socket already set' return None self.auth_path = auth_path self.auth_file = auth_file sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.bind(auth_file) sock.listen(8) self._new_socket(AUTH_SOCK, sock) return sock def listen(self): if self.auth_file is None: print 'No socket to listen on' return 1 self.shutdown_flag = 0 while not self.shutdown_flag: try: sock_list = self.poll_obj.poll() except select.error, reason: if reason[0] != EINTR: raise else: self._after_poll(sock_list) self.cleanup() return 0 def cleanup(self): for fd, status in self.hstatus.items(): del self.hstatus[fd] self.poll_obj.unregister(fd) status['sock'].close() os.unlink(self.auth_file) if self.auth_path is not None: auth_path = self.auth_path self.auth_path = None os.rmdir(auth_path) return def _process_message(self, sock): status = self.hstatus[sock.fileno()] input = status['input'] input.seek(status['in_offset']) data = input.read(4) if len(data) != 4: return msg_len, = struct.unpack(' 1024: self._write_error(sock, 'Bad message lenth') self._close_socket(sock) return data = input.read(msg_len) if len(data) < msg_len: return status['in_offset'] += 4 + msg_len try: msg = bdecode(data) except ValueError: self._write_error(sock, 'Bad message') self._close_socket(sock) return type = msg['type'] if type == 'CDV_AGENT_ADD_PASSWORD': id = sha.new('password' + msg['password']).digest() self.identities[id] = msg['password'] self._write_answer(sock, bencode({'id': id})) elif type == 'CDV_AGENT_ADD_SECRET': id = sha.new('public hash check' + msg['secret']).digest() self.identities[id] = msg['secret'] self._write_answer(sock, bencode({'id': id})) elif type == 'CDV_AGENT_ADD_ENCRYPTED_SECRET': if not self.identities.has_key(msg['id']): self._write_error(sock, 'No such identity') return secret = crypt(msg['secret'], self.identities[msg['id']])[0] id = sha.new('public hash check' + secret).digest() if id != msg['secret_id']: self._write_error(sock, 'id does not match') return self.identities[id] = secret self._write_answer(sock, bencode({})) elif type == 'CDV_AGENT_QUERY_IDENTITY': known = 0 if self.identities.has_key(msg['id']): known = 1 self._write_answer(sock, bencode({'known': known})) elif type == 'CDV_AGENT_SRP_PRIVATE_KEY': x = SRP.private_key(msg['user'], msg['s'], self.identities[msg['id']]) self._write_answer(sock, bencode({'x': x})) elif type == 'CDV_AGENT_SESSION_KEY': if not self.identities.has_key(msg['id']): self._write_error(sock, 'No such identity') return if len(msg['salt1']) < 20 or len(msg['salt2']) < 20: self._write_error(sock, 'Bad salts') return if msg['salt1'] == msg['salt2']: self._write_error(sock, 'Bad salts') return base = 'session key' + self.identities[msg['id']] + \ msg['salt1'] + msg['salt2'] key = sha.new(base).digest() answer = {'key': key} self._write_answer(sock, bencode(answer)) elif type == 'CDV_AGENT_SHUTDOWN' and self.allow_shutdown: self.shutdown_flag = 1 else: self._write_error(sock, 'Unknown command: ' + type) return def _after_poll(self, sock_list): for fd, event in sock_list: status = self.hstatus[fd] sock = status['sock'] if status['type'] == AUTH_UNUSED: return elif status['type'] == AUTH_SOCK: tries = 3 while tries: try: nsock = sock.accept() except socket.error, reason: if reason[0] != EINTR: raise tries -= 1 else: break if tries == 0: raise # XXX: python doesn't support SO_PEERCRED #sock.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED) self._new_socket(AUTH_CONNECTION, nsock[0]) elif status['type'] == AUTH_CONNECTION: if event & select.POLLHUP: self._close_socket(sock) return if event & select.POLLIN: data = sock.recv(1024) if len(data) == 0: self._close_socket(sock) return status['input'].seek(0, 2) status['input'].write(data) self._process_message(sock) if event & select.POLLOUT: self._flush_socket(sock) return def _new_socket(self, type, sock): sock.setblocking(0) self.hstatus[sock.fileno()] = {'type': type, 'input': StringIO(), 'output': StringIO(), 'in_offset': 0, 'out_offset': 0, 'sock': sock} flags = select.POLLIN if type != AUTH_SOCK: flags |= select.POLLHUP self.poll_obj.register(sock, flags) return def _close_socket(self, sock): fileno = sock.fileno() self.poll_obj.unregister(fileno) del self.hstatus[fileno] sock.close() return def _write_answer(self, sock, data): data = struct.pack(' 1024: raise AuthError, 'Bad message length from cdv-agent' self._read_sock(input, togo) input.seek(4) ans = bdecode(input.read()) if ans.has_key('error'): raise AuthError, ans['error'] return ans def _read_sock(self, input, togo): while togo > 0: try: data = self.agent_sock.recv(togo) except socket.error, reason: if reason[0] == ECONNRESET: raise AuthError, 'Unexpected close from cdv-agent' raise input.write(data) togo -= len(data) return # for client_key to use def _private_key(self, user, s, foo): msg = {'user': user, 's': s, 'id': self.pwid} msg['type'] = 'CDV_AGENT_SRP_PRIVATE_KEY' ans = self._agent_make_request(msg) return ans['x'] Codeville-0.8.0/Codeville/bencode.py0000664000076400007640000001617010340212674016640 0ustar rcohenrcohen# Written by Petru Paler and Ross Cohen # see LICENSE.txt for license information from types import IntType, LongType, StringType, ListType, TupleType, DictType def decode_int(x, f): f = f+1 newf = x.index('e', f) try: n = int(x[f:newf]) except (OverflowError, ValueError): n = long(x[f:newf]) if x[f] == '-': if x[f + 1] == '0': raise ValueError elif x[f] == '0' and newf != f+1: raise ValueError return (n, newf+1) def decode_string(x, f): colon = x.index(':', f) try: n = int(x[f:colon]) except (OverflowError, ValueError): n = long(x[f:colon]) if x[f] == '0' and colon != f+1: raise ValueError colon += 1 return (x[colon:colon+n], colon+n) def decode_list(x, f): r, f = [], f+1 while x[f] != 'e': v, f = decode_func[x[f]](x, f) r.append(v) return (r, f + 1) def decode_dict(x, f): r, f = {}, f+1 lastkey = None while x[f] != 'e': k, f = decode_string(x, f) if lastkey >= k: raise ValueError lastkey = k r[k], f = decode_func[x[f]](x, f) return (r, f + 1) decode_func = {} decode_func['l'] = decode_list decode_func['d'] = decode_dict decode_func['i'] = decode_int decode_func['0'] = decode_string decode_func['1'] = decode_string decode_func['2'] = decode_string decode_func['3'] = decode_string decode_func['4'] = decode_string decode_func['5'] = decode_string decode_func['6'] = decode_string decode_func['7'] = decode_string decode_func['8'] = decode_string decode_func['9'] = decode_string def bdecode(x): try: r, l = decode_func[x[0]](x, 0) except (IndexError, KeyError): raise ValueError if l != len(x): raise ValueError return r def test_bdecode(): try: bdecode('0:0:') assert 0 except ValueError: pass try: bdecode('ie') assert 0 except ValueError: pass try: bdecode('i341foo382e') assert 0 except ValueError: pass assert bdecode('i4e') == 4L assert bdecode('i0e') == 0L assert bdecode('i123456789e') == 123456789L assert bdecode('i-10e') == -10L try: bdecode('i-0e') assert 0 except ValueError: pass try: bdecode('i123') assert 0 except ValueError: pass try: bdecode('') assert 0 except ValueError: pass try: bdecode('i6easd') assert 0 except ValueError: pass try: bdecode('35208734823ljdahflajhdf') assert 0 except ValueError: pass try: bdecode('2:abfdjslhfld') assert 0 except ValueError: pass assert bdecode('0:') == '' assert bdecode('3:abc') == 'abc' assert bdecode('10:1234567890') == '1234567890' try: bdecode('02:xy') assert 0 except ValueError: pass try: bdecode('l') assert 0 except ValueError: pass assert bdecode('le') == [] try: bdecode('leanfdldjfh') assert 0 except ValueError: pass assert bdecode('l0:0:0:e') == ['', '', ''] try: bdecode('relwjhrlewjh') assert 0 except ValueError: pass assert bdecode('li1ei2ei3ee') == [1, 2, 3] assert bdecode('l3:asd2:xye') == ['asd', 'xy'] assert bdecode('ll5:Alice3:Bobeli2ei3eee') == [['Alice', 'Bob'], [2, 3]] try: bdecode('d') assert 0 except ValueError: pass try: bdecode('defoobar') assert 0 except ValueError: pass assert bdecode('de') == {} assert bdecode('d3:agei25e4:eyes4:bluee') == {'age': 25, 'eyes': 'blue'} assert bdecode('d8:spam.mp3d6:author5:Alice6:lengthi100000eee') == {'spam.mp3': {'author': 'Alice', 'length': 100000}} try: bdecode('d3:fooe') assert 0 except ValueError: pass try: bdecode('di1e0:e') assert 0 except ValueError: pass try: bdecode('d1:b0:1:a0:e') assert 0 except ValueError: pass try: bdecode('d1:a0:1:a0:e') assert 0 except ValueError: pass try: bdecode('i03e') assert 0 except ValueError: pass try: bdecode('l01:ae') assert 0 except ValueError: pass try: bdecode('9999:x') assert 0 except ValueError: pass try: bdecode('l0:') assert 0 except ValueError: pass try: bdecode('d0:0:') assert 0 except ValueError: pass try: bdecode('d0:') assert 0 except ValueError: pass try: bdecode('00:') assert 0 except ValueError: pass try: bdecode('l-3:e') assert 0 except ValueError: pass try: bdecode('i-03e') assert 0 except ValueError: pass class Bencached(object): __slots__ = ['bencoded'] def __init__(self, s): self.bencoded = s def bencode_int(x, b): b.extend(('i', str(x), 'e')) def bencode_string(x, b): b.extend((str(len(x)), ':', x)) def bencode_list(x, b): b.append('l') for e in x: encode_func[type(e)](e, b) b.append('e') def bencode_dict(x, b): b.append('d') klist = x.keys() klist.sort() for k in klist: assert type(k) is StringType b.extend((str(len(k)), ':', k)) encode_func[type(x[k])](x[k], b) b.append('e') def bencode_cached(x, b): b.append(x.bencoded) encode_func = {} encode_func[IntType] = bencode_int encode_func[LongType] = bencode_int encode_func[StringType] = bencode_string encode_func[ListType] = bencode_list encode_func[TupleType] = bencode_list encode_func[DictType] = bencode_dict encode_func[Bencached] = bencode_cached def bencode(x): b = [] try: encode_func[type(x)](x, b) except KeyError: raise ValueError return ''.join(b) def test_bencode(): assert bencode(4) == 'i4e' assert bencode(0) == 'i0e' assert bencode(-10) == 'i-10e' assert bencode(12345678901234567890L) == 'i12345678901234567890e' assert bencode('') == '0:' assert bencode('abc') == '3:abc' assert bencode('1234567890') == '10:1234567890' assert bencode([]) == 'le' assert bencode([1, 2, 3]) == 'li1ei2ei3ee' assert bencode([['Alice', 'Bob'], [2, 3]]) == 'll5:Alice3:Bobeli2ei3eee' assert bencode({}) == 'de' assert bencode({'age': 25, 'eyes': 'blue'}) == 'd3:agei25e4:eyes4:bluee' assert bencode({'spam.mp3': {'author': 'Alice', 'length': 100000}}) == 'd8:spam.mp3d6:author5:Alice6:lengthi100000eee' try: bencode({1: 'foo'}) except AssertionError: return assert 0 try: import psyco psyco.bind(bdecode) psyco.bind(bencode) except ImportError: pass Codeville-0.8.0/Codeville/cdv_glob.py0000664000076400007640000001064310340213023017004 0ustar rcohenrcohen# Written by Ross Cohen # see LICENSE.txt for license information from client_helpers import children_count from client_helpers import filename_to_handle import fnmatch import os, sys from os import path from path import subpath import re class Glob: def __init__(self, co, files): self.co = co self.local = co.local if sys.platform == 'win32': self._glob_cache = {} unglobbedfiles = files files = [] for file in unglobbedfiles: files += self._glob(file) del self._glob_cache # normalize the filenames self.files = [] for file in files: try: self.files.append(subpath(self.local, file)[1]) except ValueError: print 'warning - %s is not under the repository' % (file,) return def fs_walk(self, expand=1): for file in self.files: parent, name = path.split(file) if name == '...': if expand: for lfile in self._fs_expand(parent): yield (lfile, 1) continue if not path.exists(path.join(self.local, file)): print 'warning - %s does not exist' % (file,) continue yield (file, 0) return def db_walk(self, deletes=0, txn=None): for file in self.files: parent, name = path.split(file) if name == '...': for lhandle in self._db_expand(parent, deletes, txn): yield (lhandle, 1) continue handle = filename_to_handle(self.co, file, txn, deletes=deletes) if handle is None: print 'warning - %s is not in the repository' % (file,) continue yield (handle, 0) return def _fs_expand(self, local): root_cmp = path.join(self.local, '.cdv') filter = path.join(self.local, '.cdv', '') root_len = len(self.local + path.sep) filter_len = len(filter) for root, dirs, files, in os.walk(path.join(self.local, local)): if root == root_cmp: dirs[:] = [] continue if root[:filter_len] == filter: dirs[:] = [] continue nroot = root[root_len:] if nroot != local: yield nroot for file in files: yield path.join(nroot, file) return def _db_expand(self, local, deletes, txn): handle = filename_to_handle(self.co, local, txn) dirs = [handle] while len(dirs): lhandle = dirs.pop() children = children_count(self.co, lhandle, txn, deletes=deletes) dirs.extend(children) for handle in children: yield handle return def _glob(self, pathname): if not self._has_magic(pathname): return [pathname] dirname, basename = os.path.split(pathname) if not dirname: return self._glob1(os.curdir, basename) elif self._has_magic(dirname): list = self._glob(dirname) else: list = [dirname] if not self._has_magic(basename): result = [] for dirname in list: if basename or os.path.isdir(dirname): name = os.path.join(dirname, basename) if os.path.lexists(name): result.append(name) else: result = [] for dirname in list: sublist = self._glob1(dirname, basename) for name in sublist: result.append(os.path.join(dirname, name)) if len(result) == 0: result.append(pathname) return result def _glob1(self, dirname, pattern): if not dirname: dirname = os.curdir if dirname in self._glob_cache: names = self._glob_cache[dirname] else: try: names = os.listdir(dirname) except os.error: return [] self._glob_cache[dirname] = names if pattern[0]!='.': names=filter(lambda x: x[0]!='.',names) return fnmatch.filter(names,pattern) _magic_check = re.compile('[*?[]') def _has_magic(self, s): return self._magic_check.search(s) is not None Codeville-0.8.0/Codeville/client.py0000664000076400007640000027141610631557377016544 0ustar rcohenrcohen# Written by Bram Cohen and Ross Cohen # see LICENSE.txt for license information from bencode import bdecode, bencode import binascii from cdv_glob import Glob from client_helpers import new_handle, create_handle, unique_name, _set_name from client_helpers import filename_to_handle from client_helpers import handle_to_filename, _handle_to_filename, handle_name from client_helpers import set_edit, unset_edit, clean_merge_point, gen_changeset from client_helpers import _add_file, conflicts_in_file, CommitError from client_helpers import name_use_count, rename_race, children_count, parent_loop_check, _rename_safe_check from client_helpers import mark_modified_files, find_update_files, find_commit_files from client_net import ClientHandler, ClientError, ServerError from client_net import network_prep, authenticate from db import db, ChangeDBs, write_format_version, write_rebuild_version from DFS import DFS from diff import unified_diff from getpass import getpass from history import HistoryError, roothandle, rootnode from history import dmerge, damerge, rename_conflict_check, db_get from history import sync_history, is_ancestor, _is_ancestor from history import handle_contents_at_point, handles_in_branch from history import handle_name_at_point, fullpath_at_point from history import handle_last_modified from history import short_id, long_id, write_changeset, rebuild_from_points from history import server_to_tuple, tuple_to_server, repo_head from history import dump_changeinfo from history import pretty_print_dag, pretty_print_big_dag from history import simplify_precursors import locale from merge import find_conflict, find_conflict_multiple_safe, find_annotation import merge from network import NetworkError import os from os import path from path import mdir, subpath, breakup, preserving_rename from random import randrange import re from sets import Set import sha import shlex import shutil import stat from sys import maxint, stdin, stdout, version_info, platform, stderr import tempfile from time import ctime, strftime, localtime assert version_info >= (2,3), "Python 2.3 or higher is required" term_encoding = stdin.encoding text_encoding = locale.getpreferredencoding() class CheckoutError(Exception): pass class Checkout: def __init__(self, local, init=False, metadata_dir='.cdv', rw=True): self.local = local self.conf_path = path.join(local, metadata_dir) if init == True: try: os.mkdir(self.conf_path) except OSError, err: raise CheckoutError, 'Could not create metadata directory: %s' % (err[1],) self.dbenv = None self.txn = None txn = None if rw == True: flags = db.DB_CREATE|db.DB_INIT_MPOOL|db.DB_INIT_TXN|db.DB_PRIVATE flags |= db.DB_RECOVER self.dbenv = db.DBEnv() self.dbenv.set_cachesize(0, 4 * 1024 * 1024) self.dbenv.set_lg_bsize(1024 * 1024) self.dbenv.set_get_returns_none(2) self.dbenv.open(self.conf_path, flags) txn = self.txn_begin() self._openDBs(txn, init, rw) self.name_cache = {} self.handle_name_cache = {} self.db_cache = {} self.temppath = path.join(self.conf_path, 'temp') self.cpath = path.join(self.conf_path, 'contents') mdir(self.temppath) mdir(self.cpath) self.nopass = 0 self.user = None if rw == True: if init: write_format_version(self.conf_path) write_rebuild_version(self.conf_path) populate_local_repos(self, txn) self.txn_commit(txn) return def _openDBs(self, txn, init, rw): flags = 0 if not rw: flags |= db.DB_RDONLY if init: flags |= db.DB_CREATE self.lcrepo = db.DB(dbEnv=self.dbenv) self.lcrepo.open('changesets.db', dbtype=db.DB_BTREE, flags=flags, txn=txn) self.linforepo = db.DB(dbEnv=self.dbenv) self.linforepo.open('info.db', dbtype=db.DB_BTREE, flags=flags, txn=txn) self.changesdb = db.DB(dbEnv=self.dbenv) self.changesdb.open('changenums.db', dbtype=db.DB_BTREE, flags=flags, txn=txn) self.branchmapdb = db.DB(dbEnv=self.dbenv) self.branchmapdb.open('branchmap.db', dbtype=db.DB_BTREE, flags=flags, txn=txn) self.branchdb = db.DB(dbEnv=self.dbenv) self.branchdb.open('branch.db', dbtype=db.DB_RECNO, flags=flags, txn=txn) self.staticdb = db.DB(dbEnv=self.dbenv) self.staticdb.open('static.db', dbtype=db.DB_BTREE, flags=flags, txn=txn) # open the mini-dags and their indices self.contents = ChangeDBs(self.dbenv, 'content', flags, txn) self.names = ChangeDBs(self.dbenv, 'name', flags, txn) self.allnamesdb = db.DB(dbEnv=self.dbenv) self.allnamesdb.set_flags(db.DB_DUPSORT) self.allnamesdb.open('allnames.db', dbtype=db.DB_BTREE, flags=flags, txn=txn) # checkout-specific dbs self.modtimesdb = db.DB(dbEnv=self.dbenv) self.modtimesdb.open('modtimes.db', dbtype=db.DB_BTREE, flags=flags, txn=txn) self.editsdb = db.DB(dbEnv=self.dbenv) self.editsdb.open('edits.db', dbtype=db.DB_BTREE, flags=flags, txn=txn) self.varsdb = db.DB(dbEnv=self.dbenv) self.varsdb.open('vars.db', dbtype=db.DB_BTREE, flags=flags, txn=txn) try: self.filenamesdb = db.DB(dbEnv=self.dbenv) self.filenamesdb.open('filenames.db', dbtype=db.DB_BTREE, flags=flags, txn=txn) except db.DBNoSuchFileError: self.filenamesdb = None return def close(self): if self.txn is not None: self.txn_abort(self.txn) self.lcrepo.close() self.linforepo.close() self.changesdb.close() self.branchmapdb.close() self.branchdb.close() self.staticdb.close() self.contents.close() self.names.close() self.allnamesdb.close() self.modtimesdb.close() self.editsdb.close() self.varsdb.close() if self.filenamesdb is not None: self.filenamesdb.close() if self.dbenv is not None: self.dbenv.txn_checkpoint() for lfile in self.dbenv.log_archive(): os.remove(path.join(self.dbenv.db_home, lfile)) self.dbenv.close() return def txn_begin(self): self.txn = self.dbenv.txn_begin() return self.txn def txn_abort(self, txn): assert self.txn == txn self.txn = None return txn.abort() def txn_commit(self, txn): assert self.txn == txn self.txn = None return txn.commit() def cli_init(args): local = args[0] try: co = Checkout(local, init=True) co.close() except CheckoutError, msg: print 'error - %s' % (msg,) return 1 return 0 def add(co, files): ltxn = co.txn_begin() cpats = ignore_patterns(co) glob = Glob(co, files) for file, expanded in glob.fs_walk(): fpath = breakup(file) if '.cdv' in fpath: print 'warning - .cdv is a reserved name' continue # XXX: check for other frobridden names, i.e. '.' and '..' # ignore user specified patterns, but only for expansions if expanded: ignore = 0 for cpat in cpats: if cpat.search(file) is not None: ignore = 1 break if ignore: continue # add all the directories leading up to the file, then the file rep, parent = '', roothandle required = 0 for d in fpath: rep = path.join(rep, d) if rep == file: required = not expanded parent = _add_file(co, rep, parent, required, ltxn) if not parent: co.txn_abort(ltxn) return 1 co.txn_commit(ltxn) return 0 def delete(co, files): co.handle_name_cache = {} ltxn = co.txn_begin() fnames = [] for handle, expanded in Glob(co, files).db_walk(): file = handle_to_filename(co, handle) fnames.append((file, handle)) # reverse the ordering so that dirs are deleted after their contents fnames.sort() fnames.reverse() # update the database for fname, handle in fnames: linfo = handle_name(co, handle, ltxn) if co.editsdb.has_key(handle, ltxn): co.editsdb.delete(handle, txn=ltxn) if linfo.has_key('add'): #unset_edit(co, handle, {}, ltxn) del co.handle_name_cache[handle] else: set_edit(co, handle, {'delete': 1}, ltxn) co.handle_name_cache[handle]['delete'] = 1 # make sure the directories are empty for fname, handle in fnames: linfo = db_get(co, co.staticdb, handle, None) if linfo['type'] == 'dir': if len(children_count(co, handle, ltxn)): print 'error - %s is not empty' % (fname,) co.txn_abort(ltxn) return 1 # finally, do the deleting for fname, handle in fnames: print 'deleting: ' + fname linfo = db_get(co, co.staticdb, handle, None) if linfo['type'] == 'dir': try: os.rmdir(path.join(co.local, fname)) except OSError, err: print 'warning - %s: %s' % (err[1], fname) elif linfo['type'] == 'file': try: os.remove(path.join(co.local, fname)) except OSError, err: print 'warning - %s: %s' % (err[1], fname) co.modtimesdb.delete(handle, txn=ltxn) co.filenamesdb.delete(fname, txn=ltxn) co.txn_commit(ltxn) return 0 def rename(co, oldname, newname): co.handle_name_cache = {} try: loldfile, foldname = subpath(co.local, oldname) except ValueError: print 'error - ' + oldname + ' is outside repository' return 1 ohandle = filename_to_handle(co, foldname) if ohandle is None: print 'error - ' + foldname + ' is not in repository' return 1 if not path.exists(oldname): print 'error - ' + oldname + ' does not exist' return 1 try: lnewfile, fnewname = subpath(co.local, newname) except ValueError: print 'error - ' + newname + ' is outside repository' return 1 dirmove, foo = 0, path.split(newname)[1] if foo == '' or foo == '..' or foo == '.': dirmove = 1 npath, nname = fnewname, '' else: npath, nname = path.split(fnewname) nhandle = filename_to_handle(co, fnewname) if nhandle is not None: #if not dirmove or bdecode(co.staticdb.get(nhandle))['type'] != 'dir': if not dirmove or db_get(co, co.staticdb, nhandle, None)['type'] != 'dir': print 'error - ' + newname + ' already exists in repository' return 1 phandle = nhandle nname = path.split(foldname)[1] fnewname = path.join(fnewname, nname) newname = path.join(newname, nname) else: phandle = filename_to_handle(co, npath) if phandle is None: print 'error - cannot rename into directory not in repository' return 1 if nname == '.cdv': print 'error - .cdv is a reserved name' return 1 if filename_to_handle(co, fnewname) is not None: print 'error - ' + newname + ' already exists in repository' return 1 if path.exists(path.join(co.local, fnewname)): print 'error - ' + newname + ' already exists in filesystem' return 1 print 'renaming: ' + foldname + ' -> ' + fnewname ltxn = co.txn_begin() _set_name(co, ohandle, phandle, nname, ltxn) os.rename(path.join(co.local, foldname), path.join(co.local, fnewname)) _rebuild_fndb(co, ltxn) co.txn_commit(ltxn) return 0 def _rebuild_fndb(co, txn): # XXX: crude, should do partial trees co.filenamesdb.truncate(txn=txn) for handle in co.modtimesdb.keys(txn): lfile = handle_to_filename(co, handle) co.filenamesdb.put(lfile, handle, txn=txn) return def edit(co, files, by_id=False): txn = co.txn_begin() generator = None if by_id: generator = [(binascii.unhexlify(handle), 0) for handle in files] else: generator = Glob(co, files).db_walk() for handle, expanded in generator: file = handle_to_filename(co, handle) sinfo = db_get(co, co.staticdb, handle, None) if sinfo['type'] != 'file': print 'warning - %s is not a file' % (file,) continue print 'editting: %s' % (file,) set_edit(co, handle, {'hash': 1}, txn) co.txn_commit(txn) return 0 def set_password(co): network_prep(co) ch = ClientHandler(co) remote = server_to_tuple(co.repo) try: s = authenticate(co, ch, remote, None, srp=True) newpassword = getpass('New password: ') confpassword = getpass('Confirm new password: ') if newpassword != confpassword: print 'Confirmation failed. Password not changed.' return 1 ch.set_password(s, newpassword) except NetworkError, msg: print 'password failed: ' + str(msg) return 1 except ServerError, msg: print 'password failed: ' + str(msg) return 1 print 'Password changed' return 0 def merge_analyze(co, heads, rhead, named, modified, deleted, txn): # clear out the merges which the new head resolved for us old_heads = heads[:] old_heads.remove(rhead) for handle, binfo in co.editsdb.items(txn): info = bdecode(binfo) if info.has_key('nmerge'): merge = False for head in old_heads: change = handle_last_modified(co, co.names, handle, head, txn) if change is not None and not is_ancestor(co, change, rhead, txn): merge = True if not merge: unset_edit(co, handle, ['nmerge'], txn) if info.has_key('cmerge'): merge = False for head in old_heads: change = handle_last_modified(co, co.contents, handle, head, txn) if change is not None and not is_ancestor(co, change, rhead, txn): merge = True if not merge: unset_edit(co, handle, ['cmerge'], txn) # keep track of what's been merged because we have to generate explicit # merge information for them later. lnamed, lmodified, ladded, ldeleted = \ handles_in_branch(co, [rhead], heads, txn) sdeleted, sldeleted = Set(deleted), Set(ldeleted) sdmerges = sldeleted ^ sdeleted s_all_deleted = sldeleted | sdeleted lnamed = damerge(lnamed, ladded) lmodified = damerge(lmodified, ladded) slnamed, snamed = Set(lnamed), Set(named) for handle in (slnamed & snamed) - s_all_deleted: set_edit(co, handle, {'nmerge': 1}, txn) slmodified, smodified = Set(lmodified), Set(modified) for handle in (slmodified & smodified) - s_all_deleted: set_edit(co, handle, {'cmerge': 1}, txn) for handle in (snamed | smodified) & (sldeleted - sdeleted): set_edit(co, handle, {'delete': 1}, txn) for handle in (slnamed | slmodified) & (sdeleted - sldeleted): set_edit(co, handle, {'delete': 1}, txn) #for handle in named: # for head in heads: # change = handle_last_modified(co, co.names, handle, head, txn) # if change is not None and not is_ancestor(co, change, rhead, txn): # set_edit(co, handle, {'nmerge': 1}, txn) #for handle in modified: # for head in heads: # change = handle_last_modified(co, co.contents, handle, head, txn) # if change is not None and not is_ancestor(co, change, rhead, txn): # set_edit(co, handle, {'cmerge': 1}, txn) return def _update_helper(co, uinfo, named, modified, added, deleted, txn): uinfo['newfiles'] = newfiles = {} uinfo['deletes'] = deletes = {} uinfo['names'] = names = {} uinfo['infofiles'] = infofiles = [] if not co.merge: return ([(handle, 0) for handle in modified], {}) local = co.local rhead = uinfo['head'] heads = bdecode(co.linforepo.get('heads')) if co.branchmapdb.has_key(rhead) and _is_ancestor(co, rhead, heads, None): return ([], {}) named = damerge(named, added, deleted) modified = damerge(modified, added, deleted) co.handle_name_cache = {} try: named_files = find_update_files(co, rhead, named, txn) except HistoryError, msg: raise ClientError, 'Got garbage from repository: ' + str(msg) handles, nconflicts = {}, {} for handle, linfo, rinfo in named_files: if linfo is None: if handle == roothandle: continue if not rinfo.has_key('delete'): newfiles[handle] = 1 handles[handle] = 1 else: handles[handle] = 0 continue if rinfo.has_key('delete'): if not linfo.has_key('delete'): names[handle] = linfo deletes[handle] = 1 handles[handle] = 1 else: deletes[handle] = 0 handles[handle] = 0 continue elif linfo.has_key('delete'): handles[handle] = 0 continue conflict, rename_points = rename_conflict_check(linfo, rinfo) if conflict == 'remote': names[handle] = linfo handles[handle] = 1 elif conflict == 'conflict': #lfile = _handle_to_filename(co, handle, names, txn) #print 'file ' + lfile + ' was renamed both locally and remotely' names[handle] = linfo uname = unique_name(co, linfo['parent'], linfo['name'] + '.nameconflict', txn) _set_name(co, handle, linfo['parent'], uname, txn) infofiles.append((handle, rinfo['parent'], rinfo['name'])) nconflicts[handle] = 1 handles[handle] = 1 # create list of all the handles to be updated, will not include ones # which don't exist and are in a deleted state after being pulled in for handle in modified: if handles.has_key(handle): continue linfo = handle_name(co, handle, txn) if linfo is None: rinfo = handle_name_at_point(co, handle, rhead, txn) if rinfo is None or rinfo.has_key('delete'): handles[handle] = 0 continue elif linfo.has_key('delete'): handles[handle] = 0 continue handles[handle] = 1 orphans = [] for handle in deletes.keys(): for chandle in children_count(co, handle, txn): if deletes.has_key(chandle): continue if chandle in names: continue #file = _handle_to_filename(co, chandle, {}, txn) #print 'parent of ' + file + ' was deleted, orphaning' cinfo = handle_name(co, chandle, txn) names[chandle] = cinfo infofiles.append((chandle, cinfo['parent'], '')) nconflicts[chandle] = 1 handles[chandle] = 1 orphans.append((chandle, cinfo['name'])) # generate the new list of heads, the new one might encompass zero, some # or all of our existing heads pre_heads, temp_heads = heads[:], [] inserted_head = False for head in heads: if is_ancestor(co, head, rhead, txn): if not inserted_head: inserted_head = True temp_heads.append(rhead) else: temp_heads.append(head) if not inserted_head: temp_heads.append(rhead) heads = temp_heads co.linforepo.put('heads', bencode(heads), txn=txn) merge_analyze(co, heads, rhead, named, modified, deleted, txn) # clear the name cache co.handle_name_cache = {} for handle, name in orphans: uname = unique_name(co, roothandle, name + '.orphaned', txn) _set_name(co, handle, roothandle, uname, txn) for handle, linfo, rinfo in named_files: if deletes.has_key(handle): continue if not newfiles.has_key(handle) and linfo is None: continue linfo = handle_name(co, handle, txn) pinfo = handle_name(co, linfo['parent'], txn) if pinfo.has_key('delete'): #file = _handle_to_filename(co, handle, {}, txn) #print 'parent of ' + file + ' was deleted, orphaning' if not newfiles.has_key(handle): names[handle] = linfo uname = unique_name(co, roothandle, linfo['name'] + '.orphaned', txn) _set_name(co, handle, roothandle, uname, txn) infofiles.append((handle, linfo['parent'], '')) nconflicts[handle] = 1 breaks = 1 while breaks: breaks = 0 for handle, info in names.items(): if deletes.has_key(handle): continue #if bdecode(co.staticdb.get(handle, txn=txn))['type'] != 'dir': if db_get(co, co.staticdb, handle, txn)['type'] != 'dir': continue lhandle = parent_loop_check(co, handle, txn) if lhandle == handle: breaks += 1 linfo = handle_name(co, handle, txn) #file = _handle_to_filename(co, handle, names, txn) #rfile = handle_to_filename(co, linfo['parent'], txn) #rfile = path.join(rfile, linfo['name']) #print 'parent loop for ' + file + ' -> ' + rfile uname = unique_name(co, linfo['parent'], linfo['name'] + '.parentloop', txn) _set_name(co, handle, info['parent'], uname, txn) infofiles.append((handle, info['parent'], '')) nconflicts[handle] = 1 newnames = names.keys() newnames.extend(newfiles.keys()) for handle in newnames: if deletes.has_key(handle): continue lhandles = name_use_count(co, handle, txn) if len(lhandles) == 1: continue lhandles.remove(handle) lhandle = lhandles[0] #print 'name conflict for ' + _handle_to_filename(co, handle, {}, txn) linfo = handle_name(co, lhandle, txn) names[lhandle] = linfo uname = unique_name(co, linfo['parent'], linfo['name'] + '.nameconflict.local', txn) _set_name(co, lhandle, linfo['parent'], uname, txn) nconflicts[lhandle] = 1 handles[lhandle] = 1 linfo = handle_name(co, handle, txn) uname = unique_name(co, linfo['parent'], linfo['name'] + '.nameconflict.remote', txn) _set_name(co, handle, linfo['parent'], uname, txn) nconflicts[handle] = 1 for handle in newnames: if deletes.has_key(handle): continue linfo = handle_name(co, handle, txn) if newfiles.has_key(linfo['parent']): continue lfile = _handle_to_filename(co, linfo['parent'], names, txn) if not path.exists(path.join(local, lfile)): raise ClientError, 'directory ' + lfile + ' does not exist, you must revert it before updating' mode = os.lstat(path.join(local, lfile)).st_mode if not stat.S_ISDIR(mode): raise ClientError, lfile + ' is not a directory, you must revert it before updating' chandle = rename_race(co, handle, names, txn) lfile = path.join(lfile, linfo['name']) if not chandle and path.exists(path.join(local, lfile)): raise ClientError, 'file ' + lfile + ' was added or renamed remotely but already exists, you must move or delete' for handle in names.keys(): lfile = _handle_to_filename(co, handle, names, txn) if not path.exists(path.join(local, lfile)): raise ClientError, 'file ' + lfile + ' does not exist, you must revert it before updating' #linfo = bdecode(co.staticdb.get(handle, txn=txn)) linfo = db_get(co, co.staticdb, handle, txn) mode = os.lstat(path.join(local, lfile)).st_mode if linfo['type'] == 'file' and not stat.S_ISREG(mode): raise ClientError, 'file ' + lfile + ' is expected to be of type ' + linfo['type'] if linfo['type'] == 'dir' and not stat.S_ISDIR(mode): raise ClientError, 'file ' + lfile + ' is expected to be of type ' + linfo['type'] return handles.items(), nconflicts def update(co, remote, merge=True): try: network_prep(co) except NetworkError, msg: print msg return 1 editsdb, local = co.editsdb, co.local txn = co.txn_begin() mark_modified_files(co, txn) co.txn_commit(txn) co.merge = merge ch = ClientHandler(co) txn = co.txn_begin() try: s = authenticate(co, ch, remote, txn) co.txn_commit(txn) except NetworkError, msg: co.txn_abort(txn) print 'update failed: ' + str(msg) return 1 except ServerError, msg: co.txn_abort(txn) print 'update failed: ' + str(msg) return 1 txn = co.txn_begin() try: updateinfo = ch.update(s, remote[2], _update_helper, txn) except NetworkError, msg: print 'update failed: ' + str(msg) co.txn_abort(txn) return 1 except (ClientError, ServerError), msg: print 'update failed: ' + str(msg) co.txn_abort(txn) ch.close(s) return 1 ch.close(s) ch = None # record the repo head so we can do things relative to it later co.linforepo.put(tuple_to_server(remote), updateinfo['head'], txn=txn) if co.txn_commit(txn): print 'Updating local database failed, aborting...' return 1 if not co.merge: print 'Repository head is ' + short_id(co, updateinfo['head']) print 'Update succeeded' return 0 modified_files = updateinfo['modified'] if not updateinfo.has_key('newfiles'): print 'Update succeeded' return 0 newfiles = updateinfo['newfiles'] deletes = updateinfo['deletes'] names = updateinfo['names'] infofiles = updateinfo['infofiles'] # XXX: need to do something about making updates atomic txn = co.txn_begin() create_dirs = [] for handle in newfiles.keys(): lfile = _handle_to_filename(co, handle, names, None) #staticinfo = bdecode(co.staticdb.get(handle, txn=txn)) staticinfo = db_get(co, co.staticdb, handle, txn) if staticinfo['type'] == 'dir': create_dirs.append(lfile) else: co.filenamesdb.put(lfile, handle, txn=txn) do_fndb_rebuild = False if names != {}: do_fndb_rebuild = True renames = names.keys() while renames: handle = renames.pop() if deletes.has_key(handle) or newfiles.has_key(handle): continue if not _rename_safe_check(co, handle, names, None): renames.insert(0, handle) continue spath = _handle_to_filename(co, handle, names, None) names[handle]['name'] = binascii.hexlify(handle) names[handle]['parent'] = roothandle dpath = _handle_to_filename(co, handle, names, None) os.rename(path.join(local, spath), path.join(local, dpath)) delete_files, delete_dirs = [], [] for handle, present in deletes.items(): if not present: continue #info = bdecode(co.staticdb.get(handle)) info = db_get(co, co.staticdb, handle, None) lfile = _handle_to_filename(co, handle, names, None) if info['type'] == 'dir': delete_dirs.append(lfile) elif info['type'] == 'file': delete_files.append((lfile, handle)) if co.editsdb.has_key(handle, txn): co.editsdb.delete(handle, txn=txn) del names[handle] for lfile, handle in delete_files: os.remove(path.join(local, lfile)) co.modtimesdb.delete(handle, txn=txn) co.filenamesdb.delete(lfile, txn=txn) if do_fndb_rebuild: _rebuild_fndb(co, txn) delete_dirs.sort() delete_dirs.reverse() for lfile in delete_dirs: try: os.rmdir(path.join(local, lfile)) except OSError: print 'warning - %s could not be deleted because it is not empty' % \ (lfile,) create_dirs.sort() for lfile in create_dirs: mdir(path.join(local, lfile)) for handle in names.keys(): spath = _handle_to_filename(co, handle, names, None) del names[handle] dpath = _handle_to_filename(co, handle, names, None) os.rename(path.join(local, spath), path.join(local, dpath)) for handle in modified_files: if deletes.has_key(handle): continue temppath = path.join(co.temppath, binascii.hexlify(handle)) filename = path.join(co.local, _handle_to_filename(co, handle, names, txn)) preserving_rename(temppath, filename) for handle, rparent, rname in infofiles: assert not deletes.has_key(handle) info = handle_name(co, handle, txn) file = unique_name(co, info['parent'], _handle_to_filename(co, handle, names, None) + '.info', None) rfile = path.join(handle_to_filename(co, rparent), rname) h = open(path.join(local, file), 'wb') h.write(rfile + '\n') h.close() if co.txn_commit(txn): print 'Updating local database failed, aborting...' return 1 print 'Update succeeded' return 0 def cli_construct(co, spoint): point = long_id(co, spoint) # now create everything at the specified point adds, deletes = handles_in_branch(co, [rootnode], [point], None)[2:4] deletes_dict = {}.fromkeys(deletes) newfiles = [] for handle in adds: if deletes_dict.has_key(handle): continue hfile = fullpath_at_point(co, handle, point, None) htype = bdecode(co.staticdb.get(handle))['type'] newfiles.append((hfile, handle, htype)) newfiles.sort() for hfile, handle, htype in newfiles: print 'preparing: %s' % (hfile,) if htype == 'file': cinfo = handle_contents_at_point(co, handle, point, None) temppath = path.join(co.temppath, binascii.hexlify(handle)) fd = open(temppath, 'wb') fd.write('\n'.join(cinfo['lines'])) fd.close() # put together a list of all the files we are managing handles = Glob(co, [path.join(co.local, '...')]).db_walk(deletes=0) sheep = [] for handle, expanded in handles: hfile = handle_to_filename(co, handle, None) sheep.append((hfile, handle)) # delete all of them sheep.sort() sheep.reverse() for hfile, handle in sheep: print 'removing: %s' % (hfile,) destpath = path.join(co.local, hfile) htype = bdecode(co.staticdb.get(handle))['type'] try: if htype == 'dir': os.rmdir(destpath) else: os.unlink(destpath) except OSError, msg: print 'warning - %s' % (str(msg),) txn = co.txn_begin() # clear out whatever we were editing co.editsdb.truncate(txn) # rename everything to the right place co.modtimesdb.truncate(txn) co.filenamesdb.truncate(txn) for hfile, handle, htype in newfiles: print 'creating: %s' % (hfile,) destpath = path.join(co.local, hfile) if htype == 'dir': try: os.mkdir(destpath) except OSError: if not os.path.isdir(destpath): raise continue elif htype == 'file': temppath = path.join(co.temppath, binascii.hexlify(handle)) preserving_rename(temppath, destpath) mtime = int(path.getmtime(destpath)) co.modtimesdb.put(handle, bencode(mtime), txn=txn) co.filenamesdb.put(hfile, handle, txn=txn) co.linforepo.put('heads', bencode([point]), txn=txn) co.txn_commit(txn) return 0 def rebuild(co, uheads): txn = co.txn_begin() if uheads == []: heads = bdecode(co.linforepo.get('heads')) else: heads = [long_id(co, head) for head in uheads] try: rebuild_from_points(co, heads, txn) except HistoryError, msg: print 'error - ' + str(msg) co.txn_abort(txn) return 1 co.filenamesdb.truncate(txn) for handle in co.modtimesdb.keys(): hinfo = handle_name(co, handle, None) if hinfo is None or hinfo.has_key('delete'): co.modtimesdb.delete(handle, txn) if co.editsdb.has_key(handle): co.editsdb.delete(handle, txn) lfile = handle_to_filename(co, handle, txn) co.filenamesdb.put(lfile, handle, txn=txn) for handle, value in co.editsdb.items(txn): if bdecode(value).has_key('delete'): co.editsdb.delete(handle, txn) set_edit(co, handle, {'delete': 1}, txn) for handle, value in co.editsdb.items(txn): linfo = bdecode(value) merges = [] if linfo.has_key('nmerge'): merges.append('nmerge') if linfo.has_key('cmerge'): merges.append('cmerge') unset_edit(co, handle, merges, txn) for i in range(1, len(heads)): named, modified, added, deleted = \ handles_in_branch(co, heads[:i], [heads[i]], txn) named = damerge(named, added, deleted) modified = damerge(modified, added, deleted) merge_analyze(co, heads[:i+1], heads[i], named, modified, deleted, txn) print 'Rebuild done.' co.txn_commit(txn) write_rebuild_version(co.conf_path) return 0 def cli_is_ancestor(co, point1, point2): a, b = long_id(co, point1), long_id(co, point2) if is_ancestor(co, a, b, None): print point1 + ' is an ancestor of ' + point2 return 0 print point1 + ' is not an ancestor of ' + point2 return 1 def cli_print_dag(co, uheads): if uheads == []: heads = bdecode(co.linforepo.get('heads')) else: heads = [long_id(co, head) for head in uheads] pretty_print_big_dag(co, heads) return 0 def cli_print_mini_dag(co, file, uheads, by_id): if by_id: handle = binascii.unhexlify(file) else: fname = subpath(co.local, file)[1] handle = filename_to_handle(co, fname) if uheads == []: heads = bdecode(co.linforepo.get('heads')) else: heads = [long_id(co, head) for head in uheads] pretty_print_dag(co, handle, heads) return 0 def cli_handle_to_filename(co, head, handles): handle = binascii.unhexlify(handles[0]) if head == 'local': name = handle_to_filename(co, handle) else: point = long_id(co, head) name = fullpath_at_point(co, handle, point) print name return 0 def populate_local_repos(co, ltxn): if co.linforepo.has_key('heads', ltxn): return root = bencode({'precursors': [], 'handles': {roothandle: {'add': {'type': 'dir'}, 'name': ''}}}) head = sha.new(root).digest() assert head == rootnode co.lcrepo.put(head, root, txn=ltxn) co.linforepo.put('heads', bencode([head]), txn=ltxn) co.linforepo.put('branchmax', bencode(0), txn=ltxn) co.linforepo.put('lasthandle', bencode(0), txn=ltxn) sync_history(co, head, ltxn) return def _list_merge_files(co): if co.repo is None: return [] repohead = repo_head(co, co.repo) if repohead == None: repohead = rootnode heads = bdecode(co.linforepo.get('heads')) named, modified, added, deleted = \ handles_in_branch(co, [repohead], heads, None) handles = damerge(named, modified, added, deleted) named, modified, added, deleted = \ Set(named), Set(modified), Set(added), Set(deleted) files = [] for handle in handles: mletter, nletter = ' ', ' ' if handle in added: if handle in deleted: continue mletter, nletter = 'A', 'A' elif handle in deleted: mletter, nletter = 'D', 'D' else: if handle in named: nletter = 'N' if handle in modified: mletter = 'M' assert not (mletter == ' ' and nletter == ' ') files.append((handle_to_filename(co, handle), mletter + nletter)) files.sort() return files class GarbledCommitError(StandardError): pass def _commit_helper(co, commit_files): output = [] # check for a saved comment comment_file = path.join(co.conf_path, '.comment') if path.exists(comment_file): comment_fd = file(comment_file, 'r') comment = comment_fd.read() comment_fd.close() if comment[-1] == '\n': comment = comment[:-1] output.append(comment) else: output.append('') output.extend(['### Enter comment above', '### Files']) files, name_map = [], {} for handle, info in commit_files: name = handle_to_filename(co, handle) files.append((name, ''.join(_letters(info)))) name_map[name] = (handle, info) files.sort() for name, letters in files: output.append("%s\t%s" % (letters, name)) files = _list_merge_files(co) if len(files): output.append('### Merge files') for name, letters in files: output.append('%s\t%s' % (letters, name)) output.append(os.linesep) fd, fname = tempfile.mkstemp() fhandle = os.fdopen(fd, 'w+') out_str = os.linesep.join(output) out_ustr = out_str.decode('utf8') out_str = out_ustr.encode(text_encoding) fhandle.write(out_str) fhandle.close() if platform == 'win32': spawn = os.spawnv editor = os.environ['WINDIR'] + '\\notepad.exe' else: spawn = os.spawnvp editor = 'vi' if os.environ.has_key('CDVEDITOR'): editor = os.environ['CDVEDITOR'] elif os.environ.has_key('EDITOR'): editor = os.environ['EDITOR'] args = editor.split() + [fname] errored = True while errored: errored = False if spawn(os.P_WAIT, args[0], args): raise CommitError, 'Could not run editor "%s"' % (editor,) fhandle = open(fname, 'rU') text = fhandle.read() fhandle.close() try: try: utext = text.decode(text_encoding) except UnicodeDecodeError: raise GarbledCommitError, \ "Invalid %s characters in comment" % (text_encoding,) lines = utext.encode('utf8').splitlines() cur_line = 0 try: while lines[cur_line].startswith('### Error: '): lines.pop(cur_line) except IndexError: pass while cur_line < len(lines): if lines[cur_line] == '### Enter comment above': break cur_line += 1 if cur_line == len(lines): raise GarbledCommitError, "Could not find end of comment" comment = '\n'.join(lines[:cur_line]) cur_line += 1 if lines[cur_line] != '### Files': raise GarbledCommitError, "Expected '### Files' line after end of comment" fcommit_files = [] cur_line += 1 while cur_line < len(lines): line = lines[cur_line] cur_line += 1 if line.strip() == '': continue if line.startswith('### '): break try: tab = line.index('\t') except ValueError: raise GarbledCommitError, "Bad commit file line:\n%s" % \ (line[:].rstrip(),) name = line[tab+1:].rstrip() fcommit_files.append(name_map[name]) except GarbledCommitError, msg: error = msg.args[0] errored = True if errored: answer = raw_input("Error: %s\nReturn to editor? [Y/n]: " % (error,)) if answer == 'n': raise CommitError, msg output = ['### Error: %s' % (line,) for line in error.split('\n')] out_str = os.linesep.join(output) out_ustr = out_str.decode('utf8') out_str = out_ustr.encode(text_encoding) fhandle = open(fname, 'w') fhandle.write(out_str) fhandle.write(text) fhandle.close() os.remove(fname) return fcommit_files, comment def commit(co, remote, comment, tstamp=None, backup=False, files=list()): try: if remote is None: if co.user is None: co.user = co.varsdb.get('user') if co.user is None: raise NetworkError, 'You must set the "user" variable' repohead = None else: network_prep(co) repohead = repo_head(co, co.repo) if repohead is None: repohead = rootnode except NetworkError, msg: print msg return 1 co.handle_name_cache = {} ltxn = co.txn_begin() mark_modified_files(co, ltxn) co.txn_commit(ltxn) try: if files == []: handles = [(handle, 0) for handle in co.editsdb.keys()] else: handles = Glob(co, files).db_walk(deletes=1) commit_files = find_commit_files(co, handles) # get the comment from the user if it was supplied on the command line if not backup: if comment is None: commit_files, comment = _commit_helper(co, commit_files) # clean up the comment a bit comment = comment.rstrip() if comment == '' and not co.nopass: print 'No comment given, aborting.' return 1 comment = comment + '\n' except CommitError, msg: print 'commit failed: ' + str(msg) return 1 if comment is not None: # save comment in case commit fails tmp_fd, tmp_file = tempfile.mkstemp(dir=co.conf_path) tmp_fd = os.fdopen(tmp_fd, 'w') tmp_fd.write(comment) tmp_fd.close() comment_file = path.join(co.conf_path, '.comment') shutil.move(tmp_file, comment_file) # create and verify the changeset ltxn = co.txn_begin() point = None try: if not backup: point = gen_changeset(co, commit_files, comment, repohead, ltxn, tstamp=tstamp) except HistoryError, msg: co.txn_abort(ltxn) print 'error - ' + str(msg) return 1 if point is not None: try: sync_history(co, point, ltxn) except HistoryError, msg: print 'commit failed: ' + str(msg) print 'This is likely the result of a partial commit with missing dependencies. Alternatively, it could be a bug in Codeville. If you believe this to be the case, please report this to the development list.' co.txn_abort(ltxn) return 1 else: precursors = bdecode(co.linforepo.get('heads', txn=ltxn)) point = precursors[0] if len(precursors) > 1 and backup is True: print 'error - cannot use backup flag when merging' co.txn_abort(ltxn) return 1 if remote is not None: ch = ClientHandler(co) try: s = authenticate(co, ch, remote, ltxn) except NetworkError, msg: co.txn_abort(ltxn) print 'commit failed: ' + str(msg) return 1 except ServerError, msg: co.txn_abort(ltxn) print 'commit failed: ' + str(msg) return 1 try: ch.commit(s, remote[2], point, ltxn) except NetworkError, msg: co.txn_abort(ltxn) print 'commit failed: ' + str(msg) return 1 except ServerError, msg: ch.close(s) co.txn_abort(ltxn) print 'commit failed: ' + str(msg) return 1 ch.close(s) ch = None co.linforepo.put(tuple_to_server(remote), point, txn=ltxn) co.txn_commit(ltxn) # remove saved comment file os.remove(comment_file) print 'commit succeeded' return 0 def create_repo(co, remote): try: network_prep(co) except NetworkError, msg: print msg return 1 ch = ClientHandler(co) txn = co.txn_begin() try: s = authenticate(co, ch, remote, txn) co.txn_commit(txn) except NetworkError, msg: co.txn_abort(txn) print 'creation failed: ' + str(msg) return 1 except ServerError, msg: co.txn_abort(txn) print 'creation failed: ' + str(msg) return 1 try: ch.create_repo(s, remote[2]) except NetworkError, msg: print 'creation failed: ' + str(msg) return 1 except ServerError, msg: print 'creation failed: ' + str(msg) retval = 1 else: print 'creation succeeded' retval = 0 ch.close(s) return retval def remove_repo(co, remote): try: network_prep(co) except NetworkError, msg: print msg return 1 ch = ClientHandler(co) txn = co.txn_begin() try: s = authenticate(co, ch, remote, txn) co.txn_commit(txn) except NetworkError, msg: co.txn_abort(txn) print 'destroy failed: ' + str(msg) return 1 except ServerError, msg: co.txn_abort(txn) print 'destroy failed: ' + str(msg) return 1 try: ch.remove_repo(s, remote[2]) except NetworkError, msg: print 'destroy failed: ' + str(msg) return 1 except ServerError, msg: print 'destroy failed: ' + str(msg) retval = 1 else: print 'destroy succeeded' retval = 0 ch.close(s) return retval def list_repos(co): try: network_prep(co) except NetworkError, msg: print msg return 1 ch = ClientHandler(co) txn = co.txn_begin() try: s = authenticate(co, ch, server_to_tuple(co.repo), txn) co.txn_commit(txn) except NetworkError, msg: co.txn_abort(txn) print 'list failed: ' + str(msg) return 1 except ServerError, msg: co.txn_abort(txn) print 'list failed: ' + str(msg) return 1 try: rlist = ch.list_repos(s) except NetworkError, msg: print 'list failed: ' + str(msg) return 1 except ServerError, msg: print 'list failed: ' + str(msg) ch.close(s) return 1 ch.close(s) print 'Server has the following repositories:\n\t', print '\n\t'.join(rlist) return 0 def _letters(value): if value.has_key('add'): return ['A', 'A'] if value.has_key('delete'): return ['D', 'D'] letters = [' ', ' '] if value.has_key('hash'): letters[0] = 'M' if value.has_key('name'): letters[1] = 'N' return letters def describe(co, point, short, xml, dodiff, files): point = long_id(co, point) if xml: try: print dump_changeinfo(co, point) except ValueError: print 'error - XML can only be written for clean merges.' return 1 return 0 cset = bdecode(co.lcrepo.get(point)) _print_change(co, point, cset, not short) if dodiff and point != rootnode: return diff(co, [short_id(co, cset['precursors'][0]), short_id(co, point)], files, True) return 0 def status(co, files, verbose): ltxn = co.txn_begin() mark_modified_files(co, ltxn) co.txn_commit(ltxn) # go do the stuff if verbose: try: plist = _status_verbose(co, files) except re.error, msg: print 'error - bad ignore list: %s' % (str(msg),) return 1 else: plist = _status(co, files) # print the list of modified files if len(plist): print '### Files' plist.sort() olist = [] for value in plist: olist.append(value[1][0] + value[1][1] + '\t' + value[0]) print os.linesep.join(olist) # print the list of merged files plist = _list_merge_files(co) if len(plist): print '### Merge files' for name, letters in plist: print letters + '\t' + name return 0 def _status(co, files): # collect info on editted files ed_set = Set(co.editsdb.keys()) # no args means show all editted files if files == []: db_set = ed_set else: db_set = Set([handle for handle, expanded in Glob(co, files).db_walk(deletes=1)]) # print the ones we care about plist = [] for handle in (db_set & ed_set): file = handle_to_filename(co, handle) info = bdecode(co.editsdb.get(handle)) plist.append((file, _letters(info))) return plist def _status_verbose(co, files): # Read ignore patterns cpats = ignore_patterns(co) # no args means search the whole client if files == []: files.append(path.join(co.local, '...')) glob = Glob(co, files) # do the filename expansion fs_set = Set([file for file, expanded in glob.fs_walk()]) # get the list of files we manage db_set = Set([handle_to_filename(co, handle) for handle, expanded in glob.db_walk(deletes=1)]) # collect info on editted files ed_set, de_set = Set(), Set() for handle, value in co.editsdb.items(): file = handle_to_filename(co, handle) if bdecode(value).has_key('delete'): de_set.add(file) ed_set.add(file) plist = [] # record unmanaged files for file in (fs_set - db_set): ignore = 0 for cpat in cpats: if cpat.search(file) is not None: ignore = 1 break if ignore: continue plist.append((file, ['?', '?'])) # record files inconsistent in the filesystem for file in (db_set - fs_set - de_set): plist.append((file, ['!', '!'])) # record all the modified files for file in ((ed_set & db_set) - (db_set - fs_set - de_set)): handle = filename_to_handle(co, file, deletes=1) hinfo = bdecode(co.editsdb.get(handle)) plist.append((file, _letters(hinfo))) return plist def cli_heads(co): heads = bdecode(co.linforepo.get('heads')) pheads = [short_id(co, head) for head in heads] print ', '.join(pheads) return 0 def cli_last_modified(co, lname, uhead, by_id): if by_id: ohandle = binascii.unhexlify(lname) else: try: lfile, fname = subpath(co.local, lname) except ValueError: print 'error - ' + lname + ' is outside repository' return 1 ohandle = filename_to_handle(co, fname) if ohandle is None: print 'error - ' + fname + ' is not in repository' return 1 repohead = None if uhead is None: repohead = repo_head(co, co.repo) if repohead is None: heads = bdecode(co.linforepo.get('heads')) repohead = heads[0] else: repohead = long_id(co, uhead) point = handle_last_modified(co, co.contents, ohandle, repohead, None) print short_id(co, point) return 0 def ignore_patterns(co): patterns = [] try: fd = open(path.join(co.conf_path, 'ignore'), 'rU') patterns = fd.readlines() fd.close() except IOError: pass # compile all the patterns and ensure they match full paths return [re.compile('^%s$' % pat.strip()) for pat in patterns] def diff(co, revs, files, print_new): OK = 0 NEW_FILE = 1 DELETED = 2 MISSING = 3 def file_lines(handle, rev, lfile): if rev == 'local': linfo = handle_name(co, handle, None) if linfo is None: return (NEW_FILE, ['']) if linfo.has_key('delete'): return (DELETED, ['']) try: h = open(path.join(co.local, lfile), 'rb') lines = h.read().split('\n') h.close() except IOError: return (MISSING, ['']) else: linfo = handle_name_at_point(co, handle, rev, None) if linfo is None: return (NEW_FILE, ['']) if linfo.has_key('delete'): return (DELETED, ['']) pinfo = handle_contents_at_point(co, handle, rev, None) lines = pinfo['lines'] return (OK, lines) def print_format(error, lfile): if error == OK: return (1, lfile, None) elif error == NEW_FILE: return (0, '(new file)', 'File "%s" added.') elif error == DELETED: return (0, '(deleted)', 'File "%s" deleted.') elif error == MISSING: print 'WARNING - File not found: ' + lfile return (0, '(File not found!)', None) assert 0 return co.handle_name_cache = {} ltxn = co.txn_begin() mark_modified_files(co, ltxn) co.txn_commit(ltxn) editsdb = co.editsdb pathfunc, patharg = [], [] heads = bdecode(co.linforepo.get('heads')) for i in xrange(len(revs)): if revs[i] == 'repo': revs[i] = repo_head(co, co.repo) pathfunc.append(fullpath_at_point) patharg.append(revs[i]) elif revs[i] == 'local': pathfunc.append(handle_to_filename) patharg.append(None) else: if revs[i] is None: revs[i] = heads[0] else: revs[i] = long_id(co, revs[i]) pathfunc.append(fullpath_at_point) patharg.append(revs[i]) branch = [] for i in xrange(len(revs)): if revs[i] == 'local': branch.append(heads) else: branch.append([revs[i]]) # assemble a list of all files with changes named, modified, added, deleted = \ handles_in_branch(co, branch[0], branch[1], None) names2, modified2, added2, deleted2 = \ handles_in_branch(co, branch[1], branch[0], None) all_handles = dmerge(modified, modified2) # include local edits if diffing against the local tree if revs[0] == 'local' or revs[1] == 'local': all_handles = dmerge(all_handles, editsdb.keys()) handles = [] if files == []: if print_new: handles = damerge(all_handles, added, deleted, added2, deleted2) else: handles = all_handles else: dall_handles = Set(all_handles) for handle, expanded in Glob(co, files).db_walk(): if expanded and not handle in dall_handles: continue handles.append(handle) diffprog, diffpath = None, None if os.environ.has_key('CDVDIFF'): cdvdiff = shlex.split(os.environ['CDVDIFF']) if cdvdiff != []: diffargs = cdvdiff diffprog = diffargs[0] if platform == 'win32': # windows inteprets the argument, excitement abounds diffargs = ['"%s"' % (arg,) for arg in diffargs] diffpath = tempfile.mkdtemp('', 'cdv-') fd = open(path.join(diffpath, 'holder'), 'a') fd.close() hlist = [] for handle in handles: hlist.append((pathfunc[0](co, handle, patharg[0]), pathfunc[1](co, handle, patharg[1]), handle)) hlist.sort() if platform == 'win32': spawn = os.spawnv else: spawn = os.spawnvp retval = 0 for pre_lfile, lfile, handle in hlist: #linfo = bdecode(co.staticdb.get(handle)) linfo = db_get(co, co.staticdb, handle, None) if linfo['type'] != 'file': continue error, pre_lines = file_lines(handle, revs[0], pre_lfile) printable, pre_lfile, msg0 = print_format(error, pre_lfile) error, lines = file_lines(handle, revs[1], lfile) printable2, lfile, msg1 = print_format(error, lfile) printable += printable2 if printable == 0: continue if printable == 1 and not print_new: if msg0 is not None: print msg0 % lfile if msg1 is not None: print msg1 % pre_lfile continue if diffprog: file1 = path.join(diffpath, 'old', pre_lfile) file2 = path.join(diffpath, 'new', lfile) os.makedirs(path.split(file1)[0]) foo = open(file1, 'w+') foo.write('\n'.join(pre_lines)) foo.close() os.makedirs(path.split(file2)[0]) foo = open(file2, 'w+') foo.write('\n'.join(lines)) foo.close() fileargs = [file1, file2] if platform == 'win32': fileargs = ['"%s"' % (arg) for arg in fileargs] args = diffargs + fileargs try: ret = spawn(os.P_WAIT, diffprog, args) except OSError: ret = 127 os.remove(file1) os.removedirs(path.split(file1)[0]) os.remove(file2) os.removedirs(path.split(file2)[0]) if ret == 127: print "error - Could not run diff program specified by CDVDIFF" retval = 1 break else: print '--- ' + pre_lfile print '+++ ' + lfile # the diff code assumes \n after each line, not between lines if pre_lines[-1] == '': pre_lines.pop() if lines[-1] == '': lines.pop() stdout.write(unified_diff(pre_lines, lines)) if diffpath is not None: os.unlink(path.join(diffpath, 'holder')) os.rmdir(diffpath) return retval def _comment_compress(comment): try: offset = comment.index('\n') comment = comment[:offset] except ValueError: pass if len(comment) > 76: comment = comment[:73] + '...' return comment def _print_change(co, point, pinfo, v, owner=None, time=None): if not v: print 'Change %s' % (short_id(co, point),), if owner is not None: print '(%s)' % (owner,), print 'by ' + pinfo['user'], if time is not None: print 'on ' + ctime(time) elif pinfo.has_key('time'): print 'on ' + ctime(pinfo['time']) if pinfo.has_key('comment'): print '"' + _comment_compress(pinfo['comment']) + '"' else: print '### Change: ' + binascii.hexlify(point) print '### Short change: ' + short_id(co, point) if owner is not None: print '### Commit change: ' + owner if pinfo['precursors'] != []: print '### Precursors:', ps = [] for p in pinfo['precursors']: ps.append(short_id(co, p)) print ', '.join(ps) if pinfo.has_key('user'): print '### User: ' + pinfo['user'] if time is not None: print '### Date: ' + ctime(time) elif pinfo.has_key('time'): print '### Date: ' + ctime(pinfo['time']) if pinfo.has_key('comment'): print '### Comment' print pinfo['comment'].rstrip() plist = [] for handle, value in pinfo['handles'].items(): if handle == roothandle: continue plist.append((fullpath_at_point(co, handle, point), _letters(value))) if len(plist): print '### Files' plist.sort() olist = [] for value in plist: olist.append(value[1][0] + value[1][1] + '\t' + value[0]) print os.linesep.join(olist) return def _history_increment(co, handles, precursors, changes): for handle in handles: for pre in precursors: change = handle_last_modified(co, co.contents, handle, pre, None) if change is not None: changes.setdefault(change, {})[handle] = 1 return def _history_deps(node, args): co, cutoffs = args[0], args[1] if len(cutoffs) and _is_ancestor(co, node, cutoffs, None): return [] cset = bdecode(co.lcrepo.get(node)) return cset['precursors'] def _owner_deps(node, args): co, limit = args[0], args[1] # heuristic shortcut limit -= 1 if limit == 0: return [] args[1] = limit cset = bdecode(co.lcrepo.get(node)) try: return [cset['precursors'][0]] except IndexError: pass return [] def history(co, head, limit, skip, v, by_id, files): repohead = None heads = None if head is not None: heads = [head] else: heads = bdecode(co.linforepo.get('heads')) repohead = repo_head(co, co.repo) if repohead is None: if co.repo is not None: repohead = rootnode heads.insert(0, repohead) else: heads.insert(0, repohead) head = heads[0] # the repository head may have more than we have merged locally if repohead is not None: while not _is_ancestor(co, head, heads, None): head = bdecode(co.lcrepo.get(head))['precursors'][0] # make an initial list of points to print based on user-specified files owner_cutoff = limit + skip changes = None if files != []: changes = {} if by_id: handles = [binascii.unhexlify(handle) for handle in files] else: handles = [handle for handle, expanded in Glob(co, files).db_walk()] _history_increment(co, handles, heads, changes) # the heuristic breaks if we're not printing everything owner_cutoff = -1 # get a list of the clean merge heads for this repository dfs = DFS(_owner_deps, [co, owner_cutoff + 1]) dfs.search(head) owners = dfs.result() # sort all the history points in reverse print order cutoffs = [] if owners[0] != rootnode: cutoffs.append(owners[0]) dfs = DFS(_history_deps, [co, cutoffs]) dfs.search(rootnode) for head in heads: dfs.search(head) ordering = dfs.result() ordering.reverse() # pop off the root node assert ordering[-1] == rootnode ordering.pop() owner = 'local' time = None for point in ordering: hinfo = bdecode(co.lcrepo.get(point)) clean = False if clean_merge_point(hinfo): clean = True if point == owners[-1]: # committed change, but we don't know the server merge change owners.pop() time = hinfo['time'] owner = '----' if clean: # this is the server merge change owner = short_id(co, point) assert changes is None or not changes.has_key(point) continue # only display if it's not a clean merge and it was asked for if clean: continue if changes is not None and not changes.has_key(point): continue # figure out the next set of changes to print if changes is not None and changes.has_key(point): _history_increment(co, changes[point].keys(), hinfo['precursors'], changes) del changes[point] if skip > 0: skip -= 1 continue if limit == 0: break limit -=1 _print_change(co, point, hinfo, v, owner=owner, time=time) if v: print '-' * 78, print return 0 def revert(co, files, unmod_flag): co.handle_name_cache = {} txn = co.txn_begin() mark_modified_files(co, txn) heads = bdecode(co.linforepo.get('heads')) editsdb = co.editsdb #modified, names, deletes, newhandles = [], {}, {}, {} modified = [] for handle, expanded in Glob(co, files).db_walk(deletes=1): filename = handle_to_filename(co, handle, txn) filepath = path.join(co.local, filename) editted = False exists = path.exists(filepath) if editsdb.has_key(handle, txn): info = bdecode(editsdb.get(handle, txn=txn)) if info.has_key('add') or \ info.has_key('name') or \ info.has_key('delete'): print 'warning - cannot revert name operation on %s' % (filename,) # XXX: hack until reverts on name ops work if info.has_key('add'): exists = True elif info.has_key('hash'): editted = True elif exists: if not expanded: file = handle_to_filename(co, handle, txn) print 'warning - %s is not opened for edit' % (filename,) continue sinfo = bdecode(co.staticdb.get(handle, txn=txn)) if sinfo['type'] == 'file' and (editted or not exists): file_points = [] for point in heads: linfo = handle_contents_at_point(co, handle, point, None) if linfo is None: continue file_points.append((linfo['lines'], linfo['line points'], linfo['points'])) # XXX: hack until merge-through-conflict code is done if len(file_points) == 2: local = file_points[0] remote = file_points[1] lines = find_conflict(local[0], local[1], local[2], remote[0], remote[1], remote[2]) else: lines = find_conflict_multiple_safe(file_points)[0] if lines is None: print 'error - cannot revert %s' % (filename,) co.txn_abort(txn) return 1 #modified.append((handle, linfo)) ls, conflict = [], 0 for l in lines: if type(l) is str: ls.append(l) else: conflict = 1 ls.append('<<<<<<< local') ls.extend(l[0]) ls.append('=======') ls.extend(l[1]) ls.append('>>>>>>> remote') if unmod_flag: if not exists: continue h = open(filepath, 'rb') contents = h.read() h.close() if '\n'.join(ls) == contents: unset_edit(co, handle, ['hash'], txn) mtime = int(path.getmtime(filepath)) co.modtimesdb.put(handle, bencode(mtime), txn=txn) print 'reverting: %s' % (filename,) continue if not conflict and editted: unset_edit(co, handle, ['hash'], txn) hfile = path.join(co.temppath, binascii.hexlify(handle)) h = open(hfile, 'wb') h.write('\n'.join(ls)) h.close() modified.append((handle, filepath)) print 'reverting: %s' % (filename,) # XXX: use update code for handle, filename in modified: temppath = path.join(co.temppath, binascii.hexlify(handle)) destpath = path.join(co.local, filename) preserving_rename(temppath, destpath) mtime = int(path.getmtime(destpath)) co.modtimesdb.put(handle, bencode(mtime), txn=txn) co.txn_commit(txn) print 'revert succeeded' return 0 def annotate(co, rev, files): """Print each line of files with information on the last modification""" if rev == 'repo': rev = repo_head(co, co.repo) elif rev != 'local': rev = long_id(co, rev) if len(files) == 0: files = ['...'] if rev == 'local': precursors = bdecode(co.linforepo.get('heads')) repohead = repo_head(co, co.repo) if repohead is None: repohead = rootnode if repohead not in precursors: while not _is_ancestor(co, repohead, precursors, None): info = bdecode(co.lcrepo.get(repohead)) try: repohead = info['precursors'][0] except IndexError: repohead = rootnode if repohead not in precursors: precursors.insert(0, repohead) cache = {} for handle, expanded in Glob(co, files).db_walk(deletes=1): sinfo = bdecode(co.staticdb.get(handle)) if sinfo['type'] == 'file': if rev == 'local': filename = handle_to_filename(co, handle) file_points = [] pres = simplify_precursors(co, handle, co.contents, precursors, None)[0] for pre, index in pres: info = handle_contents_at_point(co, handle, pre, None, replayfunc=merge.annotate) file_points.append((info['lines'], info['line points'], info['points'])) lfile = path.join(co.local, filename) try: h = open(lfile, 'rb') except IOError: print 'error - cannot open %s' % (filename,) return 1 lines = h.read().split('\n') h.close() lpoints = find_annotation(file_points, lines) else: filename = fullpath_at_point (co, handle, rev) cinfo = handle_contents_at_point(co, handle, rev, None, replayfunc=merge.annotate) if cinfo is None or cinfo.has_key('delete'): if expanded: continue print 'error - cannot find %s' % (filename,) return 1 lines = cinfo['lines'] lpoints = cinfo['line points'] print >> stderr, 'Annotations for %s' % (filename,) print >> stderr, '***************' for i in xrange(len(lines)): point = lpoints[i] if point is None: lrev = 'local' user = '-----' time = '-----' elif cache.has_key(point): lrev, user, time = cache[point] else: lrev = short_id(co, point) pinfo = bdecode(co.lcrepo.get(point)) user = pinfo['user'] if pinfo.has_key('time'): time = strftime('%Y-%m-%d', localtime(pinfo['time'])) else: time = '-----' cache[point] = (lrev, user, time) print '%s (%s %s):' % (lrev, user, time,), lines[i] return 0 class PathError(Exception): pass def find_co(local, metadata_dir='.cdv'): if not path.exists(local): raise Exception, 'path ' + local + ' does not exist' while not path.exists(path.join(local, metadata_dir)): parent = path.split(local)[0] if local == parent: raise PathError, 'cannot find checkout, use "cdv init" to create one' local = parent return local #try: # import psyco # psyco.bind(diff, 0) #except ImportError: # pass # Everything below here is for testing def file_contents(file): h = open(file, 'rb') contents = h.read() h.close() return contents def set_file_contents(file, contents): h = open(file, 'wb') h.write(contents) h.close() def append_file_contents(file, contents): h = open(file, 'ab') h.write(contents) h.close() def rm_rf(files): for local in files: mode = os.lstat(local).st_mode if stat.S_ISDIR(mode): for lamb in os.listdir(local): rm_rf([path.join(local, lamb)]) os.rmdir(local) elif stat.S_ISREG(mode): os.unlink(local) sh = None sht = None ag = None agt = None def init_test(local, remote, remote2): global sh, sht global ag, agt if path.exists(local): rm_rf([local]) os.makedirs(path.join(local, 'repo')) set_file_contents(path.join(local, 'repo', 'codeville_repository'), '') from passwd import Passwd pw = Passwd(path.join(local, 'repo', 'passwd'), create=1) pw.add('unittest', '') pw.add('unittest2', '') from ConfigParser import ConfigParser sconfig = ConfigParser() sconfig.add_section('control') sconfig.set('control', 'backup', 'False') sconfig.set('control', 'datadir', path.join(local, 'repo')) sconfig.add_section('post-commit') sh = ServerHandler(sconfig) sh.bind(remote[1]) sh.db_init(init=True) sht = Thread(target = sh.listen_forever, args = []) sht.start() from agent import Agent ag = Agent() auth_path = tempfile.mkdtemp('', 'cdv-') auth_file = path.join(auth_path, 'agent.test') ag.listen_sock(auth_path, auth_file) agt = Thread(target = ag.listen, args = []) agt.start() os.makedirs(path.join(local, 'co')) co = Checkout(path.join(local, 'co'), init=True) txn = co.txn_begin() co.varsdb.put('user', 'unittest', txn=txn) co.txn_commit(txn) os.makedirs(path.join(local, 'co2')) co2 = Checkout(path.join(local, 'co2'), init=True) txn = co2.txn_begin() co2.varsdb.put('user', 'unittest2', txn=txn) co2.txn_commit(txn) co.nopass = co2.nopass = 1 co.repo = co2.repo = tuple_to_server(remote) create_repo(co, remote) create_repo(co, remote2) return co, co2 def shutdown_test(local, cos): if sh is not None: sh.rs.doneflag.set() sh.rs.start_connection(('localhost', 6602)) sh.shutdown.wait() sh.close() from errno import ECONNRESET import socket if ag is not None: ag.shutdown_flag = 1 sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.connect(ag.auth_file) try: sock.recv(1) except socket.error, reason: assert reason[0] == ECONNRESET else: sock.close() for co in cos: co.close() def reset_co(co): txn = co.txn_begin() co.linforepo.put('heads', bencode([rootnode]), txn=txn) co.editsdb.truncate(txn=txn) co.modtimesdb.truncate(txn=txn) co.filenamesdb.truncate(txn=txn) co.txn_commit(txn) co.handle_name_cache = {} for lamb in os.listdir(co.local): if lamb == '.cdv': continue rm_rf([path.join(co.local, lamb)]) def reset_test(co, co2, remote, remote2=None): remove_repo(co, remote) create_repo(co, remote) if remote2: remove_repo(co, remote2) create_repo(co, remote2) reset_co(co) reset_co(co2) def test_client(): global ServerHandler, Thread from server import ServerHandler from threading import Thread local = path.abspath('test') cop = path.join(local, 'co') co2p = path.join(local, 'co2') repo = server_to_tuple('cdv://localhost:6602/unittest') repo2 = server_to_tuple('cdv://localhost:6602/unittest2') co, co2 = init_test(local, repo, repo2) try: _test_client(co, cop, co2, co2p, repo, repo2) except (AssertionError, Exception): shutdown_test(local, [co, co2]) raise shutdown_test(local, [co, co2]) if path.exists(local): rm_rf([local]) return def _test_client(co, cop, co2, co2p, repo, repo2): print 'TESTING merge conflict' set_file_contents(path.join(cop, 'a'), "aaa\nbbb\nccc\nddd\neee\nfff\n") add(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 set_file_contents(path.join(cop, 'a'), "aaa\nbbb\nccc\nfoo\nddd\neee\nfff\n") os.utime(path.join(cop, 'a'), (0, 0)) assert commit(co, repo, '') == 0 set_file_contents(path.join(co2p, 'a'), "aaa\nbbb\nccc\nbar\nddd\neee\nfff\n") os.utime(path.join(co2p, 'a'), (0, 0)) assert commit(co2, repo, '') == 1 assert update(co2, repo) == 0 assert file_contents(path.join(co2p, 'a')) == "aaa\nbbb\nccc\n<<<<<<< local\nbar\n=======\nfoo\n>>>>>>> remote\nddd\neee\nfff\n" print 'TESTING add conflict' reset_test(co, co2, repo) set_file_contents(path.join(cop, 'a'), 'foo') add(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 set_file_contents(path.join(co2p, 'a'), 'bar') add(co2, [path.join(co2p, 'a')]) assert commit(co2, repo, '') == 1 assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'a.nameconflict.local')) assert path.exists(path.join(co2p, 'a.nameconflict.remote')) # use the agent for the rest of the tests os.environ['CDV_AUTH_SOCK'] = ag.auth_file co.nopass = co2.nopass = 2 print 'TESTING rename and add file of same name' reset_test(co, co2, repo) set_file_contents(path.join(cop, 'a'), '') add(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co2, path.join(co2p, 'a'), path.join(co2p, 'b')) set_file_contents(path.join(co2p, 'a'), '') add(co2, [path.join(co2p, 'a')]) assert commit(co2, repo, '') == 0 assert update(co, repo) == 0 assert path.exists(path.join(cop, 'a')) assert path.exists(path.join(cop, 'b')) print 'TESTING add file conflicting with remote rename' reset_test(co, co2, repo) set_file_contents(path.join(cop, 'a'), '') add(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'a'), path.join(cop, 'b')) assert commit(co, repo, '') == 0 set_file_contents(path.join(co2p, 'b'), '') add(co2, [path.join(co2p, 'b')]) assert commit(co2, repo, '') == 1 assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'b.nameconflict.local')) assert path.exists(path.join(co2p, 'b.nameconflict.remote')) print 'TESTING add file conflicting with remote rename and merge conflict' reset_test(co, co2, repo) set_file_contents(path.join(cop, 'a'), "foo\n") add(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 append_file_contents(path.join(cop, 'a'), "bar\n") edit(co, [path.join(cop, 'a')]) rename(co, path.join(cop, 'a'), path.join(cop, 'b')) assert commit(co, repo, '') == 0 set_file_contents(path.join(co2p, 'b'), '') add(co2, [path.join(co2p, 'b')]) append_file_contents(path.join(co2p, 'a'), "baz\n") edit(co2, [path.join(co2p, 'a')]) assert commit(co2, repo, '') == 1 assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'b.nameconflict.local')) assert path.exists(path.join(co2p, 'b.nameconflict.remote')) assert file_contents(path.join(co2p, 'b.nameconflict.remote')) == '<<<<<<< local\nfoo\nbaz\n\n=======\nfoo\nbar\n\n>>>>>>> remote' print 'TESTING conflicting local and remote rename' reset_test(co, co2, repo) set_file_contents(path.join(cop, 'a'), '') add(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'a'), path.join(cop, 'b')) assert commit(co, repo, '') == 0 rename(co2, path.join(co2p, 'a'), path.join(co2p, 'c')) assert commit(co2, repo, '') == 1 assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'c.nameconflict')) assert path.exists(path.join(co2p, 'c.nameconflict.info')) print 'TESTING multiple conflicting local and remote rename' reset_test(co, co2, repo) os.makedirs(path.join(cop, 'a')) set_file_contents(path.join(cop, 'a', 'x'), '') os.makedirs(path.join(cop, 'b')) set_file_contents(path.join(cop, 'b', 'y'), '') os.makedirs(path.join(cop, 'c')) set_file_contents(path.join(cop, 'c', 'z'), '') add(co, [path.join(cop, 'a', 'x'), path.join(cop, 'b', 'y'), path.join(cop, 'c', 'z')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'b', 'y'), path.join(cop, 'a', 'y')) rename(co, path.join(cop, 'c', 'z'), path.join(cop, 'a', 'z')) assert commit(co, repo, '') == 0 rename(co2, path.join(co2p, 'b', 'y'), path.join(co2p, 'b', 'x')) rename(co2, path.join(co2p, 'c', 'z'), path.join(co2p, 'c', 'x')) assert commit(co2, repo, '') == 1 assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'b', 'x.nameconflict')) assert path.exists(path.join(co2p, 'b', 'x.nameconflict.info')) assert path.exists(path.join(co2p, 'c', 'x.nameconflict')) assert path.exists(path.join(co2p, 'c', 'x.nameconflict.info')) print 'TESTING rename and back again' reset_test(co, co2, repo) set_file_contents(path.join(cop, 'a'), 'a') add(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'a'), path.join(cop, 'b')) assert commit(co, repo, '') == 0 rename(co, path.join(cop, 'b'), path.join(cop, 'a')) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'a')) print 'TESTING rename swap' reset_test(co, co2, repo) set_file_contents(path.join(cop, 'a'), 'a') set_file_contents(path.join(cop, 'b'), 'b') add(co, [path.join(cop, 'a'), path.join(cop, 'b')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'a'), path.join(cop, 'c')) rename(co, path.join(cop, 'b'), path.join(cop, 'a')) rename(co, path.join(cop, 'c'), path.join(cop, 'b')) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 assert file_contents(path.join(co2p, 'a')) == 'b' assert file_contents(path.join(co2p, 'b')) == 'a' print 'TESTING rename circular' reset_test(co, co2, repo) set_file_contents(path.join(cop, 'a'), 'a') set_file_contents(path.join(cop, 'b'), 'b') set_file_contents(path.join(cop, 'c'), 'c') add(co, [path.join(cop, 'a'), path.join(cop, 'b'), path.join(cop, 'c')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'a'), path.join(cop, 'd')) rename(co, path.join(cop, 'b'), path.join(cop, 'a')) rename(co, path.join(cop, 'c'), path.join(cop, 'b')) rename(co, path.join(cop, 'd'), path.join(cop, 'c')) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 assert file_contents(path.join(co2p, 'a')) == 'b' assert file_contents(path.join(co2p, 'b')) == 'c' assert file_contents(path.join(co2p, 'c')) == 'a' print 'TESTING clean reparent with loops in intermediate rename stages' reset_test(co, co2, repo) os.makedirs(path.join(cop, 'a', 'b', 'c', 'd')) add(co, [path.join(cop, 'a', 'b', 'c', 'd')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'a', 'b', 'c', 'd'), path.join(cop, 'd')) rename(co, path.join(cop, 'a', 'b', 'c'), path.join(cop, 'd', 'c')) rename(co, path.join(cop, 'a', 'b'), path.join(cop, 'd', 'c', 'b')) rename(co, path.join(cop, 'a'), path.join(cop, 'd', 'c', 'b', 'a')) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'd', 'c', 'b', 'a')) print 'TESTING reparent twisted conflict' reset_test(co, co2, repo) os.makedirs(path.join(cop, 'a')) os.makedirs(path.join(cop, 'b')) add(co, [path.join(cop, 'a'), path.join(cop, 'b')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'a'), path.join(cop, 'b', 'a')) assert commit(co, repo, '') == 0 rename(co2, path.join(co2p, 'b'), path.join(co2p, 'a', 'b')) assert commit(co2, repo, '') == 1 assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'a.parentloop', 'b')) assert path.exists(path.join(co2p, 'a.parentloop.info')) print 'TESTING reparent twisted conflict' reset_test(co, co2, repo) os.makedirs(path.join(cop, 'c', 'a')) os.makedirs(path.join(cop, 'b')) add(co, [path.join(cop, 'c', 'a'), path.join(cop, 'b')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'c', 'a'), path.join(cop, 'b', 'a')) rename(co, path.join(cop, 'c'), path.join(cop, 'b', 'a', 'c')) assert commit(co, repo, '') == 0 rename(co2, path.join(co2p, 'b'), path.join(co2p, 'c', 'a', 'b')) assert commit(co2, repo, '') == 1 assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'c.parentloop', 'a.parentloop', 'b')) assert path.exists(path.join(co2p, 'c.parentloop.info')) assert path.exists(path.join(co2p, 'c.parentloop', 'a.parentloop.info')) print 'TESTING rename incidental' reset_test(co, co2, repo) os.makedirs(path.join(cop, 'a')) add(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'a'), path.join(cop, 'b')) assert commit(co, repo, '') == 0 rename(co2, path.join(co2p, 'a'), path.join(co2p, 'b')) assert commit(co2, repo, '') == 1 assert update(co2, repo) == 0 assert commit(co2, repo, '') == 0 rename(co, path.join(cop, 'b'), path.join(cop, 'c')) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'c')) print 'TESTING rename dependent' reset_test(co, co2, repo) os.makedirs(path.join(cop, 'a', 'a', 'a')) set_file_contents(path.join(cop, 'a', 'a', 'a', 'a'), '') add(co, [path.join(cop, 'a', 'a', 'a', 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'a', 'a', 'a', 'a'), path.join(cop, 'a', 'a', 'a', 'b')) rename(co, path.join(cop, 'a', 'a', 'a'), path.join(cop, 'a', 'a', 'b')) rename(co, path.join(cop, 'a', 'a'), path.join(cop, 'a', 'b')) rename(co, path.join(cop, 'a'), path.join(cop, 'b')) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'b', 'b', 'b', 'b')) print 'TESTING update overrides coincidental name merge' reset_test(co, co2, repo) set_file_contents(path.join(cop, 'a'), '') add(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'a'), path.join(cop, 'b')) assert commit(co, repo, '') == 0 rename(co2, path.join(co2p, 'a'), path.join(co2p, 'b')) assert commit(co2, repo2, '') == 0 assert update(co, repo2) == 0 assert update(co2, repo) == 0 assert commit(co2, repo, '') == 0 assert update(co, repo) == 0 rename(co, path.join(cop, 'b'), path.join(cop, 'c')) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'c')) print 'TESTING delete orphan' reset_test(co, co2, repo) os.makedirs(path.join(cop, 'a')) add(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 set_file_contents(path.join(cop, 'a', 'a'), '') add(co, [path.join(cop, 'a', 'a')]) assert commit(co, repo, '') == 0 delete(co2, [path.join(co2p, 'a')]) assert commit(co2, repo, '') == 1 assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'a.orphaned')) assert path.exists(path.join(co2p, 'a.orphaned.info')) print 'TESTING remote orphan' reset_test(co, co2, repo) os.makedirs(path.join(cop, 'a')) add(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 delete(co2, [path.join(co2p, 'a')]) assert commit(co2, repo, '') == 0 set_file_contents(path.join(cop, 'a', 'a'), '') add(co, [path.join(cop, 'a', 'a')]) assert commit(co, repo, '') == 1 assert update(co, repo) == 0 assert path.exists(path.join(cop, 'a.orphaned')) assert path.exists(path.join(cop, 'a.orphaned.info')) print 'TESTING delete and reuse name' reset_test(co, co2, repo) os.makedirs(path.join(cop, 'a', 'a')) os.makedirs(path.join(cop, 'b')) add(co, [path.join(cop, 'a', 'a'), path.join(cop, 'b')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 delete(co, [path.join(cop, 'b')]) rename(co, path.join(cop, 'a'), path.join(cop, 'b')) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'b', 'a')) assert not path.exists(path.join(co2p, 'a')) print 'TESTING delete dependent' reset_test(co, co2, repo) os.makedirs(path.join(cop, 'a', 'a', 'a', 'a')) add(co, [path.join(cop, 'a', 'a', 'a', 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 delete(co, [path.join(cop, 'a', 'a', 'a', 'a')]) delete(co, [path.join(cop, 'a', 'a', 'a')]) delete(co, [path.join(cop, 'a', 'a')]) delete(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 assert not path.exists(path.join(co2p, 'a')) print 'TESTING delete within parent loop' reset_test(co, co2, repo) os.makedirs(path.join(cop, 'a')) os.makedirs(path.join(cop, 'b')) add(co, [path.join(cop, 'a'), path.join(cop, 'b')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'a'), path.join(cop, 'b', 'a')) assert commit(co, None, '') == 0 assert update(co, repo) == 0 delete(co, [path.join(cop, 'b', 'a')]) assert commit(co, repo, '') == 0 rename(co2, path.join(co2p, 'b'), path.join(co2p, 'a', 'b')) assert commit(co2, repo, '') == 1 assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'b.orphaned')) assert path.exists(path.join(co2p, 'b.orphaned.info')) print 'TESTING delete entire parent loop' reset_test(co, co2, repo) os.makedirs(path.join(cop, 'a')) os.makedirs(path.join(cop, 'b')) add(co, [path.join(cop, 'a'), path.join(cop, 'b')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'a'), path.join(cop, 'b', 'a')) assert commit(co, None, '') == 0 delete(co, [path.join(cop, 'b', 'a')]) assert commit(co, repo, '') == 0 rename(co2, path.join(co2p, 'b'), path.join(co2p, 'a', 'b')) assert commit(co2, None, '') == 0 delete(co2, [path.join(co2p, 'a', 'b')]) assert commit(co2, repo, '') == 0 assert update(co, repo) == 0 assert update(co2, repo) == 0 assert not path.exists(path.join(cop, 'a')) assert not path.exists(path.join(cop, 'b')) assert not path.exists(path.join(co2p, 'a')) assert not path.exists(path.join(co2p, 'b')) print 'TESTING remote add and delete' reset_test(co, co2, repo) set_file_contents(path.join(cop, 'a'), '') add(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 delete(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 assert not path.exists(path.join(co2p, 'a')) print 'TESTING unique name mangling' reset_test(co, co2, repo) set_file_contents(path.join(cop, 'a'), '') add(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'a'), path.join(cop, 'a1')) assert commit(co, repo, '') == 0 set_file_contents(path.join(co2p, 'a2.nameconflict'), '') add(co2, [path.join(co2p, 'a2.nameconflict')]) rename(co2, path.join(co2p, 'a'), path.join(co2p, 'a2')) assert update(co2, repo) == 0 assert path.exists(path.join(co2p, 'a2.nameconflict')) assert path.exists(path.join(co2p, 'a2.nameconflict2')) assert path.exists(path.join(co2p, 'a2.nameconflict2.info')) # XXX: had to relax this restriction for now, fix next history rewrite #print 'TESTING deleted file modified remotely' #reset_test(co, co2, repo) #set_file_contents(path.join(cop, 'a'), '') #add(co, [path.join(cop, 'a')]) #assert commit(co, repo, '') == 0 #assert update(co2, repo) == 0 #delete(co, [path.join(cop, 'a')]) #assert commit(co, repo, '') == 0 #set_file_contents(path.join(co2p, 'a'), 'foo') #edit(co2, [path.join(co2p, 'a')]) #assert commit(co2, repo, '') == 1 #assert update(co2, repo) == 0 #assert commit(co2, repo, '') == 0 #assert update(co, repo) == 0 #assert not path.exists(path.join(cop, 'a')) print 'TESTING independent local and remote deletes' set_file_contents(path.join(cop, 'a'), '') add(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 delete(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 delete(co2, [path.join(co2p, 'a')]) assert commit(co2, repo, '') == 0 assert update(co2, repo) == 0 assert not path.exists(path.join(cop, 'a')) assert not path.exists(path.join(co2p, 'a')) print 'TESTING local filesystem conflicts with repository' reset_test(co, co2, repo) os.makedirs(path.join(cop, 'a')) add(co, [path.join(cop, 'a')]) assert commit(co, repo, '') == 0 set_file_contents(path.join(co2p, 'a'), '') assert update(co2, repo) == 1 assert path.isfile(path.join(co2p, 'a')) print 'TESTING non-existent parents needed for update' reset_test(co, co2, repo) os.makedirs(path.join(cop, 'a')) os.makedirs(path.join(cop, 'b')) add(co, [path.join(cop, 'a',), path.join(cop, 'b',)]) assert commit(co, repo, '') == 0 assert update(co2, repo) == 0 rename(co, path.join(cop, 'a'), path.join(cop, 'b', 'a')) assert commit(co, repo, '') == 0 os.rmdir(path.join(co2p, 'a')) assert update(co2, repo) == 1 os.rmdir(path.join(co2p, 'b')) assert update(co2, repo) == 1 set_file_contents(path.join(co2p, 'b'), '') assert update(co2, repo) == 1 assert not path.exists(path.join(co2p, 'a')) assert path.isfile(path.join(co2p, 'b')) Codeville-0.8.0/Codeville/client_helpers.py0000664000076400007640000005106010631557377020255 0ustar rcohenrcohen# Written by Ross Cohen # see LICENSE.txt for license information from bencode import bdecode, bencode from db import db from history import roothandle, dmerge, rename_conflict_check from history import handle_name_at_point, __handle_name_at_point from history import _handle_name_at_point from history import handle_contents_at_point from history import handle_last_modified from history import _name_use_count, _children_count from history import write_diff, write_index, db_get, db_put, HistoryError from history import clean_merge_point, simplify_precursors, repo_head from history import _is_ancestor from merge import find_resolution import os from os import path from path import breakup import sha import stat from time import time import zlib class CommitError(Exception): pass def new_handle(co, txn): """Create a temporary handle for new files in the working copy""" num = bdecode(co.linforepo.get('lasthandle', txn=txn)) + 1 handle = sha.new(str(num)).digest() co.linforepo.put('lasthandle', bencode(num), txn=txn) return handle def create_handle(precursors, hinfo): """Create a permanent identifier for use in a changeset description""" binfo = bencode({'precursors': precursors, 'handle': hinfo}) return sha.new(binfo).digest() def handle_name(co, handle, txn): """Returns a dict specifying name, parent and other info""" if co.handle_name_cache.has_key(handle): return co.handle_name_cache[handle] # XXX: reading this repeatedly is excessive heads = bdecode(co.linforepo.get('heads', txn=txn)) einfo = {} if co.editsdb.has_key(handle, txn): einfo = bdecode(co.editsdb.get(handle, txn=txn)) if einfo.has_key('name'): einfo['points'] = ['1'] for head in heads: pinfo = __handle_name_at_point(co, handle, head, txn) if pinfo is None: continue einfo['points'] = dmerge(einfo['points'], pinfo['points']) einfo['rename point'] = ['1'] co.handle_name_cache[handle] = einfo return einfo state = None for point in heads: pinfo = __handle_name_at_point(co, handle, point, txn) if pinfo is None: continue if pinfo.has_key('delete'): co.handle_name_cache[handle] = pinfo return pinfo if state is None: state = pinfo continue conflict, rename_points = rename_conflict_check(state, pinfo) if conflict == 'remote': state['name'] = pinfo['name'] state['parent'] = pinfo['parent'] state['rename point'] = rename_points state['points'] = dmerge(state['points'], pinfo['points']) if einfo.has_key('delete'): state['delete'] = einfo['delete'] co.handle_name_cache[handle] = state return state def _filename_to_handle(co, parent, name, txn, deletes=0): """Given a parent handle and a name, return all associated handles""" handles = [] cursor = co.allnamesdb.cursor(txn=txn) lookup = parent + name try: key, value = cursor.set(lookup) except (db.DBNotFoundError, TypeError): key = None while key == lookup: hinfo = handle_name(co, value, txn) if hinfo is not None and hinfo['name'] == name and hinfo['parent'] == parent: if not hinfo.has_key('delete'): handles.append(value) elif deletes == 1 and co.editsdb.has_key(value): if bdecode(co.editsdb.get(value, txn=txn)).has_key('delete'): handles.append(value) # next_dup() is broken foo = cursor.next() if foo is None: break key, value = foo cursor.close() return handles def filename_to_handle(co, file, txn=None, deletes=0): """Given a full file path, return the associated handle""" if file == '': return roothandle lpath = breakup(file) handle = roothandle for name in lpath: handles = _filename_to_handle(co, handle, name, txn, deletes=deletes) if handles == []: return None handle = handles[0] return handle def _handle_to_filename(co, handle, names, txn): """Same as handle_to_filename, but takes an override mapping for handles""" lpath, hseen = [], {} while handle != roothandle: if names.has_key(handle): info = names[handle] else: info = handle_name(co, handle, txn) if info is None: lpath.insert(0, '???') break if hseen.has_key(handle): lpath.insert(0, info['name'] + '(loop)') break lpath.insert(0, info['name']) hseen[handle] = 1 handle = info['parent'] try: return path.join(*lpath) except TypeError: pass return '' def handle_to_filename(co, handle, txn=None): """Convert the given handle to a full path name.""" return _handle_to_filename(co, handle, {}, txn) def set_edit(co, handle, info, txn): """Specify that a new attribute on a file is being edited.""" if co.editsdb.has_key(handle): oinfo = bdecode(co.editsdb.get(handle, txn=txn)) oinfo.update(info) info = oinfo co.editsdb.put(handle, bencode(info), txn=txn) def unset_edit(co, handle, info, txn): """Remove an edit attribute on a file.""" oinfo = bdecode(co.editsdb.get(handle, txn=txn)) for attr in info: del oinfo[attr] if oinfo == {}: co.editsdb.delete(handle, txn=txn) else: co.editsdb.put(handle, bencode(oinfo), txn=txn) def _add_file(co, rep, parent, required, ltxn): mode = os.lstat(path.join(co.local, rep)).st_mode handle = filename_to_handle(co, rep) if handle: #info = bdecode(co.staticdb.get(handle)) info = db_get(co, co.staticdb, handle, None) if info['type'] == 'dir' and not stat.S_ISDIR(mode): print 'error - %s already added as a %s' % (rep, info['type']) return None if info['type'] == 'file' and not stat.S_ISREG(mode): print 'error - %s already added as a %s' % (rep, info['type']) return None if required: print 'warning - %s already added' % (rep,) else: print 'adding: ' + rep if stat.S_ISDIR(mode): type = 'dir' elif stat.S_ISREG(mode): type = 'file' else: print 'error - unrecognized file type for %s' % (rep,) return None if rep == '': handle = roothandle else: handle = new_handle(co, ltxn) #co.staticdb.put(handle, bencode({'type': type}), txn=ltxn) db_put(co, co.staticdb, handle, {'type': type}, ltxn) info = {'name': path.split(rep)[1], 'parent': parent, 'add': {}} set_edit(co, handle, info, ltxn) co.allnamesdb.put(parent + info['name'], handle, flags=db.DB_NODUPDATA, txn=ltxn) if type == 'file': co.modtimesdb.put(handle, bencode(0), txn=ltxn) co.filenamesdb.put(rep, handle, txn=ltxn) return handle def _set_name(co, handle, parent, name, txn): linfo = {'parent': parent, 'name': name} co.handle_name_cache[handle] = linfo set_edit(co, handle, linfo, txn) try: co.allnamesdb.put(parent + name, handle, flags=db.DB_NODUPDATA, txn=txn) except db.DBKeyExistError: pass def name_use_count(co, handle, txn): """Returns a list of handles with the same name as the given handle.""" def _name_func(co, handle, foo, txn): return handle_name(co, handle, txn) return _name_use_count(co, handle_name(co, handle, txn), None, _name_func, txn) def rename_race(co, handle, names, txn): #def _name_func(co, handle, names, txn): # if names.has_key(handle): # return names[handle] # return None #named = _name_use_count(co, info, names, _name_func, txn) info = handle_name(co, handle, txn) cursor = co.allnamesdb.cursor(txn=txn) lookup = info['parent'] + info['name'] try: key, value = cursor.set(lookup) except (db.DBNotFoundError, TypeError): return None while key == lookup: if names.has_key(value): vinfo = names[value] if vinfo['parent'] == info['parent'] and vinfo['name'] == info['name']: cursor.close() return value foo = cursor.next() if foo is None: break key, value = foo cursor.close() return None def children_count(co, handle, txn, deletes=0): """Returns a list of children of the specified handle.""" def _name_func(co, handle, foo, txn): hinfo = handle_name(co, handle, txn) if hinfo is not None and hinfo.has_key('delete') and deletes == 1: # if the editsdb has it, then it was just deleted if co.editsdb.has_key(handle, txn): del hinfo['delete'] return hinfo return _children_count(co, handle, None, _name_func, txn) def _parent_loop_check(co, handle, names, txn): """Same as parent_loop_check, but takes a dict of handle mapping overrides.""" hseen = {} while handle != roothandle: if hseen.has_key(handle): return handle hseen[handle] = 1 if names.has_key(handle): pinfo = names[handle] else: pinfo = handle_name(co, handle, txn) if pinfo.has_key('delete'): return handle handle = pinfo['parent'] return None def parent_loop_check(co, handle, txn): """Check whether this handle exists under itself in the directory tree.""" return _parent_loop_check(co, handle, {}, txn) def _rename_safe_check(co, handle, names, txn): temp = names[handle] del names[handle] result = _parent_loop_check(co, handle, names, txn) names[handle] = temp if result is None: return 1 return None def unique_name(co, parent, file, txn): """Generate a name not already in use by the system.""" if _filename_to_handle(co, parent, file, txn) == []: return file post = 2 while _filename_to_handle(co, parent, file + str(post), txn) != []: post = post + 1 return file + str(post) def conflicts_in_file(co, file): h = open(path.join(co.local, file), 'rb') lines = h.read().split('\n') h.close() for l in lines: if l == '<<<<<<< local' or \ l == '=======' or \ l == '>>>>>>> remote': return 1 return 0 def find_update_files(co, rhead, named, txn): rnamed = [] for handle in named: linfo = handle_name(co, handle, txn) rnamed.append((handle, linfo, __handle_name_at_point(co, handle, rhead, txn))) return rnamed def mark_modified_files(co, txn): if co.varsdb.has_key('edit-mode'): if co.varsdb.get('edit-mode') == '1': return modtimesdb, editsdb, local = co.modtimesdb, co.editsdb, co.local for lfile, handle in co.filenamesdb.items(txn): if editsdb.has_key(handle, txn): info = bdecode(editsdb.get(handle, txn=txn)) if info.has_key('hash') or info.has_key('add'): continue #info = bdecode(co.staticdb.get(handle, txn=txn)) info = db_get(co, co.staticdb, handle, txn) if info['type'] != 'file': continue lfile = path.join(local, lfile) try: mtime = int(path.getmtime(lfile)) except OSError: continue minfo = bdecode(modtimesdb.get(handle, txn=txn)) if mtime != minfo: modtimesdb.put(handle, bencode(mtime), txn=txn) set_edit(co, handle, {'hash': 1}, txn) def gen_diff(co, handle, precursors, lines, txn): pres, plen = simplify_precursors(co, handle, co.contents, precursors, txn) file_points = [] for pre, index in pres: info = handle_contents_at_point(co, handle, pre, txn) file_points.append((info['lines'], info['line points'], info['points'])) result, ms, newlines = find_resolution(file_points, lines) # explanation of conditions: # 1: check for a merge # 2: check if new lines were added by the user # 3: safety for the 4th condition # 4: check if the first match in the first (only) file covers everything if len(pres) > 1 or \ len(newlines) != 0 or \ not len(ms[0]) or \ ms[0][0][2] != len(file_points[0][0]): # create a set of correct matches, minus ones which are optimized out matches = [[] for i in xrange(plen)] i = 0 for pre, index in pres: matches[index] = ms[i] i += 1 return {'matches': matches, 'newlines': newlines} return None def gen_changeset(co, files, comment, repohead, txn, tstamp=None): def per_file_hash(co, handle, hinfo, precursors, lfile, txn): try: h = open(lfile, 'rb') except IOError: raise HistoryError, 'Could not open file ' + lfile lines = h.read().split('\n') h.close() dinfo = gen_diff(co, handle, precursors, lines, txn) if hinfo.has_key('add'): dinfo['add'] = 1 if hinfo.has_key('delete'): dinfo['delete'] = 1 try: diff = bencode(dinfo) except ValueError: return None mtime = int(path.getmtime(lfile)) co.modtimesdb.put(handle, bencode(mtime), txn=txn) hinfo['hash'] = sha.new(diff).digest() return zlib.compress(diff, 6) precursors = bdecode(co.linforepo.get('heads')) # include the last known repository head in the list of changes. this is # extra useful info and also forces a merge change which a comment can # then be attached to. # XXX: can do the wrong thing in odd merge-and-not-update cases if repohead is not None and repohead not in precursors: while not _is_ancestor(co, repohead, precursors, txn): info = bdecode(co.lcrepo.get(repohead, txn=txn)) try: repohead = info['precursors'][0] except IndexError: repohead = rootnode if repohead not in precursors: precursors.insert(0, repohead) changeset = {'precursors': precursors} changeset['handles'] = handles = {} adds, nedits, edits, types, names = {}, [], [], {}, {} for handle, linfo in files: if linfo.has_key('add'): adds[handle] = 1 #types[handle] = bdecode(co.staticdb.get(handle))['type'] types[handle] = db_get(co, co.staticdb, handle, None)['type'] handles[handle] = cinfo = {} if linfo.has_key('delete'): assert not linfo.has_key('add') assert not linfo.has_key('hash') cinfo['delete'] = 1 elif linfo.has_key('name') or \ linfo.has_key('nmerge'): nedits.append((handle, linfo)) if linfo.has_key('add'): assert not linfo.has_key('hash') cinfo['add'] = {'type': types[handle]} elif linfo.has_key('hash') or \ linfo.has_key('cmerge'): assert types[handle] == 'file' edits.append(handle) co.editsdb.delete(handle, txn=txn) # generate the name diffs for handle, linfo in nedits: # check if this is really a merge or not # XXX: theoretically we can trust the 'nmerge' flag as set (and # cleared) by _update_helper() merge = False change = prev_change = None for head in precursors: change = handle_last_modified(co, co.names, handle, head, txn) if change is None: continue if prev_change is None: prev_change = change continue left_anc = _is_ancestor(co, prev_change, [change], txn) right_anc = _is_ancestor(co, change, [prev_change], txn) if left_anc: prev_change = change elif not right_anc: merge = True break # XXX: sanity check for now, we have to do most of the work anyway assert not (linfo.has_key('nmerge') ^ merge) # no merge, but maybe the user made an explicit change if not linfo.has_key('nmerge') and change is not None: old_info = handle_name_at_point(co, handle, change, txn, lookup=False) if old_info['name'] == linfo['name'] and \ old_info['parent'] == linfo['parent']: continue # looks like we need to include an explicit name change cinfo = handles[handle] hinfo = handle_name(co, handle, txn) cinfo['parent'] = hinfo['parent'] cinfo['name'] = hinfo['name'] names[handle] = cinfo # generate the diffs indices = {} for handle in edits: lfile = path.join(co.local, _handle_to_filename(co, handle, names, txn)) diff = per_file_hash(co, handle, handles[handle], precursors, lfile, txn) if diff is None: continue indices[handle] = write_diff(co, handle, diff, txn) # clear out things which didn't actually have changes for handle, linfo in files: if handles[handle] == {}: del handles[handle] # change all the temporary IDs to permanent, verifiable ones ladds, nmap = adds.keys(), {} while len(ladds): handle = ladds.pop() # check if this handle was already dealt with if not adds.has_key(handle): continue parent = handles[handle]['parent'] # if the parent was also added, it needs to be renumbered first if adds.has_key(parent): ladds.extend((handle, parent)) continue hinfo = handles[handle] # if the parent has been renumbered, pick up the change if nmap.has_key(parent): hinfo['parent'] = nmap[parent] # generate the permanent ID if types[handle] == 'file': # generate diffs fname = _handle_to_filename(co, handle, names, txn) lfile = path.join(co.local, fname) diff = per_file_hash(co, handle, handles[handle], [], lfile, txn) newhandle = create_handle(precursors, hinfo) indices[newhandle] = write_diff(co, newhandle, diff, txn) # update the db accordingly co.modtimesdb.delete(handle, txn=txn) mtime = int(path.getmtime(lfile)) co.modtimesdb.put(newhandle, bencode(mtime), txn=txn) co.filenamesdb.put(fname, newhandle, txn=txn) else: newhandle = create_handle(precursors, hinfo) handles[newhandle] = handles[handle] names[newhandle] = names[handle] types[newhandle] = types[handle] del handles[handle] del adds[handle] # more db updating co.staticdb.delete(handle, txn=txn) #co.staticdb.put(newhandle, bencode({'type': types[handle]}), txn=txn) db_put(co, co.staticdb, newhandle, {'type': types[handle]}, txn) nmap[handle] = newhandle # XXX: clean up allnamesdb # do reparenting of all the non-added files for handle in names.keys(): if nmap.has_key(handle): continue hinfo = handles[handle] if hinfo.has_key('delete'): continue if nmap.has_key(hinfo['parent']): hinfo['parent'] = nmap[hinfo['parent']] if changeset['handles'] == {} and len(changeset['precursors']) == 1: return None # fill in a few other pieces of information if comment is not None: changeset['comment'] = comment changeset['user'] = co.user if tstamp is None: tstamp = time() changeset['time'] = int(tstamp) # put together the changeset and calculate the point bchangeset = bencode(changeset) point = sha.new(bchangeset).digest() # write the file locations of the diffs to the db for handle, index in indices.items(): write_index(co, point, handle, index, txn) # write the new change to the db and make it the new head co.lcrepo.put(point, bchangeset, txn=txn) co.linforepo.put('heads', bencode([point]), txn=txn) #co.editsdb.truncate(txn=txn) return point def find_commit_files(co, handles): r = [] for handle, expanded in handles: lfile = handle_to_filename(co, handle) if not co.editsdb.has_key(handle): if not expanded: print 'warning - %s is not opened for edit' % (lfile,) continue linfo = bdecode(co.editsdb.get(handle)) if linfo.has_key('hash'): if conflicts_in_file(co, lfile): raise CommitError, 'conflicts in %s, must resolve first' % (lfile,) r.append((handle, linfo)) return r Codeville-0.8.0/Codeville/client_net.py0000664000076400007640000006176210631557377017413 0ustar rcohenrcohen# Written by Ross Cohen # see LICENSE.txt for license information from auth import Auth, AuthError from bencode import bdecode, bencode import binascii from client_helpers import handle_name, handle_to_filename, conflicts_in_file from client_helpers import _handle_to_filename, set_edit from crypt import crypt from entropy import random_string, string_to_long, long_to_string from history import read_diff, sync_history, is_ancestor from history import WriteDiff, write_changeset from history import dmerge, handles_in_branch, simplify_precursors from history import fullpath_at_point, handle_contents_at_point from history import tuple_to_server import hmac from merge import find_conflict_multiple_safe, find_conflict, find_resolution from network import Request, Response, NetworkHandler, NetworkError from os import path from RawServer import RawServer import sha import SRP from threading import Event import zlib UpdateInfo = 2 Queue = 3 Flushed = 4 class ClientError(Exception): pass class ServerError(Exception): pass class ClientHandler: def __init__(self, co): self.co = co self.socket = {} self.rs = RawServer(Event(), 100, 1000) self.nh = NetworkHandler(self) # Functions called from lower level code def message_came_in(self, s, data): try: msg = bdecode(data) except ValueError: self.close(s) raise NetworkError, 'garbage data' if msg.has_key('error'): raise ServerError, msg['error'] socket = self.socket[s] srp = socket['srp'] if socket['state'] == 1: K, m = self.auth.client_key(msg['s'], msg['B'], msg['u'], srp['keys']) socket['key'], socket['m_out'] = K, m self._send_msg(s, {'m': socket['m_out'].digest()}) socket['state'] = 2 elif socket['state'] == 2: socket['m_in'] = SRP.host_authenticator(socket['key'], srp['keys'][0], socket['m_out'].digest()) if socket['m_in'].digest() != msg['auth']: raise ServerError, 'Bad host authentication' return self.nh.set_hmac(s, socket['m_in'], socket['m_out']) self.rs.doneflag.set() elif socket['state'] == 3: self.socket[s]['hash'] = msg['hash'] self.rs.doneflag.set() elif socket['state'] == 4: self.close(s) secret = crypt(msg['secret'], socket['key'])[0] self.auth.save_secret(secret) self.rs.doneflag.set() elif socket['state'] == 5: self.close(s) self.rs.doneflag.set() elif socket['state'] == 6: if len(msg['salt']) < 20: self._send_error(s, None, 'Bad salt length') self.close(s) raise NetworkError, 'Bad salt from server' salt = random_string(20) key = self.auth.session_key(salt, msg['salt']) socket['m_in'] = hmac.new(key, '', sha) key = self.auth.session_key(msg['salt'], salt) socket['m_out'] = hmac.new(key, '', sha) self._send_msg(s, {'auth': socket['m_in'].digest(), 'salt': salt}) socket['state'] = 7 elif socket['state'] == 7: if msg['auth'] != socket['m_out'].digest(): self._send_error(s, None, 'Bad auth') self.close(s) raise NetworkError, 'Bad server auth' self._req_mode(s, 1) self.nh.set_hmac(s, socket['m_in'], socket['m_out']) self.socket[s] = [{}, {}, {}, [], 1] self.rs.doneflag.set() else: self.close(s) def connection_flushed(self, s): queue = self.socket[s][Queue] socket = self.socket[s] socket[Flushed] = 1 while len(queue) and socket[Flushed] == 1: mid, msg = queue.pop(0) diff = read_diff(self.co, msg['handle'], msg['changenum'], None) socket[Flushed] = self._send_response(s, mid, {'diff': diff}) def connection_lost(self, s, msg): del self.socket[s] raise NetworkError, msg def request_came_in(self, s, mid, data): try: msg = bdecode(data) except ValueError: self.close(s) raise NetworkError, 'garbage request' self.request_handlers[msg['request']](self, s, mid, msg) def response_came_in(self, s, mid, data): try: msg = bdecode(data) except ValueError: self.close(s) raise NetworkError, 'bad data from server' rstate = self.socket[s][Request][mid] if msg.has_key('error'): # XXX: grody hack, diffs which don't belong in history if rstate['request'] == 'get diff': msg['diff'] = zlib.compress(bencode('')) else: raise ServerError, msg['error'] if self.response_handlers[rstate['request']](self, s, mid, msg, rstate): del self.socket[s][Request][mid] # request handlers def _request_get_change(self, s, mid, msg): resp = {'changeset': self.co.lcrepo.get(msg['changenum'], txn=self.txn)} self._send_response(s, mid, resp) def _request_get_diff(self, s, mid, msg): if self.socket[s][Flushed] == 1: diff = read_diff(self.co, msg['handle'], msg['changenum'], self.txn) self.socket[s][Flushed] = self._send_response(s, mid, {'diff': diff}) else: self.socket[s][Queue].append((mid, msg)) request_handlers = {'get change': _request_get_change, 'get diff': _request_get_diff} # response handlers def _response_get_head(self, s, mid, msg, rstate): return self._merge_change(s, mid, msg['head']) def _merge_change(self, s, mid, head): lstate = self.socket[s][Request][mid] lstate['head'] = head self.socket[s][UpdateInfo]['head'] = head if self.co.lcrepo.has_key(head): named, modified, added, deleted = \ handles_in_branch(self.co, lstate['heads'], [head], None) self._update_checks(s, mid, named, modified) self._update_handle_list(s, lstate, named, modified, added, deleted) self._update_finish(s, lstate) self.socket[s][UpdateInfo]['head'] = head self.rs.doneflag.set() return 1 rid = self._get_change(s, head) self.socket[s][Request][rid] = {'request': 'get change', 'changenum': head, 'ref': mid} lstate['requests'][head] = 1 lstate['count'] = 1 return 0 def _response_get_change(self, s, mid, msg, rstate): lstate = self.socket[s][Request][rstate['ref']] if sha.new(msg['changeset']).digest() != rstate['changenum']: raise ServerError, 'bad changeset' # write it out, decode and eject from memory write_changeset(self.co, rstate['changenum'], msg['changeset'], lstate['txn']) changeset = bdecode(msg['changeset']) lstate['changes'][rstate['changenum']] = changeset del msg['changeset'] # get any precursors we don't have and haven't yet requested for change in changeset['precursors']: if self.co.lcrepo.has_key(change): continue if lstate['changes'].has_key(change): continue if lstate['requests'].has_key(change): continue rid = self._get_change(s, change) self.socket[s][Request][rid] = {'request': 'get change', 'changenum': change, 'ref': rstate['ref']} lstate['requests'][change] = 1 lstate['count'] += 1 # record all the diffs we'll need to request diffs = lstate['diffs'] for handle, hinfo in changeset['handles'].items(): if not hinfo.has_key('hash'): continue if not diffs.has_key(handle): diffs[handle] = {} lstate['counts'][handle] = 0 diffs[handle][rstate['changenum']] = 1 lstate['counts'][handle] += 1 # clean up state del lstate['requests'][rstate['changenum']] lstate['count'] -= 1 if lstate['count'] == 0: sync_history(self.co, lstate['head'], lstate['txn'], cache=lstate['changes']) named, modified, added, deleted = \ handles_in_branch(self.co, lstate['heads'], [lstate['head']], lstate['txn'], cache=lstate['changes']) del lstate['changes'] self._update_checks(s, rstate['ref'], named, modified) self._update_handle_list(s, lstate, named, modified, added, deleted) handle_list = lstate['handle list'] # get all the related file diffs for i in xrange(len(handle_list)-1, -1, -1): handle = handle_list[i][1] if not diffs.has_key(handle): continue changes = diffs[handle] requested = 0 for change in changes.keys(): requested = 1 self._queue_diff(s, change, handle, rstate['ref']) lstate['count'] += requested self._get_diff(s, rstate['ref']) if lstate['count'] == 0: self._update_finish(s, lstate) del self.socket[s][Request][rstate['ref']] self.rs.doneflag.set() return 1 def _response_get_diff(self, s, mid, msg, rstate): lstate = self.socket[s][Request][rstate['ref']] # send out the next one lstate['req-outstanding'] -= 1 self._get_diff(s, rstate['ref']) diffs = lstate['diffs'] diffs[rstate['handle']][rstate['change']] = msg['diff'] lstate['counts'][rstate['handle']] -= 1 if lstate['counts'][rstate['handle']] == 0: lstate['count'] -= 1 # XXX: do better ordering WD = WriteDiff(self.co, rstate['handle'], lstate['txn']) for change, diff in diffs[rstate['handle']].items(): WD.write(diff, change) WD.close() if lstate['modified'].has_key(rstate['handle']): updates(self.co, self.socket[s][UpdateInfo], lstate, rstate['handle']) #del diffs[rstate['handle']] if lstate['count'] == 0: self._update_finish(s, lstate) self.socket[s][UpdateInfo]['head'] = lstate['head'] del self.socket[s][Request][rstate['ref']] self.rs.doneflag.set() return 1 def _response_return(self, s, mid, msg, rstate): self.rs.doneflag.set() return 1 def _response_list_repositories(self, s, mid, msg, rstate): self.socket[s][UpdateInfo] = msg['list'] self.rs.doneflag.set() return 1 response_handlers = {'get head': _response_get_head, 'get change': _response_get_change, 'get diff': _response_get_diff, 'commit': _response_return, 'create repository': _response_return, 'destroy repository': _response_return, 'list repositories': _response_list_repositories} # Internal helper functions def _update_checks(self, s, mid, named, modified): lstate = self.socket[s][Request][mid] check_modified(self.co, modified) lstate['modified'] = {} for handle in modified: lstate['modified'][handle] = 1 self.socket[s][UpdateInfo]['modified'] = [] self.socket[s][UpdateInfo]['modified conflicts'] = [] def _update_handle_list(self, s, lstate, named, modified, added, deleted): uinfo = self.socket[s][UpdateInfo] func = lstate['update function'] handles, nconflicts = func(self.co, uinfo, named, modified, added, deleted, lstate['txn']) handle_list = lstate['handle list'] for handle, show in handles: letters = (' ', ' ') if uinfo['newfiles'].has_key(handle): letters = ('A', 'A') elif uinfo['deletes'].has_key(handle): letters = ('D', 'D') elif uinfo['names'].has_key(handle): letters = (' ', 'N') if nconflicts.has_key(handle): letters = (letters[0], 'C') handle_list.append((handle_to_filename(self.co, handle, lstate['txn']), handle, show, letters)) handle_list.sort() handle_list.reverse() uinfo['name conflicts'] = nconflicts.keys() def _update_finish(self, s, lstate): updates(self.co, self.socket[s][UpdateInfo], lstate, None) def _req_mode(self, s, mode): self.nh.req_mode(s, mode) def _commit(self, s, repo, head): rid = self._send_request(s, {'request': 'commit', 'repository': repo, 'changenum': head}) self.socket[s][Request][rid] = {'request': 'commit'} def _update(self, s, repo, func, txn): heads = bdecode(self.co.linforepo.get('heads')) lstate = {'request': 'get head', 'txn': txn, 'changes': {}, 'requests': {}, 'diffs': {}, 'count': 0, 'counts': {}, 'handle list': [], 'heads': heads, 'reqq': [], 'req-outstanding': 0, 'update function': func} if repo.startswith('{') and repo.endswith('}'): head = binascii.unhexlify(repo[1:-1]) rid = self.nh.next_id(s) self.socket[s][Request][rid] = lstate if self._merge_change(s, rid, head): del self.socket[s][Request][rid] else: rid = self._send_request(s, {'request': 'get head', 'repository': repo}) self.socket[s][Request][rid] = lstate return def _create_repo(self, s, repo): rid = self._send_request(s, {'request': 'create repository', 'repository': repo}) self.socket[s][Request][rid] = {'request': 'create repository', 'repository': repo} def _remove_repo(self, s, repo): rid = self._send_request(s, {'request': 'destroy repository', 'repository': repo}) self.socket[s][Request][rid] = {'request': 'destroy repository', 'repository': repo} def _list_repos(self, s): rid = self._send_request(s, {'request': 'list repositories'}) self.socket[s][Request][rid] = {'request': 'list repositories'} def _get_change(self, s, change): req = {'request': 'get change', 'changenum': change} return self._send_request(s, req) def _queue_diff(self, s, change, handle, mid): rstate = self.socket[s][Request][mid] rstate['reqq'].append((change, handle)) def _get_diff(self, s, mid): rstate = self.socket[s][Request][mid] while len(rstate['reqq']) and rstate['req-outstanding'] <= 40: change, handle = rstate['reqq'].pop(0) req = {'request': 'get diff', 'changenum': change, 'handle': handle} state = {'request': 'get diff', 'ref': mid, 'change': change, 'handle': handle} rid = self._send_request(s, req) self.socket[s][Request][rid] = state rstate['req-outstanding'] += 1 def _send_msg(self, s, data): self.nh.send_msg(s, bencode(data)) def _send_error(self, s, mid, msg): self._send_response(s, mid, {'error': msg}) def _send_request(self, s, data): return self.nh.send_request(s, bencode(data)) def _send_response(self, s, mid, data): return self.nh.send_response(s, mid, bencode(data)) # Functions to be called from higher level code def start_connection(self, dns): s = self.nh.start_connection(dns) self.socket[s] = {'state': 0, 'srp': {}} return s def close(self, s): self.nh.close(s) del self.socket[s] def get_hash(self, s): self._send_msg(s, {'op': 'get hash', 'user': self.co.user}) self.socket[s]['state'] = 3 self.rs.listen_forever(self.nh) self.rs.doneflag.clear() secret_hash = self.socket[s]['hash'] del self.socket[s]['hash'] return secret_hash def srp_auth(self, s): socket = self.socket[s] assert socket['state'] == 0 or socket['state'] == 3 srp = socket['srp'] = {} srp['keys'] = SRP.client_begin() self._send_msg(s, {'op': 'srp auth', 'user': self.co.user, 'A': srp['keys'][0]}) socket['state'] = 1 self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def get_secret(self, s): self._send_msg(s, {'op': 'get secret'}) self.socket[s]['state'] = 4 self.rs.listen_forever(self.nh) self.rs.doneflag.clear() return def set_password(self, s, password): socket = self.socket[s] salt, v = SRP.new_passwd(self.co.user, password) cypherv = crypt(long_to_string(v), socket['key'])[0] self._send_msg(s, {'op': 'set password', 's': salt, 'v': cypherv}) socket['state'] = 5 self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def secret_auth(self, s): socket = self.socket[s] self._send_msg(s, {'op': 'secret auth', 'user': self.co.user}) socket['state'] = 6 self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def commit(self, s, repo, changenum, txn): self.txn = txn self._commit(s, repo, changenum) self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def update(self, s, repo, func, txn): self._update(s, repo, func, txn) self.rs.listen_forever(self.nh) self.rs.doneflag.clear() updateinfo = self.socket[s][UpdateInfo] del self.socket[s][UpdateInfo] return updateinfo def create_repo(self, s, repo): self._create_repo(s, repo) self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def remove_repo(self, s, repo): self._remove_repo(s, repo) self.rs.listen_forever(self.nh) self.rs.doneflag.clear() def list_repos(self, s): self._list_repos(s) self.rs.listen_forever(self.nh) self.rs.doneflag.clear() rlist = self.socket[s][UpdateInfo] del self.socket[s][UpdateInfo] return rlist def network_prep(co): if co.user is None: co.user = co.varsdb.get('user') if co.user is None: raise NetworkError, 'You must set the "user" variable' if co.repo is None: raise NetworkError, 'No repository given' def authenticate(co, ch, remote, txn, srp=False): for i in xrange(3): try: s = _authenticate(co, ch, remote, srp, txn) except (AuthError, ServerError), msg: print msg else: return s raise ServerError, 'Authentication failed' def _authenticate(co, ch, remote, srp, txn): stuple = (remote[0], remote[1], '') server = tuple_to_server(stuple) auth = Auth(co, co.user, server, txn) ch.auth = auth s = ch.start_connection((remote[0], remote[1])) if srp: ch.srp_auth(s) return s hash = ch.get_hash(s) if not auth.check_hash(hash): # we don't know about the secret, fall back to SRP ch.srp_auth(s) ch.get_secret(s) s = ch.start_connection((remote[0], remote[1])) ch.secret_auth(s) auth.forget() return s # XXX: functions below don't really belong here def updates(co, uinfo, lstate, handle): handle_list = lstate['handle list'] modified = uinfo['modified'] mod_conflicts = uinfo['modified conflicts'] keep_list, i = [], 0 for i in xrange(len(handle_list)): lfile, lhandle, show, letters = handle_list.pop() mletter, nletter = letters if lhandle != handle and handle is not None: # it's possible that the server will send us stuff in a different # order than we asked for it. if lstate['modified'].has_key(lhandle): keep_list.append((lfile, lhandle, show, letters)) continue elif lstate['modified'].has_key(lhandle): if show: try: diff = lstate['diffs'][lhandle] del lstate['diffs'][lhandle] except KeyError: diff = {} l = update_file(co, lhandle, lstate['heads'], lstate['head'], uinfo['names'], diff, lstate['txn']) if l: mletter = 'C' mod_conflicts.append(lhandle) elif mletter == ' ': mletter = 'M' del lstate['modified'][lhandle] modified.append(lhandle) else: # XXX: perhaps run handle_contents_at_point as sanity check pass if show: print '%c%c\t%s' % (mletter, nletter, lfile) if lhandle == handle: break keep_list.reverse() handle_list.extend(keep_list) return def update_file(co, handle, pre_heads, rhead, names, dcache, txn): def gen_file_points(prune): file_points, points = [], ['1'] true_pre_heads = simplify_precursors(co, handle, co.contents, pre_heads, txn)[0] # don't use pre_heads which are ancestors of rhead for pre, index in true_pre_heads: if prune and is_ancestor(co, pre, rhead, txn): continue info = handle_contents_at_point(co, handle, pre, txn, dcache=dcache) if info is None: continue points = dmerge(points, info['points']) file_points.append((info['lines'], info['line points'], info['points'])) return (file_points, points) if not co.merge: return 0 rinfo = handle_contents_at_point(co, handle, rhead, txn, dcache=dcache) if rinfo.has_key('delete'): # File was deleted remotely, we're done # XXX: validate remote history return 0 elif co.editsdb.has_key(handle) and bdecode(co.editsdb.get(handle)).has_key('hash'): lfile = _handle_to_filename(co, handle, names, txn) lfile = path.join(co.local, lfile) h = open(lfile, 'rb') lines = h.read().split('\n') h.close() file_points, points = gen_file_points(0) line_points = find_resolution(file_points, lines)[0] for i in xrange(len(line_points)): if line_points[i] is None: line_points[i] = '1' olines = find_conflict(lines, line_points, points, rinfo['lines'], rinfo['line points'], rinfo['points']) else: file_points, points = gen_file_points(1) if file_points == []: # The remote copy is a superset of local changes olines = rinfo['lines'] else: lines, line_points, points = find_conflict_multiple_safe(file_points) olines = find_conflict(lines, line_points, points, rinfo['lines'], rinfo['line points'], rinfo['points']) ls, conflict = [], 0 for l in olines: if type(l) is str: ls.append(l) else: conflict = 1 ls.append('<<<<<<< local') ls.extend(l[0]) ls.append('=======') ls.extend(l[1]) ls.append('>>>>>>> remote') lfile = path.join(co.temppath, binascii.hexlify(handle)) h = open(lfile, 'wb') h.write('\n'.join(ls)) h.close() if conflict: set_edit(co, handle, {'hash': 1}, txn) mtime = int(path.getmtime(lfile)) co.modtimesdb.put(handle, bencode(mtime), txn=txn) return conflict def check_modified(co, modified_files): for handle in modified_files: if handle_name(co, handle, None) is None: continue pinfo = co.editsdb.get(handle) if pinfo is None or not bdecode(pinfo).has_key('hash'): continue file = handle_to_filename(co, handle) if conflicts_in_file(co, file): msg = 'file ' + file + ' has conflicts, resolve before updating' raise ClientError, msg try: import psyco psyco.bind(update_file, 0) except ImportError: pass Codeville-0.8.0/Codeville/crypt.py0000664000076400007640000000110210340212675016370 0ustar rcohenrcohen# Written by Ross Cohen # see LICENSE.txt for license information from entropy import string_to_long, long_to_string import sha def crypt(text, key, counter=0L): keylen, length = len(key), len(text) pos, cyphertext = 0, [] while pos < length: scounter = long_to_string(counter, keylen) hash = sha.new("ctr mode crypt" + key + scounter).digest() for i in xrange(min(length-pos, len(hash))): cyphertext.append(chr(ord(hash[i]) ^ ord(text[pos]))) pos += 1 counter += 1 return (''.join(cyphertext), counter) Codeville-0.8.0/Codeville/db.py0000664000076400007640000000413510400656513015625 0ustar rcohenrcohen# Written by Ross Cohen # See LICENSE.txt for license information. from os import path try: from bsddb3 import db version_info = db.__version__.split('.') if version_info < [4,1]: raise ImportError if db.version() < (4,1): raise ImportError except ImportError: from bsddb import db version_info = db.__version__.split('.') if version_info < [4,1]: raise ImportError, 'bsddb 4.1 or higher is required' if db.version() < (4,1): raise ImportError, 'berkeleydb 4.1 or higher is required' history_format_version = 1 rebuild_format_version = 4 class VersionMismatchException(Exception): pass def check_format_version(dir): try: fd = open(path.join(dir, 'format'), 'r') ver = int(fd.read()) fd.close() except IOError: ver = 0 if ver != history_format_version: raise VersionMismatchException, (history_format_version, ver) return def check_rebuild_version(dir): try: fd = open(path.join(dir, 'rebuild_format'), 'r') ver = int(fd.read()) fd.close() except IOError: ver = 0 if ver != rebuild_format_version: raise VersionMismatchException, (rebuild_format_version, ver) return def write_format_version(dir): fd = open(path.join(dir, 'format'), 'w') fd.write(str(history_format_version)) fd.close() return def write_rebuild_version(dir): fd = open(path.join(dir, 'rebuild_format'), 'w') fd.write(str(rebuild_format_version)) fd.close() return class ChangeDBs: def __init__(self, dbenv, name, flags, txn): self.indexdb = db.DB(dbEnv=dbenv) self.indexdb.open(name[0] + 'index.db', dbtype=db.DB_BTREE, flags=flags, txn=txn) self.dagdb = db.DB(dbEnv=dbenv) self.dagdb.open(name + '-dag.db', dbtype=db.DB_BTREE, flags=flags, txn=txn) self.mergedb = db.DB(dbEnv=dbenv) self.mergedb.open(name + '-merges.db', dbtype=db.DB_BTREE, flags=flags, txn=txn) return def close(self): self.indexdb.close() self.dagdb.close() self.mergedb.close() return Codeville-0.8.0/Codeville/diff.py0000664000076400007640000001022010340212675016140 0ustar rcohenrcohen#!/usr/bin/python # Written by Ross Cohen # see LICENSE.txt for license information from lcsmatch import find_matches def standard_diff(lines1, lines2): matches = find_matches(lines1, lines2) if len(matches) == 0 or matches[0][0] != 0 or matches[0][1] != 0: matches.insert(0, (0, 0, 0)) if matches[-1][0]+matches[-1][2] != len(lines1) or \ matches[-1][1]+matches[-1][2] != len(lines2): matches.append((len(lines1), len(lines2), 0)) text = [] i = 0 j = 0 for i2, j2, l in matches: if i2 == i and j2 == j: i += l j += l continue if i2 == i: at = str(i) elif i2 == i + 1: at = str(i2) else: at = "%d,%d" % (i+1, i2) if j2 == j: bt = str(j) elif j2 == j + 1: bt = str(j2) else: bt = "%d,%d" % (j+1, j2) if i2 > i: if j2 > j: text.extend([at, 'c', bt, "\n"]) else: text.extend([at, 'd', bt, "\n"]) else: text.extend([at, 'a', bt, "\n"]) for i in xrange(i, i2): text.extend(["< ", lines1[i], "\n"]) if i2 > i and j2 > j: text.append("---\n") for j in xrange(j, j2): text.extend(["> ", lines2[j], "\n"]) i = i2 + l j = j2 + l return ''.join(text) def unified_diff(lines1, lines2): matches = find_matches(lines1, lines2) if len(matches) == 0 or \ matches[-1][0]+matches[-1][2] != len(lines1) or \ matches[-1][1]+matches[-1][2] != len(lines2): matches.append((len(lines1), len(lines2), 0)) text = [] if matches[0][:2] == (0, 0): line = line2 = max(matches[0][2] - 3, 0) length = min(matches[0][2], 3) m = 1 else: line, line2 = -1, 0 length = 0 m = 0 while m < len(matches): # capture as much as possible under this header m_end = m while True: line_end, line2_end, length_end = matches[m_end] if length_end > 6 or m_end + 1 == len(matches): break m_end += 1 length_end = min(length_end, 3) # print the diff header for all the hunks we can cover lfudge = max(0, line) header = "@@ -%d,%d +%d,%d @@\n" % \ (lfudge + 1, line_end - lfudge + length_end, line2 + 1, line2_end - line2 + length_end) text.append(header) # print out all the covered hunks with context for m in xrange(m, m_end+1): # fill out the previous context for line in xrange(line, line+length): text.append(' ' + lines1[line] + "\n") line2 += length - 1 s1, s2, length = matches[m] # print the removed and added lines for line in xrange(line+1, s1): text.append('-' + lines1[line] + "\n") for line2 in xrange(line2+1, s2): text.append('+' + lines2[line2] + "\n") line += 1 line2 += 1 len_full = length length = min(3, length) # print the trailing context for i in xrange(line, line + length): text.append(' ' + lines1[i] + "\n") # setup stuff for the next iteration line = s1 + len_full - length line2 = s2 + len_full - length m += 1 return ''.join(text) if __name__ == '__main__': from sys import argv, exit from getopt import gnu_getopt, GetoptError try: optlist, args = gnu_getopt(argv, 'u') except GetoptError, msg: print "Unknown option \"%s\"" % (msg.args[0],) exit(1) if len(args) != 3: print "Usage: %s [-u] file1 file2" % (args[0],) exit(1) diff_func = standard_diff for opt, arg in optlist: if opt == '-u': diff_func = unified_diff h = open(args[1], 'rb') pre_lines = [line[:-1] for line in h.readlines()] h.close() h = open(args[2], 'rb') post_lines = [line[:-1] for line in h.readlines()] h.close() print diff_func(pre_lines, post_lines) Codeville-0.8.0/Codeville/entropy.py0000664000076400007640000000176610340212675016747 0ustar rcohenrcohen# Written by Ross Cohen # See LICENSE.txt for license information. import sys try: from os import urandom as random_string except ImportError: if sys.platform == 'win32': from winrandom import winrandom as random_string else: dev = None def random_string(bytes): """Generate a random string with the given length.""" global dev if dev is None: dev = open('/dev/urandom', 'r') return dev.read(bytes) def random_long(bits): """Generate a random long integer with the given number of bits.""" return string_to_long(random_string((bits+7)/8)) % (1L< 0: s = chr(i & 255) + s i = i >> 8 s = '\x00' * (length - len(s)) + s return s Codeville-0.8.0/Codeville/history.py0000664000076400007640000012262610633634376016762 0ustar rcohenrcohen# Written by Bram Cohen and Ross Cohen # see LICENSE.txt for license information from bencode import bdecode, bencode import binascii from db import db from DFS import DFS from merge import MergeError, replay from os import path from sha import sha import struct from time import localtime, strftime from xml.sax import saxutils import zlib roothandle = '\xdfp\xce\xbaNt5\xf9\xa1\xabd\xf2\xc4\x87\xc5\xd1\x0bI\x8d\xb4' rootnode = '\xb3L\xac\x1f\x98B\x15\\\x8c\t0&\xd7m\xecK\xdd\n\x81\xc4' class HistoryError(Exception): pass def damerge(*dargs): dc = {} for dm in dargs: dc.update({}.fromkeys(dm)) return dc.keys() # keeping this one for possible performance reasons, need to measure def dmerge(da, db): dc = {}.fromkeys(da) dc.update({}.fromkeys(db)) return dc.keys() def rename_conflict_check(linfo, rinfo): def _is_applied(line_points, points): assert list == type(line_points) for p in line_points: if p in points: return True return False # if we're merging, maybe nothing is different. this shields the root # directory from causing problems if linfo == rinfo: return ('local', linfo['rename point']) lapplied = _is_applied(linfo['rename point'], rinfo['points']) rapplied = _is_applied(rinfo['rename point'], linfo['points']) try: if linfo['name'] == rinfo['name'] and \ linfo['parent'] == rinfo['parent']: return ('local', dmerge(linfo['rename point'], rinfo['rename point'])) except KeyError: assert linfo['rename point'] == rinfo['rename point'] == [rootnode] raise HistoryError, 'cannot rename root directory' if rapplied and not lapplied: return ('local', linfo['rename point']) if not rapplied and lapplied: return ('remote', rinfo['rename point']) return ('conflict', None) def _name_use_count(co, state, point, func, txn): cursor = co.allnamesdb.cursor(txn=txn) lookup = state['parent'] + state['name'] try: key, value = cursor.set(lookup) except (db.DBNotFoundError, TypeError): return [] named = [] while key == lookup: vinfo = func(co, value, point, txn) if vinfo is not None: if not vinfo.has_key('delete') and vinfo['parent'] == state['parent'] and vinfo['name'] == state['name']: named.append(value) # next_dup() is broken foo = cursor.next() if foo is None: break key, value = foo cursor.close() return named def name_use_count(co, state, point, txn): return _name_use_count(co, state, point, handle_name_at_point, txn) def _children_count(co, handle, point, func, txn): cursor = co.allnamesdb.cursor(txn=txn) try: foo = cursor.set_range(handle) except (db.DBNotFoundError, TypeError): foo = None children = {} while foo is not None: key, value = foo parent = key[:20] if parent != handle: break vinfo = func(co, value, point, txn) if vinfo is not None: if not vinfo.has_key('delete') and vinfo['parent'] == handle: children[value] = None foo = cursor.next() cursor.close() return children.keys() def children_count(co, handle, point, txn): return _children_count(co, handle, point, handle_name_at_point, txn) def parent_loop_check(co, handle, point, txn): hseen = {} while handle != roothandle: if hseen.has_key(handle): return handle hseen[handle] = 1 phandle = handle_name_at_point(co, handle, point, txn)['parent'] if phandle is None: return handle handle = phandle return None def is_ancestor(co, ancestor, point, txn): if point is None: return 0 return _is_ancestor(co, ancestor, [point], txn) def _is_ancestor(co, ancestor, points, txn): #ainfo = bdecode(co.branchmapdb.get(ancestor, txn=txn)) ainfo = db_get(co, co.branchmapdb, ancestor, txn) points = points[:] state = {} while len(points): pnext = points.pop() if pnext == ancestor: return 1 # if it's in state, then we've already walked it if pnext in state: continue state[pnext] = 1 #pinfo = bdecode(co.branchmapdb.get(pnext, txn=txn)) pinfo = db_get(co, co.branchmapdb, pnext, txn) if pinfo['generation'] <= ainfo['generation']: continue ps = pinfo['precursors'][:] ps.reverse() points.extend(ps) return 0 def read_diff(co, handle, point, txn): hinfo = bdecode(co.contents.dagdb.get(handle + point, txn=txn)) hfile = open(path.join(co.cpath, binascii.hexlify(handle)), 'rb') #hfile = open(path.join(co.cpath, 'diffs'), 'rb') diff = _read_diff(hinfo, hfile) hfile.close() return diff def _read_diff(index, hfile): if not index.has_key('handle') or not index['handle'].has_key('offset'): return None try: hfile.seek(index['handle']['offset']) except IOError: print index raise return hfile.read(index['handle']['length']) def write_diff(co, handle, diff, txn): cdagdb = co.contents.dagdb try: fend = struct.unpack(' c_%s [style=%s]' % (short_id(co, pre), sid, style) style = "dashed" points.extend([point for point in info['precursors']]) print '}' def pretty_print_dag(co, handle, heads): print 'digraph {' points = [] for point in heads: head = handle_last_modified(co, co.contents, handle, point, None) if head is not None: points.append(head) cdagdb = co.contents.dagdb cache = {} while len(points): point = points.pop() if cache.has_key(point): continue cache[point] = 1 sid = short_id(co, point) print 'c_%s [label="%s"]' % (sid, sid) info = bdecode(cdagdb.get(handle + point)) for pre, foo in info['precursors']: print 'c_%s -> c_%s' % (sid, short_id(co, pre)) points.extend([point for point, index in info['precursors']]) print '}' def _update_mini_dag(co, changedbs, helper, handles, cset, txn): indexdb = changedbs.indexdb dagdb = changedbs.dagdb pres = cset['precursors'] point = cset['point'] #bminfo = bdecode(co.branchmapdb.get(point, txn=txn)) bminfo = db_get(co, co.branchmapdb, point, txn) bnum = struct.pack('>II', bminfo['branch'], bminfo['branchnum']) untouched = [] for handle in handles: precursors = simplify_precursors(co, handle, changedbs, pres, txn)[0] mdinfo = {'handle': {}} # there's some broken history, need to work around it for now # deleted files are deleted, regardless of later edits deleted = None #if len(precursors) > 1: if True: # only deletes can be implicitly merged all_deleted = True for pre in precursors: phinfo = bdecode(dagdb.get(handle + pre[0], txn=txn)) if phinfo['handle'].has_key('delete'): deleted = phinfo else: all_deleted = False if all_deleted: if len(precursors) > 1: # affects finding when the handle was modified untouched.append(handle) continue if deleted is not None: mdinfo['handle'] = deleted['handle'] elif cset['handles'].has_key(handle): mdinfo['handle'] = helper(co, handle, point, precursors, cset['handles'][handle], txn) if mdinfo['handle'] == {}: if deleted is None: if len(precursors) > 1: raise HistoryError, 'cannot automatically merge changes' del mdinfo['handle'] mdinfo['precursors'] = precursors if precursors == []: assert cset['handles'][handle].has_key('add') dagdb.put(handle + point, bencode(mdinfo), txn=txn) indexdb.put(handle + bnum, point, txn=txn) return untouched def simplify_precursors(co, handle, changedbs, pres, txn): # map big DAG precursors to little DAG dagdb = changedbs.dagdb precursors, indices = [], [] for i in xrange(len(pres)): last = handle_last_modified(co, changedbs, handle, pres[i], txn) if last is None: continue precursors.append(last) indices.append(i) # second pass to eliminate precursors which are ancestors of others retval = [] for i in xrange(len(precursors)): pre = precursors.pop(0) index = indices.pop(0) if _is_ancestor(co, pre, precursors, txn): continue precursors.append(pre) retval.append((pre, index)) return retval, len(pres) def _update_helper_content(co, handle, point, precursors, hinfo, txn): oinfo = {} if hinfo.has_key('hash'): if co.contents.dagdb.has_key(handle + point, txn): dinfo = bdecode(co.contents.dagdb.get(handle + point, txn=txn)) oinfo['offset'] = dinfo['handle']['offset'] oinfo['length'] = dinfo['handle']['length'] else: # XXX: ugly in general oinfo['offset'] = -1 if hinfo.has_key('add'): oinfo['add'] = 1 elif hinfo.has_key('delete'): oinfo['delete'] = 1 return oinfo def _update_helper_name(co, handle, point, precursors, hinfo, txn): oinfo = {} if hinfo.has_key('name'): # XXX: work around some broken history, remove this later # check to see if the file was already deleted. for pre in precursors: phinfo = bdecode(co.names.dagdb.get(handle + pre[0], txn=txn)) if phinfo['handle'].has_key('delete'): oinfo = phinfo['handle'] return oinfo oinfo['name'] = hinfo['name'] try: oinfo['parent'] = hinfo['parent'] co.allnamesdb.put(hinfo['parent'] + hinfo['name'], handle, flags=db.DB_NODUPDATA, txn=txn) except KeyError: assert handle == roothandle assert hinfo.has_key('add') except db.DBKeyExistError: pass if hinfo.has_key('add'): assert len(precursors) == 0 oinfo['add'] = 1 elif hinfo.has_key('delete'): # part of making lookups faster phinfo = bdecode(co.names.dagdb.get(handle + precursors[0][0], txn=txn)) oinfo = phinfo['handle'] oinfo['delete'] = 1 else: # something to make name lookups faster for pre in precursors: phinfo = bdecode(co.names.dagdb.get(handle + pre[0], txn=txn)) if phinfo.has_key('delete'): oinfo = phinfo['handle'] break if oinfo != {}: assert oinfo.has_key('name') or oinfo.has_key('delete') return oinfo def changes_in_branch(co, lpoints, bpoints, txn, cache=None): points = bpoints[:] seen, changes = {}, [] while len(points): pnext = points.pop() if seen.has_key(pnext): continue seen[pnext] = 1 if _is_ancestor(co, pnext, lpoints, txn): continue if cache: if cache.has_key(pnext): pinfo = cache[pnext] else: pinfo = bdecode(co.lcrepo.get(pnext, txn=txn)) cache[pnext] = pinfo else: pinfo = bdecode(co.lcrepo.get(pnext, txn=txn)) points.extend(pinfo['precursors']) changes.append(pnext) return changes def handles_in_branch(co, lpoints, bpoints, txn, cache=None): points = bpoints[:] seen = {} named, modified, added, deleted = {}, {}, {}, {} while len(points): pnext = points.pop() if seen.has_key(pnext): continue seen[pnext] = 1 if _is_ancestor(co, pnext, lpoints, txn): continue if cache: if cache.has_key(pnext): pinfo = cache[pnext] else: pinfo = bdecode(co.lcrepo.get(pnext, txn=txn)) cache[pnext] = pinfo else: pinfo = bdecode(co.lcrepo.get(pnext, txn=txn)) for handle, hinfo in pinfo['handles'].items(): if hinfo.has_key('add'): added[handle] = 1 if hinfo.has_key('name'): named[handle] = 1 if hinfo.has_key('hash'): modified[handle] = 1 if hinfo.has_key('delete'): deleted[handle] = 1 points.extend(pinfo['precursors']) return (named.keys(), modified.keys(), added.keys(), deleted.keys()) def handle_last_modified(co, changedbs, handle, change, txn): retval = _handle_last_modified(co, changedbs, handle, change, txn) if retval is None: return retval # optimization step. we may have gotten an artifact of the indexing # structure. because of how the mini-dags are constructed, this can be # corrected by following a single precursor. pinfo = bdecode(changedbs.dagdb.get(handle + retval, txn=txn)) if not pinfo.has_key('handle') and len(pinfo['precursors']) == 1: retval = _handle_last_modified(co, changedbs, handle, pinfo['precursors'][0][0], txn) return retval def _handle_last_modified(co, changedbs, handle, change, txn): indexdb = changedbs.indexdb dagdb = changedbs.dagdb if dagdb.has_key(handle + change, txn): return change #hinfo = bdecode(co.branchmapdb.get(change, txn=txn)) hinfo = db_get(co, co.branchmapdb, change, txn) bbranch, bnum = int(hinfo['branch']), hinfo['branchnum'] retval = None cursor = indexdb.cursor(txn=txn) while True: branchpoint = struct.pack('>20sII', handle, bbranch, bnum) try: foo = cursor.set_range(branchpoint) except db.DBNotFoundError: foo = None if foo is None: foo = cursor.last() if foo is None: break key, value = foo else: key, value = foo if key != branchpoint: try: foo = cursor.prev_nodup() except db.DBNotFound: foo = None if foo is None: break key, value = foo rhandle, rbranch, rnum = struct.unpack('>20sII', key) if rhandle == handle and rbranch == bbranch: retval = value break #pinfo = bdecode(co.branchdb.get(bbranch, txn=txn)) pinfo = db_get(co, co.branchdb, bbranch, txn) try: bbranch, bnum = int(pinfo['parent']), pinfo['parentnum'] except KeyError: #_print_indexdb(co, indexdb, handle, txn) break cursor.close() return retval def _print_indexdb(co, indexdb, handle, txn): print 'indexdb' cursor = indexdb.cursor(txn=txn) foo = cursor.set_range(handle) while foo != None: key, value = foo nhandle, branch, num = struct.unpack('>20sII', key) if handle != nhandle: break print "%d, %d" % (branch, num) foo = cursor.next() cursor.close() def handle_name_at_point(co, handle, point, txn, lookup=True): change = point if lookup: change = handle_last_modified(co, co.names, handle, point, txn) if change is None: return None blinfo = co.names.dagdb.get(handle + change, txn=txn) linfo = bdecode(blinfo) return linfo['handle'] def __handle_name_at_point(co, handle, point, txn, docache=True): #key = handle + point #if co.name_cache.has_key(key): # return co.name_cache[key] change = handle_last_modified(co, co.names, handle, point, txn) if change is None: return None return _handle_name_at_point(co, handle, change, txn, docache=docache) def _handle_name_from_precursors(precursors, resolution, point): if resolution.has_key('delete'): resolution.update(precursors[0]) return resolution['rename point'] = [] for pre in precursors: if pre.has_key('delete'): resolution.update(pre) return if pre['name'] == resolution['name'] and \ pre['parent'] == resolution['parent']: assert list == type(pre['rename point']) resolution['rename point'].extend(pre['rename point']) if resolution['rename point'] == []: resolution['rename point'] = [point] return # figure out the whether we should assign a new rename point. this handles # 2 cases: # 1) ambiguous merge picking the line points of an ancestor # 2) ancestors are A and B where B should win, but user reassigned to A for pre in precursors: outcome, rename_point = rename_conflict_check(resolution, pre) if outcome != 'local': resolution['rename point'] = [point] break return def _deleted_in_precursors(precursors): deleted = True for pre in precursors: if not pre.has_key('delete'): deleted = False break return deleted def _handle_name_at_point(co, handle, point, txn, docache=False): cset = bdecode(co.names.dagdb.get(handle + point, txn=txn)) precursors, points = [], [point] for pre, index in cset['precursors']: pre_name = _handle_name_at_point(co, handle, pre, txn, docache=docache) if pre_name is None: continue points = dmerge(points, pre_name['points']) precursors.append(pre_name) state = {} if not cset.has_key('handle'): if len(cset['precursors']) != 1: # we don't require deletes to be explicitly merged if not _deleted_in_precursors(precursors): raise HistoryError, 'implicit name merges not permitted' state = precursors[0] elif cset['handle'].has_key('delete'): state['delete'] = 1 else: state['name'] = cset['handle']['name'] try: state['parent'] = cset['handle']['parent'] except KeyError: assert handle == roothandle assert cset['handle'].has_key('add') state['points'] = points _handle_name_from_precursors(precursors, state, point) if not docache: return state co.name_cache[handle + point] = state return state def fullpath_at_point(co, handle, point, txn=None): name = None while handle != roothandle: pinfo = handle_name_at_point(co, handle, point, txn) if pinfo is None: return None if name is None: name = pinfo['name'] else: name = path.join(pinfo['name'], name) handle = pinfo['parent'] return name def _mini_dag_refcount(co, handle, point, txn, cache=None, info_cache=None): assert info_cache is not None if cache is None: cache = {} points = [point] while len(points): point = points.pop() if cache.has_key(point): cache[point]['refcount'] += 1 continue cache[point] = {'refcount': 1} pinfo = bdecode(co.contents.dagdb.get(handle + point, txn=txn)) info_cache[point] = pinfo for p, i in pinfo['precursors']: points.append(p) return cache def _merge_check_deps(node, args): co, handle, txn, hcache = args[0], args[1], args[2], args[3] hinfo = bdecode(co.contents.dagdb.get(handle + node, txn=txn)) hcache[node] = hinfo return [p[0] for p in hinfo['precursors']] def handle_merge_check(co, handle, point, txn): change = handle_last_modified(co, co.contents, handle, point, txn) if change is None: return None hcache = {} cDFS = DFS(_merge_check_deps, [co, handle, txn, hcache]) cDFS.search(change) ordering = cDFS.result() for point in ordering: hinfo = hcache[point] if not hinfo.has_key('handle') and len(hinfo['precursors']) > 1: raise HistoryError, 'cannot automatically merge changes' return def _content_deps(node, args): hcache = args[0] return [p[0] for p in hcache[node]['precursors']] def handle_contents_at_point(co, handle, point, txn, dcache=None, replayfunc=replay): if dcache is None: dcache = {} #staticinfo = bdecode(co.staticdb.get(handle, txn=txn)) staticinfo = db_get(co, co.staticdb, handle, txn) if staticinfo['type'] != 'file': raise ValueError, 'Expected type \"file\", got type \"%s\"' % \ (staticinfo['type'],) change = handle_last_modified(co, co.contents, handle, point, txn) if change is None: return None hcache = {} cache = _mini_dag_refcount(co, handle, change, txn, info_cache=hcache) hfile = open(path.join(co.cpath, binascii.hexlify(handle)), 'rb') #hfile = open(path.join(co.cpath, 'diffs'), 'rb') cDFS = DFS(_content_deps, [hcache]) cDFS.search(change) ordering = cDFS.result() for point in ordering: hinfo = hcache[point] if hinfo['handle'].has_key('delete'): # Pick contents of an ancestor, doesn't really matter cache[point]['info'] = cache[hinfo['precursors'][0][0]].copy() cache[point]['info']['delete'] = hinfo['handle']['delete'] # put together the precursor list and decrement refcounts precursors = [] for pre, foo in hinfo['precursors']: precursors.append(cache[pre]['info']) cache[pre]['refcount'] -= 1 if cache[pre]['refcount'] == 0: del cache[pre] if hinfo['handle'].has_key('delete'): # This has to be done after decrementing refcounts, whereas the # content setting has to be done before. continue # read the diff if dcache.has_key(point): diff = dcache[point] else: diff = _read_diff(hinfo, hfile) if diff is None: if len(hinfo['precursors']) > 1: raise HistoryError, 'cannot automatically merge changes' raise HistoryError, 'change with no diff' diff = bdecode(zlib.decompress(diff)) # finally, get the contents cache[point]['info'] = _handle_contents_at_point(point, hinfo, precursors, diff, replayfunc=replayfunc) hfile.close() cache[change]['info']['type'] = staticinfo['type'] return cache[change]['info'] def _handle_contents_at_point(point, hinfo, precursors, diff, replayfunc=replay): state = {} matches = [] for pre, index in hinfo['precursors']: matches.append(diff['matches'][index]) if diff.has_key('add'): if precursors != []: raise HistoryError, 'handle already exists' elif precursors == []: raise HistoryError, 'cannot modify non-existent file' if diff.has_key('delete'): state['delete'] = diff['delete'] if not state.has_key('delete'): fpre = [] for pre in precursors: fpre.append((pre['lines'], pre['line points'], pre['points'])) try: lines, line_points, points = replayfunc(fpre, matches, diff['newlines'], point) except MergeError, msg: raise HistoryError, 'merge error: ' + str(msg) except KeyError: raise HistoryError, 'malformed change' points.append(point) state['lines'] = lines state['line points'] = line_points state['points'] = points return state def print_file_with_points(pre): def _print_points(line_points): ps = [] for line_point in line_points: ps.append(binascii.hexlify(line_point)) return ', '.join(ps) lines, line_points, points = pre out = [_print_points(line_points[0])] for i in xrange(len(lines)): out.append(lines[i]) out.append(_print_points(line_points[i+1])) return '\n'.join(out) from merge import _find_conflict def print_conflict(co, fpre): p1, p2 = fpre[0], fpre[1] olines, oline_points = _find_conflict(fpre[0][0], fpre[0][1], fpre[0][2], fpre[1][0], fpre[1][1], fpre[1][2])[:2] ls = [] offset = [0, 0] for i in xrange(len(olines)): l = olines[i] if type(l) is str: offset[0] += 1 offset[1] += 1 continue print '@@ -%d,%d +%d,%d @@' % (offset[0], len(l[0]), offset[0], len(l[1])) offset[0] += len(l[0]) offset[1] += len(l[1]) lp = oline_points[i] ls.append('<<<<<<< local') ps = ', '.join([short_id(co, p) for p in lp[0][0]]) ls.append(ps) for j in xrange(len(l[0])): ls.append(l[0][j]) ps = ', '.join([short_id(co, p) for p in lp[0][j+1]]) ls.append(ps) ls.append('=======') ps = ', '.join([short_id(co, p) for p in lp[1][0]]) ls.append(ps) for j in xrange(len(l[1])): ls.append(l[1][j]) ps = ', '.join([short_id(co, p) for p in lp[1][j+1]]) ls.append(ps) ls.append('>>>>>>> remote') return '\n'.join(ls) def rebuild_from_points(co, points, txn): co.changesdb.truncate(txn) co.branchdb.truncate(txn) co.branchmapdb.truncate(txn) co.names.indexdb.truncate(txn) co.names.dagdb.truncate(txn) co.contents.indexdb.truncate(txn) # we don't truncate the cdagdb because it contains the offsets and lengths # for the diffs in the files, which we can't recreate. the sync below will # read those parameters out and rewrite the cdagdb, anyway. co.linforepo.put('branchmax', bencode(0), txn=txn) cdagdb = co.contents.dagdb for key, value in cdagdb.items(txn): if len(key) != 40: continue if not bdecode(value).has_key('handle'): cdagdb.delete(key, txn=txn) for point in points: sync_history(co, point, txn) def clean_merge_point(info): if info['handles'] != {}: return 0 if len(info['precursors']) != 2: return 0 if info.has_key('comment'): return 0 return 1 def short_id(co, point): apoint = binascii.hexlify(point) length = 3 key = apoint[:length] cursor = co.changesdb.cursor() while key.startswith(apoint[:length]) and length < len(apoint): length += 1 try: key, value = cursor.set_range(apoint[:length]) except (db.DBNotFoundError, TypeError): break if key != apoint: continue foo = cursor.next() if foo is None: break key, value = foo cursor.close() return apoint[:length] class ChangeNotKnown(Exception): pass class ChangeNotUnique(Exception): pass class ChangeBadRepository(Exception): pass def long_id(co, point): if point.startswith('cdv://'): key = repo_head(co, tuple_to_server(server_to_tuple(point))) if key is None: raise ChangeBadRepository, point return key cursor = co.changesdb.cursor() try: key, value = cursor.set_range(point) except (db.DBNotFoundError, TypeError): cursor.close() raise ChangeNotKnown, point if not key.startswith(point): cursor.close() raise ChangeNotKnown, point try: key2, value = cursor.next() except (db.DBNotFoundError, TypeError): cursor.close() return binascii.unhexlify(key) if key2.startswith(point): keys = [key] while key2.startswith(point): keys.append(key2) try: key2, value = cursor.next() except (db.DBNotFoundError, TypeError): break cursor.close() raise ChangeNotUnique, (point, keys) cursor.close() return binascii.unhexlify(key) def repo_head(co, repo): if repo is None: return None repo = tuple_to_server(server_to_tuple(repo)) return co.linforepo.get(repo) def server_to_tuple(server_string): if server_string.startswith('cdv://'): server_string = server_string[6:] temp = server_string.split('/', 1) if len(temp) != 2 or temp[1] == '': repo = None else: repo = temp[1] if repo[-1] == '/': repo = repo[:-1] temp = temp[0].split(':', 1) if len(temp) == 1: port = 6601 else: try: port = int(temp[1]) except ValueError: return None server_string = temp[0] return (temp[0], port, repo) def tuple_to_server(tuple): return 'cdv://%s:%d/%s' % tuple def print_change(co, change, changeset): escape = saxutils.escape output = [] output.append("") output.append("" + binascii.hexlify(change) + "") output.append("" + short_id(co, change) + "") output.append("" + escape(changeset['user']) + "") try: output.append("" + escape(changeset['comment']) + "") except KeyError: pass output.append("" + str(changeset['time']) + "") output.append("" + strftime('%a %b %d %H:%M:%S %Y %Z', localtime(changeset['time'])) + "") adds, deletes, renames, mods = {}, {}, {}, {} for handle, value in changeset['handles'].items(): if value.has_key('add'): adds[handle] = value['add'] else: if value.has_key('delete'): deletes[handle] = 1 if value.has_key('name'): renames[handle] = 1 if value.has_key('hash'): mods[handle] = 1 output.append("") for handle, info in adds.items(): output.append("") output.append("" + escape(fullpath_at_point(co, handle, change)) + "") output.append("" + info['type'] + "") output.append("") output.append("") output.append("") for handle in deletes.keys(): output.append("") output.append("" + escape(fullpath_at_point(co, handle, change)) + "") output.append("") output.append("") output.append("") for handle in renames.keys(): output.append("") output.append("" + escape(fullpath_at_point(co, handle, change)) + "") output.append("") output.append("") output.append("") for handle in mods.keys(): output.append("") output.append("" + escape(fullpath_at_point(co, handle, change)) + "") output.append("") output.append("") output.append("") return output def dump_changeinfo(co, change, repo=None): output = [] changeset = bdecode(co.lcrepo.get(change)) if not clean_merge_point(changeset): raise ValueError output.append("") if repo: output.append("" + saxutils.escape(repo) + "") output.append("" + binascii.hexlify(change) + "") output.append("" + short_id(co, change) + "") output.append("" + saxutils.escape(changeset['user']) + "") output.append("" + str(changeset['time']) + "") for change in changes_in_branch(co, [changeset['precursors'][0]], changeset['precursors'][1:], None): changeset = bdecode(co.lcrepo.get(change)) if not clean_merge_point(changeset): output.extend(print_change(co, change, changeset)) output.append("\n") return '\n'.join(output) def db_get(co, cdb, key, txn): try: cache = co.db_cache[db] except KeyError: cache = co.db_cache[db] = {} if key in cache: return cache[key] cache[key] = bdecode(cdb.get(key, txn=txn)) #try: # return cache[key] #except KeyError: # cache[key] = bdecode(cdb.get(key, txn=txn)) return cache[key] def db_put(co, cdb, key, value, txn): cdb.put(key, bencode(value), txn=txn) try: cache = co.db_cache[db] except KeyError: cache = co.db_cache[db] = {} cache[key] = value try: import psyco psyco.bind(_is_ancestor, 1) psyco.bind(handle_last_modified, 1) except ImportError: pass Codeville-0.8.0/Codeville/lcsmatch.py0000664000076400007640000001575610340212675017051 0ustar rcohenrcohen#!/usr/bin/env python # Written by Bram Cohen and Uoti Urpala from bisect import bisect def unique_lcs(a, b): # set index[line in a] = position of line in a unless # unless a is a duplicate, in which case it's set to None index = {} for i in xrange(len(a)): line = a[i] if line in index: index[line] = None else: index[line]= i # make btoa[i] = position of line i in a, unless # that line doesn't occur exactly once in both, # in which case it's set to None btoa = [None] * len(b) index2 = {} for pos, line in enumerate(b): next = index.get(line) if next is not None: if line in index2: # unset the previous mapping, which we now know to # be invalid because the line isn't unique btoa[index2[line]] = None del index[line] else: index2[line] = pos btoa[pos] = next # this is the Patience sorting algorithm # see http://en.wikipedia.org/wiki/Patience_sorting backpointers = [None] * len(b) stacks = [] lasts = [] k = 0 for bpos, apos in enumerate(btoa): if apos is None: continue # skip over solitary matches with no surrounding matching context if (bpos == 0 or apos == 0 or a[apos-1] != b[bpos-1]) and \ (bpos == len(b)-1 or apos == len(a)-1 or a[apos+1] != b[bpos+1]): continue # as an optimization, check if the next line comes right after # the previous line, because usually it does if stacks and stacks[k] < apos and (k == len(stacks) - 1 or stacks[k+1] > apos): k += 1 else: k = bisect(stacks, apos) if k > 0: backpointers[bpos] = lasts[k-1] if k < len(stacks): stacks[k] = apos lasts[k] = bpos else: stacks.append(apos) lasts.append(bpos) if len(lasts) == 0: return [] result = [] k = lasts[-1] while k is not None: result.append((btoa[k], k)) k = backpointers[k] result.reverse() return result def test_lcs(): assert unique_lcs('', '') == [] assert unique_lcs('a', 'a') == [] assert unique_lcs('a', 'b') == [] assert unique_lcs('ab', 'ab') == [(0, 0), (1, 1)] assert unique_lcs('abcde', 'cdeab') == [(2, 0), (3, 1), (4, 2)] assert unique_lcs('cdeab', 'abcde') == [(0, 2), (1, 3), (2, 4)] assert unique_lcs('abXde', 'abYde') == [(0, 0), (1, 1), (3, 3), (4, 4)] return def recurse_matches(a, b, ahi, bhi, answer, maxrecursion, is_junk): # this will never happen normally, this check is to prevent DOS attacks if maxrecursion < 0: return oldlength = len(answer) # check for new matches after the last confirmed match ends if oldlength == 0: alo = 0 blo = 0 else: lasta, lastb, lastlen = answer[-1] alo = lasta + lastlen blo = lastb + lastlen if alo == ahi or blo == bhi: return last_enda = alo last_endb = blo last_checkb = blo # extend individual line matches into match sections for apos, bpos in unique_lcs(a[alo:ahi], b[blo:bhi]): apos += alo bpos += blo # don't overlap with an existing match or check something which # already got thrown out as junk if bpos < last_checkb or apos < last_enda: continue # extend line match as far in either direction as possible enda = apos + 1 endb = bpos + 1 while enda < ahi and endb < bhi and a[enda] == b[endb]: enda += 1 endb += 1 while apos > last_enda and bpos > last_endb and a[apos-1] == b[bpos-1]: apos -= 1 bpos -= 1 # mark what's been checked now, so even if it's junked it doesn't # have to be checked again last_checkb = endb # if this section is junk, skip it numreal = 0 for k in xrange(apos, enda): if not is_junk(a[k]): numreal += 1 if numreal >= 2: break else: # Niklaus Wirth can bite me continue last_enda = enda last_endb = endb # find matches which come before the new match section # this can happen because there may be lines which weren't unique # in the whole file but are unique in the subsection recurse_matches(a, b, apos, bpos, answer, maxrecursion - 1, is_junk) answer.append((apos, bpos, enda - apos)) if len(answer) > oldlength: # find matches between the last match and the end recurse_matches(a, b, ahi, bhi, answer, maxrecursion - 1, is_junk) # else: fall back to difflib (possibly a good idea, possibly not) def default_is_junk(x): return len(x.strip()) <= 2 def find_matches(a, b, is_junk = default_is_junk): # single-line identical files match, despite being too short for # a real match if they were part of a larger file if a == b: return [(0, 0, len(a))] answer = [] recurse_matches(a, b, len(a), len(b), answer, 10, is_junk) return answer try: import psyco psyco.bind(unique_lcs, 0) except ImportError: pass # Stuff below here is for testing def x(a, b): t1 = time.time() r = unique_lcs(a, b) t2 = time.time() #print r for i, j in r: assert a[i] == b[j] #print a[i] print print 'time:', t2-t1 def return_false(x): return False def x2(a, b, is_junk = return_false): t1 = time.time() r = find_matches(a, b, is_junk) t2 = time.time() #print r for i, j, l in r: assert a[i:i+l] == b[j:j+l] #print ''.join(a[i:i+l]), print print 'time:', t2-t1 def completely_random_test(x): a = [str(i) for i in range(100000)] b = list(a) random.shuffle(b) #print ' '.join(b) #print x(a, b) def random_with_ranges_test(x): d = [[str(i), str(i+1)] for i in xrange(0, 100000, 2)] c = list(d) random.shuffle(c) a = [] for i in d: a.extend(i) b = [] for i in c: b.extend(i) #print ' '.join(b) #print x(a, b) def is_A_test(s): return s == 'A' def test_lcsmatch(): global random, time import random import time cur_time = time.time() random.seed(cur_time) print 'Seeded tests with %s' % (cur_time,) completely_random_test(x) random_with_ranges_test(x) x2('0123456789abc', ' 01 89a 34 67') x2('AB 1 CD 1 AB P XY Q AB 1 CD 1 AB', 'AB 2 CD 2 AB Q XY P AB 2 CD 2 AB') x2('AjBjC', 'AjC') x2('AjC', 'AjBjC') x2('AjBjC', 'AjC', is_A_test) x2('AjC', 'AjBjC', is_A_test) x2('x', 'x') x2('01 2', '01') x2('ABPXYQAB', 'ABQXYPAB') return Codeville-0.8.0/Codeville/merge.py0000664000076400007640000007633610340212675016353 0ustar rcohenrcohen# Written by Bram Cohen # see LICENSE.txt for license information from lcsmatch import find_matches from sets import Set class SelectLinePoint: def __init__(self): self.lp = Set() self.rp = Set() return def push(self, points): self.lp.union_update(self.rp) self.rp = Set(points) return # this behaves like the general merge algorithm on a point by point basis def select(self, p1, p2): assert p2 is not None if p1 is None: return p2 elif p1 == p2: return p1 elif p2 in self.lp and p1 not in self.rp: return p1 elif p2 not in self.lp and p1 in self.rp: return p2 # it's a coincidental match, canonically choose the larger return max(p1, p2) def dump(self): return [p for p in self.lp | self.rp] def find_conflict(local, local_line_points, local_points, remote, remote_line_points, remote_points): return _find_conflict(local, local_line_points, local_points, remote, remote_line_points, remote_points)[0] def find_conflict_safe(local, local_line_points, local_points, remote, remote_line_points, remote_points): return find_conflict_multiple_safe([(local, local_line_points, local_points), (remote, remote_line_points, remote_points)])[:2] def find_conflict_multiple_safe(precursors): olds = precursors[:] lines, line_points, points = olds.pop(0) while olds: nlines, nlinepoints, npoints = olds.pop(0) lines, line_points, points = _find_conflict(lines, line_points, points, nlines, nlinepoints, npoints) if None in line_points: return (None, None, None) return lines, line_points, points def _is_applied(ll, pdict): for i in ll: if i not in pdict: return 0 return 1 def _find_conflict(local, local_line_points, local_points, remote, remote_line_points, remote_points): assert len(local_line_points) == len(local) + 1 assert len(remote_line_points) == len(remote) + 1 lpdict = Set(local_points) for point in local_line_points: assert point in lpdict rpdict = Set(remote_points) for point in remote_line_points: assert point in rpdict result = [] result_line_points = [] lastl, lastr = 0, 0 selector = SelectLinePoint() selector.push(local_points) selector.push(remote_points) for (lpos, rpos, matchlen) in find_matches(local, remote) + [(len(local), len(remote), 0)]: rapplied = _is_applied(remote_line_points[lastr:rpos + 1], lpdict) lapplied = _is_applied(local_line_points[lastl:lpos + 1], rpdict) if rapplied and not lapplied: result.extend(local[lastl:lpos + matchlen]) result_line_points.extend(local_line_points[lastl:lpos + 1]) elif lapplied and not rapplied: result.extend(remote[lastr:rpos + matchlen]) result_line_points.extend(remote_line_points[lastr:rpos + 1]) elif (lastl, lastr) == (len(local), len(remote)): result_line_points.append(selector.select(local_line_points[-1], remote_line_points[-1])) elif (lpos, rpos) == (0, 0): result.extend(local[:matchlen]) result_line_points.append(selector.select(local_line_points[0], remote_line_points[0])) else: result.append((local[lastl:lpos], remote[lastr:rpos])) result.extend(local[lpos:lpos + matchlen]) result_line_points.append((local_line_points[lastl:lpos + 1], remote_line_points[lastr:rpos + 1])) result_line_points.append(None) for i in xrange(1, matchlen): result_line_points.append(selector.select(local_line_points[lpos + i], remote_line_points[rpos + i])) lastl, lastr = lpos + matchlen, rpos + matchlen result_points = selector.dump() return (result, result_line_points, result_points) def test_conflict(): # conflict at begin ## local zero ### local wins assert _find_conflict(['aaa', 'bbb'], ['b', 'a', 'a'], ['a', 'b', 'c'], ['x', 'aaa', 'bbb'], ['a', 'c', 'a', 'a'], ['a', 'c'])[:2] == (['aaa', 'bbb'], ['b', 'a', 'a']) ### remote wins #### remote violate at begin assert _find_conflict(['aaa', 'bbb'], ['a', 'a', 'a'], ['a'], ['x', 'y', 'aaa', 'bbb'], ['b', 'a', 'a', 'a', 'a'], ['a', 'b'])[:2] == (['x', 'y', 'aaa', 'bbb'], ['b', 'a', 'a', 'a', 'a']) #### remote violate at end assert _find_conflict(['aaa', 'bbb'], ['a', 'a', 'a'], ['a'], ['x', 'y', 'aaa', 'bbb'], ['a', 'a', 'b', 'a', 'a'], ['a', 'b'])[:2] == (['x', 'y', 'aaa', 'bbb'], ['a', 'a', 'b', 'a', 'a']) ### neither win #### remote violate at begin assert _find_conflict(['aaa', 'bbb'], ['c', 'a', 'a'], ['a', 'c'], ['x', 'y', 'aaa', 'bbb'], ['b', 'a', 'a', 'a', 'a'], ['a', 'b'])[:2] == ([([], ['x', 'y']), 'aaa', 'bbb'], [(['c'], ['b', 'a', 'a']), None, 'a', 'a']) #### remote violate at end assert _find_conflict(['aaa', 'bbb'], ['c', 'a', 'a'], ['a', 'c'], ['x', 'y', 'aaa', 'bbb'], ['a', 'a', 'b', 'a', 'a'], ['a', 'b'])[:2] == ([([], ['x', 'y']), 'aaa', 'bbb'], [(['c'], ['a', 'a', 'b']), None, 'a', 'a']) ## remote zero ### local wins assert _find_conflict(['x', 'aaa', 'bbb'], ['a', 'b', 'a', 'a'], ['a', 'b'], ['aaa', 'bbb'], ['a', 'a', 'a'], ['a'])[:2] == (['x', 'aaa', 'bbb'], ['a', 'b', 'a', 'a']) ### remote wins assert _find_conflict(['x', 'aaa', 'bbb'], ['a', 'b', 'a', 'a'], ['a', 'b'], ['aaa', 'bbb'], ['c', 'a', 'a'], ['a', 'b', 'c'])[:2] == (['aaa', 'bbb'], ['c', 'a', 'a']) ### neither win #### local violate at begin assert _find_conflict(['x', 'y', 'aaa', 'bbb'], ['b', 'a', 'a', 'a', 'a'], ['a', 'b'], ['aaa', 'bbb'], ['c', 'a', 'a'], ['a', 'c'])[:2] == ([(['x', 'y'], []), 'aaa', 'bbb'], [(['b', 'a', 'a'], ['c']), None, 'a', 'a']) #### local violate at end assert _find_conflict(['x', 'y', 'aaa', 'bbb'], ['a', 'a', 'b', 'a', 'a'], ['a', 'b'], ['aaa', 'bbb'], ['c', 'a', 'a'], ['a', 'c'])[:2] == ([(['x', 'y'], []), 'aaa', 'bbb'], [(['a', 'a', 'b'], ['c']), None, 'a', 'a']) ## neither zero ### local wins assert _find_conflict(['x', 'y', 'aaa', 'bbb'], ['a', 'b', 'a', 'a', 'a'], ['a', 'b'], ['z', 'w', 'aaa', 'bbb'], ['a', 'a', 'a', 'a', 'a'], ['a'])[:2] == (['x', 'y', 'aaa', 'bbb'], ['a', 'b', 'a', 'a', 'a']) ### remote wins #### violates at beginning assert _find_conflict(['x', 'y', 'aaa', 'bbb'], ['a', 'a', 'a', 'a', 'a'], ['a'], ['z', 'w', 'aaa', 'bbb'], ['b', 'a', 'a', 'a', 'a'], ['a', 'b'])[:2] == (['z', 'w', 'aaa', 'bbb'], ['b', 'a', 'a', 'a', 'a']) #### violates at end assert _find_conflict(['x', 'y', 'aaa', 'bbb'], ['a', 'a', 'a', 'a', 'a'], ['a'], ['z', 'w', 'aaa', 'bbb'], ['a', 'a', 'b', 'a', 'a'], ['a', 'b'])[:2] == (['z', 'w', 'aaa', 'bbb'], ['a', 'a', 'b', 'a', 'a']) ### neither win #### violates at begin assert _find_conflict(['x', 'y', 'aaa', 'bbb'], ['c', 'a', 'a', 'a', 'a'], ['a', 'c'], ['z', 'w', 'aaa', 'bbb'], ['b', 'a', 'a', 'a', 'a'], ['a', 'b'])[:2] == ([(['x', 'y'], ['z', 'w']), 'aaa', 'bbb'], [(['c', 'a', 'a'], ['b', 'a', 'a']), None, 'a', 'a']) #### violates at end assert _find_conflict(['x', 'y', 'aaa', 'bbb'], ['a', 'a', 'c', 'a', 'a'], ['a', 'c'], ['z', 'w', 'aaa', 'bbb'], ['a', 'a', 'b', 'a', 'a'], ['a', 'b'])[:2] == ([(['x', 'y'], ['z', 'w']), 'aaa', 'bbb'], [(['a', 'a', 'c'], ['a', 'a', 'b']), None, 'a', 'a']) # conflict in middle ## local zero ### local wins assert _find_conflict(['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'a', 'b', 'a', 'a'], ['a', 'b', 'c'], ['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'c', 'c', 'a', 'a'], ['a', 'c'])[:2] == (['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'a', 'b', 'a', 'a']) ### remote wins #### violate at begin assert _find_conflict(['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'a', 'b', 'a', 'a'], ['a', 'b'], ['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'c', 'a', 'a', 'a'], ['a', 'b', 'c'])[:2] == (['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'c', 'a', 'a', 'a']) #### violate at end assert _find_conflict(['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'a', 'b', 'a', 'a'], ['a', 'b'], ['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'a', 'c', 'a', 'a'], ['a', 'b', 'c'])[:2] == (['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'a', 'c', 'a', 'a']) ### neither win #### remote violate at begin assert _find_conflict(['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'a', 'b', 'a', 'a'], ['a', 'b'], ['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'c', 'a', 'a', 'a'], ['a', 'c'])[:2] == (['aaa', 'bbb', ([], ['x']), 'ccc', 'ddd'], ['a', 'a', (['b'], ['c', 'a']), None, 'a', 'a']) #### remote violate at end assert _find_conflict(['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'a', 'b', 'a', 'a'], ['a', 'b'], ['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'a', 'c', 'a', 'a'], ['a', 'c'])[:2] == (['aaa', 'bbb', ([], ['x']), 'ccc', 'ddd'], ['a', 'a', (['b'], ['a', 'c']), None, 'a', 'a']) ## remote zero ### local wins assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'x', 'x', 'a', 'a'], ['a', 'x', 'c'], ['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'a', 'c', 'a', 'a'], ['a', 'c'])[:2] == (['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'x', 'x', 'a', 'a']) ### remote wins assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'x', 'x', 'a', 'a'], ['a', 'x'], ['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'a', 'c', 'a', 'a'], ['a', 'x', 'c'])[:2] == (['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'a', 'c', 'a', 'a']) ### neither win #### local violate at begin assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'x', 'a', 'a', 'a'], ['a', 'x'], ['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'a', 'c', 'a', 'a'], ['a', 'c'])[:2] == (['aaa', 'bbb', (['x'], []), 'ccc', 'ddd'], ['a', 'a', (['x', 'a'], ['c']), None, 'a', 'a']) #### local violate at end assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'a', 'x', 'a', 'a'], ['a', 'x'], ['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'a', 'c', 'a', 'a'], ['a', 'c'])[:2] == (['aaa', 'bbb', (['x'], []), 'ccc', 'ddd'], ['a', 'a', (['a', 'x'], ['c']), None, 'a', 'a']) ## neither zero ### local wins assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'x', 'x', 'a', 'a'], ['a', 'x', 'y'], ['aaa', 'bbb', 'y', 'ccc', 'ddd'], ['a', 'a', 'y', 'y', 'a', 'a'], ['a', 'y'])[:2] == (['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'x', 'x', 'a', 'a']) ### remote wins #### violate at begin assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'x', 'x', 'a', 'a'], ['a', 'x'], ['aaa', 'bbb', 'y', 'ccc', 'ddd'], ['a', 'a', 'y', 'a', 'a', 'a'], ['a', 'x', 'y'])[:2] == (['aaa', 'bbb', 'y', 'ccc', 'ddd'], ['a', 'a', 'y', 'a', 'a', 'a']) #### violate at end assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'x', 'x', 'a', 'a'], ['a', 'x'], ['aaa', 'bbb', 'y', 'ccc', 'ddd'], ['a', 'a', 'a', 'y', 'a', 'a'], ['a', 'x', 'y'])[:2] == (['aaa', 'bbb', 'y', 'ccc', 'ddd'], ['a', 'a', 'a', 'y', 'a', 'a']) ### neither win #### violates at begin assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'x', 'a', 'a', 'a'], ['a', 'x'], ['aaa', 'bbb', 'y', 'ccc', 'ddd'], ['a', 'a', 'y', 'a', 'a', 'a'], ['a', 'y'])[:2] == (['aaa', 'bbb', (['x'], ['y']), 'ccc', 'ddd'], ['a', 'a', (['x', 'a'], ['y', 'a']), None, 'a', 'a']) #### violates at end assert _find_conflict(['aaa', 'bbb', 'x', 'ccc', 'ddd'], ['a', 'a', 'a', 'x', 'a', 'a'], ['a', 'x'], ['aaa', 'bbb', 'y', 'ccc', 'ddd'], ['a', 'a', 'a', 'y', 'a', 'a'], ['a', 'y'])[:2] == (['aaa', 'bbb', (['x'], ['y']), 'ccc', 'ddd'], ['a', 'a', (['a', 'x'], ['a', 'y']), None, 'a', 'a']) # conflict at end ## local zero ### local wins assert _find_conflict(['aaa', 'bbb'], ['a', 'a', 'b'], ['a', 'b', 'x'], ['aaa', 'bbb', 'x'], ['a', 'a', 'x', 'x'], ['a', 'x'])[:2] == (['aaa', 'bbb'], ['a', 'a', 'b']) ### remote wins #### violate at begin assert _find_conflict(['aaa', 'bbb'], ['a', 'a', 'b'], ['a', 'b'], ['aaa', 'bbb', 'x'], ['a', 'a', 'x', 'a'], ['a', 'b', 'x'])[:2] == (['aaa', 'bbb', 'x'], ['a', 'a', 'x', 'a']) #### violate at end assert _find_conflict(['aaa', 'bbb'], ['a', 'a', 'b'], ['a', 'b'], ['aaa', 'bbb', 'x'], ['a', 'a', 'a', 'x'], ['a', 'b', 'x'])[:2] == (['aaa', 'bbb', 'x'], ['a', 'a', 'a', 'x']) ### neither win #### remote violate at begin assert _find_conflict(['aaa', 'bbb'], ['a', 'a', 'b'], ['a', 'b'], ['aaa', 'bbb', 'x'], ['a', 'a', 'x', 'a'], ['a', 'x'])[:2] == (['aaa', 'bbb', ([], ['x'])], ['a', 'a', (['b'], ['x', 'a']), None]) #### remote violate at end assert _find_conflict(['aaa', 'bbb'], ['a', 'a', 'b'], ['a', 'b'], ['aaa', 'bbb', 'x'], ['a', 'a', 'a', 'x'], ['a', 'x'])[:2] == (['aaa', 'bbb', ([], ['x'])], ['a', 'a', (['b'], ['a', 'x']), None]) ## remote zero ### local wins assert _find_conflict(['aaa', 'bbb', 'x'], ['a', 'a', 'x', 'x'], ['a', 'b', 'x'], ['aaa', 'bbb'], ['a', 'a', 'b'], ['a', 'b'])[:2] == (['aaa', 'bbb', 'x'], ['a', 'a', 'x', 'x']) ### remote wins assert _find_conflict(['aaa', 'bbb', 'x'], ['a', 'a', 'x', 'x'], ['a', 'x'], ['aaa', 'bbb'], ['a', 'a', 'b'], ['a', 'b', 'x'])[:2] == (['aaa', 'bbb'], ['a', 'a', 'b']) ### neither win #### local violate at begin assert _find_conflict(['aaa', 'bbb', 'x'], ['a', 'a', 'x', 'a'], ['a', 'x'], ['aaa', 'bbb'], ['a', 'a', 'b'], ['a', 'b'])[:2] == (['aaa', 'bbb', (['x'], [])], ['a', 'a', (['x', 'a'], ['b']), None]) #### local violate at end assert _find_conflict(['aaa', 'bbb', 'x'], ['a', 'a', 'a', 'x'], ['a', 'x'], ['aaa', 'bbb'], ['a', 'a', 'b'], ['a', 'b'])[:2] == (['aaa', 'bbb', (['x'], [])], ['a', 'a', (['a', 'x'], ['b']), None]) ## neither zero ### local wins assert _find_conflict(['aaa', 'bbb', 'x'], ['a', 'a', 'x', 'x'], ['a', 'x', 'y'], ['aaa', 'bbb', 'y'], ['a', 'a', 'y', 'y'], ['a', 'y'])[:2] == (['aaa', 'bbb', 'x'], ['a', 'a', 'x', 'x']) ### remote wins assert _find_conflict(['aaa', 'bbb', 'x'], ['a', 'a', 'x', 'x'], ['a', 'x'], ['aaa', 'bbb', 'y'], ['a', 'a', 'y', 'y'], ['a', 'x', 'y'])[:2] == (['aaa', 'bbb', 'y'], ['a', 'a', 'y', 'y']) ### neither win #### violates at begin assert _find_conflict(['aaa', 'bbb', 'x'], ['a', 'a', 'x', 'a'], ['a', 'x'], ['aaa', 'bbb', 'y'], ['a', 'a', 'y', 'a'], ['a', 'y'])[:2] == (['aaa', 'bbb', (['x'], ['y'])], ['a', 'a', (['x', 'a'], ['y', 'a']), None]) #### violates at end assert _find_conflict(['aaa', 'bbb', 'x'], ['a', 'a', 'a', 'x'], ['a', 'x'], ['aaa', 'bbb', 'y'], ['a', 'a', 'a', 'y'], ['a', 'y'])[:2] == (['aaa', 'bbb', (['x'], ['y'])], ['a', 'a', (['a', 'x'], ['a', 'y']), None]) # whole file conflict ## local zero ### local wins assert _find_conflict([], ['a'], ['a', 'b'], ['x'], ['b', 'b'], ['b'])[:2] == ([], ['a']) ### remote wins #### violate at begin assert _find_conflict([], ['a'], ['a'], ['x'], ['b', 'a'], ['a', 'b'])[:2] == (['x'], ['b', 'a']) #### violate at end assert _find_conflict([], ['a'], ['a'], ['x'], ['a', 'b'], ['a', 'b'])[:2] == (['x'], ['a', 'b']) ### neither win #### remote violate at begin assert _find_conflict([], ['a'], ['a', 'c'], ['x'], ['b', 'c'], ['b', 'c'])[:2] == ([([], ['x'])], [(['a'], ['b', 'c']), None]) #### remote violate at end assert _find_conflict([], ['a'], ['a', 'c'], ['x'], ['c', 'b'], ['b', 'c'])[:2] == ([([], ['x'])], [(['a'], ['c', 'b']), None]) ## remote zero ### local wins assert _find_conflict(['a'], ['a', 'a'], ['a', 'b'], [], ['b'], ['b'])[:2] == (['a'], ['a', 'a']) ### remote wins assert _find_conflict(['a'], ['a', 'a'], ['a'], [], ['b'], ['a', 'b'])[:2] == ([], ['b']) ### neither win #### local violate at begin assert _find_conflict(['a'], ['a', 'c'], ['a', 'c'], [], ['b'], ['b', 'c'])[:2] == ([(['a'], [])], [(['a', 'c'], ['b']), None]) #### local violate at end assert _find_conflict(['a'], ['c', 'a'], ['a', 'c'], [], ['b'], ['b', 'c'])[:2] == ([(['a'], [])], [(['c', 'a'], ['b']), None]) ## neither zero ### local wins assert _find_conflict(['a'], ['a', 'a'], ['a', 'b'], ['b'], ['b', 'b'], ['b'])[:2] == (['a'], ['a', 'a']) ### remote wins assert _find_conflict(['a'], ['a', 'a'], ['a'], ['b'], ['b', 'b'], ['a', 'b'])[:2] == (['b'], ['b', 'b']) ### neither win #### violate at begin assert _find_conflict(['a'], ['a', 'c'], ['a', 'c'], ['b'], ['b', 'c'], ['c', 'b'])[:2] == ([(['a'], ['b'])], [(['a', 'c'], ['b', 'c']), None]) #### violate at end assert _find_conflict(['a'], ['c', 'a'], ['a', 'c'], ['b'], ['c', 'b'], ['c', 'b'])[:2] == ([(['a'], ['b'])], [(['c', 'a'], ['c', 'b']), None]) # XXX: is this still relevant? # union counts for either #assert _find_conflict(['aaa', 'bbb'], [['a'], ['a', 'b'], ['a']], ['a', 'b'], ['ccc', 'ddd'], [['a'], ['c'], ['a']], ['a', 'c']) == (['ccc', 'ddd'], [['a'], ['c'], ['a']]) # XXX: need more tests here # coincidental matches ## begin assert _find_conflict(['aaa', 'bbb'], ['b', 'a', 'a'], ['a', 'b'], ['aaa', 'bbb'], ['c', 'a', 'a'], ['a', 'c'])[:2] == (['aaa', 'bbb'], ['c', 'a', 'a']) ## middle assert _find_conflict(['aaa', 'bbb'], ['a', 'b', 'a'], ['a', 'b'], ['aaa', 'bbb'], ['a', 'c', 'a'], ['a', 'c'])[:2] == (['aaa', 'bbb'], ['a', 'c', 'a']) ## end assert _find_conflict(['aaa', 'bbb'], ['a', 'a', 'b'], ['a', 'b'], ['aaa', 'bbb'], ['a', 'a', 'c'], ['a', 'c'])[:2] == (['aaa', 'bbb'], ['a', 'a', 'c']) # multiple conflicts, different resolutions assert _find_conflict(['x', 'aaa', 'bbb', 'y', 'ccc', 'ddd', 'z'], ['x', 'x', 'a', 'y', 'y', 'a', 'z', 'z'], ['a', 'x', 'y', 'z', 'p'], ['p', 'aaa', 'bbb', 'q', 'ccc', 'ddd', 'r'], ['p', 'p', 'a', 'q', 'q', 'a', 'r', 'r'], ['a', 'p', 'q', 'r', 'z'])[:2] == (['x', 'aaa', 'bbb', (['y'], ['q']), 'ccc', 'ddd', 'r'], ['x', 'x', 'a', (['y', 'y'], ['q', 'q']), None, 'a', 'r', 'r']) class NoMoreMatchesException: pass def find_resolution(precursors, resolution): result = [None] * (len(resolution) + 1) ms = [[] for i in xrange(len(precursors))] selector = SelectLinePoint() ms = [] for i in xrange(len(precursors)): ms.append(find_matches(precursors[i][0], resolution)) pre = precursors[i][0] pre_line_points = precursors[i][1] selector.push(precursors[i][2]) for (matchold, matchnew, matchlen) in ms[i]: begina, beginb, mlen = matchold + 1, matchnew + 1, matchlen - 1 if (matchold, matchnew) == (0, 0): (begina, beginb, mlen) = (0, 0, mlen + 1) if (matchold + matchlen, matchnew + matchlen) == (len(pre), len(resolution)): mlen += 1 for j in xrange(mlen): result[beginb + j] = selector.select(result[beginb + j], pre_line_points[begina + j]) # mark each line as to whether we need a newline for it hits = [False] * len(resolution) for i in xrange(len(ms)): for (matchold, matchnew, matchlen) in ms[i]: hits[matchnew:matchnew + matchlen] = [True] * matchlen # this may seem weird, but there's this really annoying ambiguous clean # merge case which sometimes turns up when the match algorithm finds an # "incorrect" match. the same conflicts which the person has just resolved # will happen again on merges through other repositories. this code # puts in new line points to prevent that from happening. if len(precursors) > 1: breaks = [] for i in xrange(len(precursors)): l, lp, p = precursors[i] lastl, lastr = 0, 0 ms[i].append((len(l), len(resolution), 0)) for (lpos, rpos, mlen) in ms[i]: # if it doesn't declare itself the winner, it will cause # conflicts later rapplied = _is_applied(result[lastr:rpos + 1], p) if (lpos, rpos) == (0, 0) or \ (lastl, lastr) == (len(l), len(resolution)): pass elif rapplied: breaks.append((lastr, rpos - lastr)) # fix hits so newlines are created hits[lastr:rpos] = [False] * (rpos - lastr) lastl, lastr = lpos + mlen, rpos + mlen breaks.sort() # make the matches not overlap the breaks for i in xrange(len(ms)): newms = [] cur_match = 0 try: for (bpos, blen) in breaks: lpos, rpos, mlen = ms[i][cur_match] # no overlap, nothing to do while rpos + mlen <= bpos: newms.append(ms[i][cur_match]) cur_match += 1 if cur_match >= len(ms[i]): raise NoMoreMatchesException lpos, rpos, mlen = ms[i][cur_match] # some overlap at the end of the match if rpos < bpos: newms.append((lpos, rpos, bpos - rpos)) # full overlap while rpos + mlen <= bpos + blen: cur_match += 1 if cur_match >= len(ms[i]): raise NoMoreMatchesException lpos, rpos, mlen = ms[i][cur_match] # partial overlap, fix it up for the next iteration if rpos < bpos + blen: adj = bpos + blen - rpos m = (lpos + adj, bpos + blen, mlen - adj) ms[i][cur_match] = m except NoMoreMatchesException: pass # add remaining matches at the end, dropping the dummy newms.extend(ms[i][cur_match:-1]) ms[i] = newms # reset results for (bpos, blen) in breaks: result[bpos:bpos + blen + 1] = [None] * (blen + 1) # figure out what's not covered by matches newlines = [] pos = 0 while pos < len(hits): if hits[pos]: pos += 1 else: n = [] j = pos while j < len(hits) and not hits[j]: n.append(resolution[j]) j += 1 newlines.append((pos, n)) pos = j return (result, ms, newlines) def test_resolve(): assert find_resolution([(['aaa', 'bbb', 'ccc'], ['x', 'y', 'z', 'w'], ['x', 'y', 'z', 'w']), (['aaa', 'bbb', 'ccc'], ['p', 'q', 'z', 'w'], ['p', 'q', 'z', 'w'])], ['aaa', 'bbb', 'ccc'])[0] == ['x', 'y', 'z', 'w'] assert find_resolution([(['aaa', 'bbb', 'qqq'], ['a', 'b', 'c', 'd'], ['a', 'b', 'c', 'd']), (['qqq', 'bbb', 'ccc'], ['e', 'f', 'g', 'h'], ['e', 'f', 'g', 'h'])], ['aaa', 'bbb', 'ccc'])[0] == ['a', 'b', 'g', 'h'] assert find_resolution([(['aaa'], ['a', 'a'], ['a'])], ['bbb'])[0] == [None, None] assert find_resolution([(['aaa'], ['x', 'x'], ['x']), (['aaa'], ['x', 'x'], ['x'])], ['aaa'])[1] == [[(0, 0, 1)], [(0, 0, 1)]] assert find_resolution([(['aaa'], ['x', 'x'], ['x']), (['aaa'], ['x', 'y'], ['x', 'y'])], ['aaa'])[1] == [[(0, 0, 1)], [(0, 0, 1)]] assert find_resolution([(['aaa', 'aaa'], ['x', 'x', 'x'], ['x'])], ['aaa', 'aaa', 'bbb', 'ccc'])[2] == [(2, ['bbb', 'ccc'])] assert find_resolution([(['aaa', 'aaa'], ['x', 'x', 'x'], ['x'])], ['bbb', 'ccc', 'aaa', 'aaa'])[2] == [(0, ['bbb', 'ccc'])] assert find_resolution([(['aaa', 'aaa'], ['x', 'x', 'x'], ['x'])], ['bbb', 'ccc'])[2] == [(0, ['bbb', 'ccc'])] assert find_resolution([(['aaa', 'aaa'], ['a', 'a', 'a'], ['a', 'b']), (['bbb', 'bbb'], ['b', 'b', 'b'], ['a', 'b'])], ['aaa', 'aaa']) == ([None, None, None], [[], []], [(0, ['aaa', 'aaa'])]) assert find_resolution([(['aaa', 'bbb', 'bbb', 'bbb'], ['a', 'a', 'a', 'a', 'a'], ['a', 'b']), (['bbb', 'bbb', 'bbb', 'ccc'], ['b', 'b', 'b', 'b', 'b'], ['a', 'b'])], ['aaa', 'bbb', 'bbb', 'ccc']) == ([None, None, 'b', None, None], [[(1, 1, 2)], [(1, 1, 2)]], [(0, ['aaa']), (3, ['ccc'])]) assert find_resolution([(['aaa', 'aaa', 'aaa'], ['a', 'c', 'a', 'c'], ['a', 'c']), (['aaa', 'aaa', 'aaa'], ['a', 'a', 'b', 'b'], ['a', 'b'])], ['aaa', 'aaa', 'aaa']) == (['a', 'c', 'b', 'c'], [[(0, 0, 3)], [(0, 0, 3)]], []) def replay(precursors, matches, newlines, current_point): resultlength = 0 for i in matches: if not len(i): continue matchold, matchnew, matchlen = i[-1] if matchnew + matchlen > resultlength: resultlength = matchnew + matchlen try: foo = newlines[-1][0] + len(newlines[-1][1]) if foo > resultlength: resultlength = foo except IndexError: pass r = [None] * resultlength rpoints = [None] * (resultlength + 1) selector = SelectLinePoint() delpoints = [0] for i in xrange(len(matches)): selector.push(precursors[i][2]) try: if matches[i][0][0] == 0 and matches[i][0][1] == 0: rpoints[0] = selector.select(rpoints[0], precursors[i][1][0]) except IndexError: pass try: matchold, matchnew, matchlen = matches[i][-1] if matchnew + matchlen == resultlength and matchold + matchlen == len(precursors[i][0]): rpoints[resultlength] = selector.select(rpoints[resultlength], precursors[i][1][-1]) except IndexError: pass for (matchold, matchnew, matchlen) in matches[i]: foo = precursors[i][0] if matchold + matchlen > len(foo): raise MergeError, 'match too long' for j in xrange(matchlen): pos = matchnew + j v = foo[matchold + j] if r[pos] is not None and r[pos] != v: raise MergeError, 'conflicting values' r[pos] = v delpoints.append(matchnew + matchlen) pre_lp = precursors[i][1] for j in xrange(1, matchlen): rpoints[matchnew + j] = selector.select(rpoints[matchnew + j], pre_lp[matchold + j]) points = selector.dump() # fill in line points around new lines for (begin, lines) in newlines: for i in xrange(len(lines)): if r[begin + i] is not None: raise MergeError, 'match covers newline' r[begin + i] = lines[i] for i in xrange(len(lines) + 1): assert rpoints[begin + i] is None rpoints[begin + i] = current_point # fill in line points between matches in case the hunk had no new lines for index in delpoints: if rpoints[index] is None: rpoints[index] = selector.select(rpoints[index], current_point) # sanity checks if None in r: raise MergeError, 'unfilled line' if None in rpoints: raise MergeError, 'unset line point' return (r, rpoints, points) class MergeError(StandardError): pass def test_replay(): assert replay([(['aaa', 'bbb'], ['a', 'b', 'c'], ['a', 'b', 'c'])], [[(0, 0, 2)]], [(2, ['ccc', 'ddd'])], 'z')[:2] == (['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'b', 'z', 'z', 'z']) assert replay([(['aaa', 'bbb'], ['a', 'b', 'c'], ['a', 'b', 'c'])], [[(0, 2, 2)]], [(0, ['ccc', 'ddd'])], 'z')[:2] == (['ccc', 'ddd', 'aaa', 'bbb'], ['z', 'z', 'z', 'b', 'c']) assert replay([(['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'b', 'c', 'd', 'e'], ['a', 'b', 'c', 'd', 'e']), (['bbb', 'ccc', 'ddd', 'eee'], ['f', 'c', 'g', 'h', 'i'], ['f', 'c', 'g', 'h', 'i'])], [[(0, 0, 4)], [(0, 1, 4)]], [], 'z')[:2] == (['aaa', 'bbb', 'ccc', 'ddd', 'eee'], ['a', 'b', 'c', 'g', 'h', 'i']) assert replay([(['aaa', 'bbb', 'ccc', 'ddd', 'eee'], ['a', 'a', 'a', 'a', 'a', 'a'], ['a'])], [[(0, 0, 2), (3, 2, 2)]], [], 'z')[:2] == (['aaa', 'bbb', 'ddd', 'eee'], ['a', 'a', 'z', 'a', 'a']) assert replay([(['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'a', 'a', 'a', 'a'], ['a'])], [[(2, 0, 2)]], [], 'z')[:2] == (['ccc', 'ddd'], ['z', 'a', 'a']) assert replay([(['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'a', 'a', 'a', 'a'], ['a'])], [[(0, 0, 2)]], [], 'z')[:2] == (['aaa', 'bbb'], ['a', 'a', 'z']) def annotate(precursors, matches, newlines, current_point): # figure out how long the result is going to be resultlength = 0 for i in matches: if not len(i): continue matchold, matchnew, matchlen = i[-1] if matchnew + matchlen > resultlength: resultlength = matchnew + matchlen try: lastline = newlines[-1][0] + len(newlines[-1][1]) if lastline > resultlength: resultlength = lastline except IndexError: pass selector = SelectLinePoint() r = [None] * resultlength rpoints = [None] * resultlength for i in xrange(len(matches)): selector.push(precursors[i][2]) for (matchold, matchnew, matchlen) in matches[i]: pre_lines = precursors[i][0] pre_lp = precursors[i][1] for j in xrange(matchlen): new_pos = matchnew + j old_pos = matchold + j # fill in lines from matching sections v = pre_lines[old_pos] if r[new_pos] is not None and r[new_pos] != v: raise MergeError, 'conflicting values' r[new_pos] = v # fill in annotation rpoints[new_pos] = selector.select(rpoints[new_pos], pre_lp[old_pos]) points = selector.dump() # fill in annotation for new lines for (begin, lines) in newlines: for i in xrange(len(lines)): # fill in result lines if r[begin + i] is not None: raise MergeError, 'match covers newline' r[begin + i] = lines[i] # then the current annotation assert rpoints[begin + i] is None rpoints[begin + i] = current_point # sanity checks if None in r: raise MergeError, 'unfilled line' if None in rpoints: raise MergeError, 'unset annotation' return (r, rpoints, points) def find_annotation(precursors, resolution): result = [None] * len(resolution) selector = SelectLinePoint() for i in xrange(len(precursors)): match = find_matches(precursors[i][0], resolution) pre = precursors[i][0] pre_line_points = precursors[i][1] selector.push(precursors[i][2]) for (matchold, matchnew, matchlen) in match: for j in xrange(matchlen): result[matchnew + j] = selector.select(result[matchnew + j], pre_line_points[matchold + j]) return result def test_annotate(): assert annotate([(['aaa', 'bbb'], ['a', 'b'], ['a', 'b'])], [[(0, 0, 2)]], [(2, ['ccc', 'ddd'])], 'z')[:2] == (['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'b', 'z', 'z']) assert annotate([(['aaa', 'bbb'], ['a', 'b'], ['a', 'b'])], [[(0, 2, 2)]], [(0, ['ccc', 'ddd'])], 'z')[:2] == (['ccc', 'ddd', 'aaa', 'bbb'], ['z', 'z', 'a', 'b']) assert annotate([(['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'b', 'c', 'd'], ['a', 'b', 'c', 'd']), (['bbb', 'ccc', 'ddd', 'eee'], ['e', 'c', 'f', 'g'], ['e', 'c', 'f', 'g'])], [[(0, 0, 4)], [(0, 1, 4)]], [], 'z')[:2] == (['aaa', 'bbb', 'ccc', 'ddd', 'eee'], ['a', 'e', 'c', 'f', 'g']) assert annotate([(['aaa', 'bbb', 'ccc', 'ddd', 'eee'], ['a', 'a', 'a', 'a', 'a'], ['a'])], [[(0, 0, 2), (3, 2, 2)]], [], 'z')[:2] == (['aaa', 'bbb', 'ddd', 'eee'], ['a', 'a', 'a', 'a']) assert annotate([(['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'a', 'a', 'a'], ['a'])], [[(2, 0, 2)]], [], 'z')[:2] == (['ccc', 'ddd'], ['a', 'a']) assert annotate([(['aaa', 'bbb', 'ccc', 'ddd'], ['a', 'a', 'a', 'a'], ['a'])], [[(0, 0, 2)]], [], 'z')[:2] == (['aaa', 'bbb'], ['a', 'a']) try: import psyco psyco.bind(replay) except ImportError: pass Codeville-0.8.0/Codeville/mergelcs.py0000744000076400007640000001156210247166126017047 0ustar rcohenrcohen#!/usr/bin/python # Written by Uoti Urpala from array import array def unique_lcs(a, b, alo, ahi, blo, bhi): index = {} for i in xrange(blo, bhi): line = b[i] if line in index: index[line] = None else: index[line] = i s = {} get = index.get for i in xrange(alo, ahi): line = a[i] if get(line) is not None: if line in s: del s[line] del index[line] else: s[line] = index[line] + 1 back = {} values = {} values[0] = 0 matches = {} sz = bhi - blo + 1 while True: sz2 = sz sz |= sz >> 1 if sz == sz2: break sz += 1 tree = array('b', [0] * (2 * sz)) sz2 = sz while sz2 > 0: tree[sz2] = 1 sz2 >>= 1 get = s.get i = alo while i < ahi: line = a[i] i += 1 j = get(line) if j is None: continue l = 1 j2 = i while i < ahi and j < bhi and a[i] == b[j]: if a[i] in s: l += 1 i += 1 j += 1 j -= blo matches[j] = j2 - 1 left = j + sz while True: tree[left] = 1 j2 = left >> 1 if tree[j2]: break left = j2 j2 = left while not left & 1 or not tree[left - 1]: left >>= 1 left -= 1 while not left & sz: left <<= 1 if tree[left + 1]: left += 1 left -= sz back[j] = left l += values[left] values[j] = l while True: while j2 & 1 or not tree[j2 + 1]: j2 >>= 1 if j2 == 0: break j2 += 1 while not j2 & sz: j2 <<= 1 if not tree[j2]: j2 += 1 j = values[j2-sz] if j > l: break while True: tree[j2] = 0 if tree[j2 ^ 1]: break j2 >>= 1 if j == l: break j = 1 while not j & sz: j <<= 1 if tree[j + 1]: j += 1 r = [] j -= sz while j: i = matches[j] j2 = s[a[i]] - 1 r.append((i, j2, j + blo - j2)) j = back[j] r.reverse() return r def recurse(a, b, alo, ahi, blo, bhi, answer, maxrecursion=20): if maxrecursion < 0: return m = unique_lcs(a, b, alo, ahi, blo, bhi) i = alo j = blo for i2, j2, l in m: while i2 > i and j2 > j and a[i2-1] == b[j2-1]: i2 -= 1 j2 -= 1 l += 1 if l < 2: continue if i2 > alo and j2 > blo: recurse(a, b, i, i2, j, j2, answer, maxrecursion - 1) answer.append((i2, j2, l)) i = i2 + l j = j2 + l if i > alo and i < ahi and j < bhi: recurse(a, b, i, ahi, j, bhi, answer, maxrecursion - 1) def x(a, b): import time t1 = time.time() r = unique_lcs(a, b, 0, len(a), 0, len(b)) t2 = time.time() print r for i, j, l in r: assert a[i:i+l] == b[j:j+l] print ''.join(a[i:i+l]), print print 'time:', t2-t1 def x2(a, b): import time t1 = time.time() r = [] recurse(a, b, 0, len(a), 0, len(b), r) t2 = time.time() print r for i, j, l in r: assert a[i:i+l] == b[j:j+l] print ''.join(a[i:i+l]), print print 'time:', t2-t1 def test(x): a = [str(i) for i in range(100000)] import random b = list(a) random.shuffle(b) print ' '.join(b) print x(a, b) #test(x) #x2('0123456789abc', ' 01 89a 34 67') #x2('AB 1 CD 1 AB P XY Q AB 1 CD 1 AB', 'AB 2 CD 2 AB Q XY P AB 2 CD 2 AB') #x2('AjBjC', 'AjC') #x2('01 2', '01') try: import psyco psyco.bind(recurse) except ImportError: pass def find_matches(a, b): r = [] recurse(a, b, 0, len(a), 0, len(b), r) return r if __name__ == '__main__': import sys a = file(sys.argv[1]).readlines() b = file(sys.argv[2]).readlines() r = [] recurse(a, b, 0, len(a), 0, len(b), r) i = 0 j = 0 for i2, j2, l in r: for i in range(i, i2): print '-' + a[i], for j in range(j, j2): print '+' + b[j], if l <= 6: for i in range(i2, i2 + l): print ' ' + a[i], else: if i2 > 0 or j2 > 0: for i in range(i2, i2 + 3): print ' ' + a[i], print '@@ -'+str(i2+l-2)+' +'+str(j2+l-2)+' @@' for i in range(i2 + l - 3, i2 + l): print ' ' + a[i], i = i2 + l j = j2 + l for i in range(i, len(a)): print '-' + a[i], for j in range(j, len(b)): print '+' + b[j], Codeville-0.8.0/Codeville/network.py0000664000076400007640000001314610403371625016734 0ustar rcohenrcohen# Written by Ross Cohen # see LICENSE.txt for license information import binascii import socket import struct Request = 0 Response = 1 class NetworkError(StandardError): pass class NetworkHandler: def __init__(self, handler): self.socket = {} self.hfmt = "!I20s" self.rrfmt = "!BxxxI" self.hlen = struct.calcsize(self.hfmt) self.rrlen = struct.calcsize(self.rrfmt) self.handler = handler self.handler.nh = self self.rs = handler.rs def external_connection_made(self, s): self.socket[s] = {'message': [], 'len': self.hlen, 'hread': 0, 'mid': 0, 'nextid': 0, 'mlen': 0, 'skip': 0, 'req-mode': 0} self.handler.external_connection_made(s) def start_connection(self, dns): try: s = self.rs.start_connection(dns, handler = self) except socket.error, e: raise NetworkError, e[1] self.socket[s] = {'message': [], 'len': self.hlen, 'hread': 0, 'mid': 0, 'nextid': 0, 'mlen': 0, 'skip': 0, 'req-mode': 0} return s def _read_sock(self, socket, length): count, datas = 0, [] skip = socket['skip'] while count < length: data = socket['message'][0] dlength = len(data) count += dlength - skip if count > length: data = data[skip:dlength-(count-length)] skip = dlength-(count-length) else: del socket['message'][0] if skip != 0: data = data[skip:] skip = 0 datas.append(data) socket['mlen'] -= length socket['skip'] = skip retval = ''.join(datas) assert len(retval) == length return retval def _write_sock(self, socket, data): socket['mlen'] += len(data) socket['message'].append(data) def data_came_in(self, s, data): socket = self.socket[s] self._write_sock(socket, data) while socket['mlen'] >= socket['len']: if socket['hread'] == 0: header = self._read_sock(socket, self.hlen) socket['len'], socket['mac'] = struct.unpack(self.hfmt, header) socket['hread'] = 1 else: if socket['req-mode']: header = self._read_sock(socket, self.rrlen) if socket.has_key('h_in'): socket['h_in'].update(header) data = self._read_sock(socket, socket['len'] - self.rrlen) else: data = self._read_sock(socket, socket['len']) if socket.has_key('h_in'): socket['h_in'].update(data) if socket['mac'] != socket['h_in'].digest(): s.close() print binascii.hexlify(socket['mac']) print binascii.hexlify(socket['h_in'].digest()) self.connection_lost(s, 'Bad HMAC') if socket['req-mode']: (rr, mid) = struct.unpack(self.rrfmt, header) if rr == Request: self.handler.request_came_in(s, mid, data) elif rr == Response: self.handler.response_came_in(s, mid, data) else: self.handler.message_came_in(s, data) socket['len'] = self.hlen socket['hread'] = 0 def connection_lost(self, s, msg): self.handler.connection_lost(s, msg) del self.socket[s] def connection_flushed(self, s): if self.socket[s]['req-mode']: self.handler.connection_flushed(s) def _send_msg(self, s, data): socket = self.socket[s] if socket.has_key('h_out'): socket['h_out'].update(data) mac = socket['h_out'].digest() else: mac = '\x00' * 20 return s.write(struct.pack(self.hfmt, len(data), mac) + data) # functions to be called by higher layer def set_hmac(self, s, h_in, h_out): self.socket[s]['h_in'] = h_in self.socket[s]['h_out'] = h_out def get_req_mode(self, s): return self.socket[s]['req-mode'] def req_mode(self, s, mode): self.socket[s]['req-mode'] = mode def send_msg(self, s, data): socket = self.socket[s] if socket['req-mode']: raise NetworkError, 'send_msg called in wrong mode' return self._send_msg(s, data) def send_request(self, s, data): socket = self.socket[s] if not socket['req-mode']: raise NetworkError, 'send_request called in wrong mode' socket['nextid'] += 1 self._send_msg(s, struct.pack(self.rrfmt, Request, socket['nextid']) + data) return socket['nextid'] def send_response(self, s, mid, data): socket = self.socket[s] if not socket['req-mode']: raise NetworkError, 'send_response called in wrong mode' return self._send_msg(s, struct.pack(self.rrfmt, Response, mid) + data) def next_id(self, s): socket = self.socket[s] socket['nextid'] += 1 return socket['nextid'] def close(self, s): s.close() self.socket[s]['mlen'] = 0 del self.socket[s] Codeville-0.8.0/Codeville/passwd.py0000664000076400007640000000671110340213025016531 0ustar rcohenrcohen# Written by Ross Cohen # see LICENSE.txt for license information import binascii from entropy import random_string from os import chmod, fdopen, path, remove, rename, stat import SRP from sys import platform from tempfile import mkstemp if platform != 'win32': from os import lchown class Passwd: def __init__(self, pw_file, create=False, uid=-1): self.users = {} self.pw_file = pw_file self.pw_dir = path.split(pw_file)[0] self.modtime = 0 if create: h = open(self.pw_file, 'a') if uid != -1: lchown(self.pw_file, uid, -1) chmod(self.pw_file, 0600) h.close() self._read() def get(self, user): self._read() if not self.users.has_key(user): raise KeyError info = self.users[user] if not info.has_key('secret'): info['secret'] = random_string(20) return info def add(self, user, password): self._read() if self.users.has_key(user): raise ValueError, 'User exists' s, v = SRP.new_passwd(user, password) self.users[user] = {'s': s, 'v': v} self._dump() def set(self, user, password): self._read() if not self.users.has_key(user): raise ValueError, 'No such user' s, v = SRP.new_passwd(user, password) self.users[user] = {'s': s, 'v': v} self._dump() def define(self, user, v, s): self._read() if not self.users.has_key(user): raise ValueError, 'No such user' self.users[user] = {'s': s, 'v': v} self._dump() def delete(self, user): self._read() try: del self.users[user] except KeyError: raise ValueError, 'No such user' self._dump() def _read(self): h = open(self.pw_file, 'rU') modtime = path.getmtime(self.pw_file) if self.modtime == modtime: return self.modtime = modtime l, used = None, {} while l != '': l = h.readline() if l.strip() == '': continue try: user, salt, verifier, garbage = l.split(':', 3) salt = binascii.unhexlify(salt) except ValueError: print 'garbage in passwd file' raise ValueError if not self.users.has_key(user): self.users[user] = {'s': salt, 'v': long(verifier)} else: self.users[user]['s'] = salt self.users[user]['v'] = long(verifier) used[user] = 1 h.close() # remove the deleted entries for user in self.users.keys(): if not used.has_key(user): del self.users[user] self.modtime = path.getmtime(self.pw_file) def _dump(self): text = [] for user, pw in self.users.items(): text.append('%s:%s:%d:\n' % (user, binascii.hexlify(pw['s']), pw['v'])) text.sort() fd, fname = mkstemp('', 'cdv', self.pw_dir) h = fdopen(fd, 'w') h.writelines(text) h.close() if platform != 'win32': statinfo = stat(self.pw_file) lchown(fname, statinfo.st_uid, statinfo.st_gid) chmod(fname, 0600) if platform == 'win32': remove(self.pw_file) rename(fname, self.pw_file) self.modtime = path.getmtime(self.pw_file) Codeville-0.8.0/Codeville/path.py0000664000076400007640000000267510560007063016200 0ustar rcohenrcohen# Written by Ross Cohen # see LICENSE.txt for license information import os def preserving_rename(src, dest): # preserve whatever permissions the user set try: statinfo = os.stat(dest) os.chmod(src, statinfo.st_mode) # windows doesn't allow atomic renames to overwrite os.remove(dest) except OSError: # file may be missing from the filesystem pass os.rename(src, dest) return def mdir(d): try: os.makedirs(d) except OSError: pass def abspath(rel_path): "Because os.path.abspath on Windows drops '...'" parent_path, filename = os.path.split(rel_path) if filename == '...': return os.path.join(os.path.abspath(parent_path), filename) return os.path.abspath(rel_path) def subpath(local, file): llocal = breakup(os.path.abspath(local)) lfile = breakup(abspath(file)) if lfile[:len(llocal)] != llocal: raise ValueError fname = '' for d in lfile[len(llocal):]: fname = os.path.join(fname, d) return (lfile, fname) def breakup(s): def _breakup(s): if s == '': return [] a, b = os.path.split(s) return _breakup(a) + [b] drive, s = os.path.splitdrive(s) bdrive, bsep = [], [] if drive != '': bdrive = [drive] bsep = [os.sep] if s.startswith(os.sep): bsep = [os.sep] s = s[len(os.sep):] return bdrive + bsep + _breakup(s) Codeville-0.8.0/Codeville/selectpoll.py0000664000076400007640000000515110412020344017373 0ustar rcohenrcohen# Written by Bram Cohen # see LICENSE.txt for license information import os from select import select, error from time import sleep from types import IntType from bisect import bisect POLLIN = 1 POLLOUT = 2 POLLERR = 8 POLLHUP = 16 POLLERR = 32 class poll: def __init__(self): self.rlist = [] self.wlist = [] self.elist = [] def register(self, f, t): if type(f) != IntType: f = f.fileno() if (t & POLLIN) != 0: insert(self.rlist, f) else: remove(self.rlist, f) if (t & POLLOUT) != 0: insert(self.wlist, f) else: remove(self.wlist, f) if (t & POLLERR) != 0: insert(self.elist, f) else: remove(self.elist, f) def unregister(self, f): if type(f) != IntType: f = f.fileno() remove(self.rlist, f) remove(self.wlist, f) remove(self.elist, f) def poll(self, timeout = None): if self.rlist != [] or self.wlist != [] or self.elist != []: r, w, e = select(self.rlist, self.wlist, self.elist, timeout) else: sleep(timeout) return [] result = {} for s in r: result[s] = POLLIN for s in w: res = result.get(s, 0) res |= POLLOUT result[s] = res for s in e: res = result.get(s, 0) res |= POLLERR result[s] = res return result.items() def remove(list, item): i = bisect(list, item) if i > 0 and list[i-1] == item: del list[i-1] def insert(list, item): i = bisect(list, item) if i == 0 or list[i-1] != item: list.insert(i, item) def test_remove(): x = [2, 4, 6] remove(x, 2) assert x == [4, 6] x = [2, 4, 6] remove(x, 4) assert x == [2, 6] x = [2, 4, 6] remove(x, 6) assert x == [2, 4] x = [2, 4, 6] remove(x, 5) assert x == [2, 4, 6] x = [2, 4, 6] remove(x, 1) assert x == [2, 4, 6] x = [2, 4, 6] remove(x, 7) assert x == [2, 4, 6] x = [2, 4, 6] remove(x, 5) assert x == [2, 4, 6] x = [] remove(x, 3) assert x == [] def test_insert(): x = [2, 4] insert(x, 1) assert x == [1, 2, 4] x = [2, 4] insert(x, 3) assert x == [2, 3, 4] x = [2, 4] insert(x, 5) assert x == [2, 4, 5] x = [2, 4] insert(x, 2) assert x == [2, 4] x = [2, 4] insert(x, 4) assert x == [2, 4] x = [2, 3, 4] insert(x, 3) assert x == [2, 3, 4] x = [] insert(x, 3) assert x == [3] Codeville-0.8.0/Codeville/server.py0000664000076400007640000006603310631557377016571 0ustar rcohenrcohen# Written by Ross Cohen # see LICENSE.txt for license information from bencode import bdecode, bencode import binascii from db import db, ChangeDBs, write_format_version, write_rebuild_version from crypt import crypt from entropy import random_string, string_to_long, long_to_string from history import sync_history, is_ancestor from history import roothandle, rootnode from history import read_diff, WriteDiff, write_changeset from history import handles_in_branch, handle_contents_at_point, handle_merge_check from history import handle_name_at_point, handle_last_modified, HistoryError from history import clean_merge_point, dump_changeinfo import hmac from network import Request, Response, NetworkHandler import os from os import makedirs, path from passwd import Passwd from random import randrange from RawServer import RawServer import re import sha import SRP from sys import version_info from threading import Event from time import time, sleep from traceback import print_exc, print_stack assert version_info >= (2,2), "Python 2.2 or higher is required" User = 2 Queue = 3 Flushed = 4 class ServerError(Exception): pass class ServerRepository: def _db_init(self, local, metadata_dir='.cdv', rw=True, init=False): self.conf_path = path.join(local, metadata_dir) flags = 0 if init: os.makedirs(self.conf_path) flags = db.DB_CREATE self.txns = {} cwd = os.getcwd() self.dbenv = None ltxn = None if rw == True: self.dbenv = db.DBEnv() self.dbenv.set_cachesize(0, 8 * 1024 * 1024) self.dbenv.set_lg_bsize(1024 * 1024) self.dbenv.set_get_returns_none(2) self.dbenv.open(self.conf_path, db.DB_CREATE|db.DB_INIT_MPOOL|db.DB_INIT_TXN|db.DB_PRIVATE|db.DB_RECOVER) ltxn = self.txn_begin() else: os.chdir(self.conf_path) flags = db.DB_RDONLY self.lcrepo = db.DB(dbEnv=self.dbenv) self.lcrepo.open('changesets.db', dbtype=db.DB_BTREE, flags=flags, txn=ltxn) self.changesdb = db.DB(dbEnv=self.dbenv) self.changesdb.open('changenums.db', dbtype=db.DB_BTREE, flags=flags, txn=ltxn) self.branchmapdb = db.DB(dbEnv=self.dbenv) self.branchmapdb.open('branchmap.db', dbtype=db.DB_BTREE, flags=flags, txn=ltxn) self.branchdb = db.DB(dbEnv=self.dbenv) self.branchdb.open('branch.db', dbtype=db.DB_RECNO, flags=flags, txn=ltxn) self.staticdb = db.DB(dbEnv=self.dbenv) self.staticdb.open('static.db', dbtype=db.DB_BTREE, flags=flags, txn=ltxn) self.linforepo = db.DB(dbEnv=self.dbenv) self.linforepo.open('info.db', dbtype=db.DB_BTREE, flags=flags, txn=ltxn) self.repolistdb = db.DB(dbEnv=self.dbenv) self.repolistdb.open('repolist.db', dbtype=db.DB_BTREE, flags=flags, txn=ltxn) # open the mini-dags and their indices self.contents = ChangeDBs(self.dbenv, 'content', flags, ltxn) self.names = ChangeDBs(self.dbenv, 'name', flags, ltxn) self.allnamesdb = db.DB(dbEnv=self.dbenv) self.allnamesdb.set_flags(db.DB_DUPSORT) self.allnamesdb.open('allnames.db', dbtype=db.DB_BTREE, flags=flags, txn=ltxn) self.name_cache = {} self.db_cache = {} self.cpath = path.join(self.conf_path, 'contents') # populate the repository if init: root = bencode({'precursors': [], 'handles': {roothandle: {'add': {'type': 'dir'}, 'name': ''}}}) self.lcrepo.put(rootnode, root, txn=ltxn) self.linforepo.put('branchmax', bencode(0), txn=ltxn) try: makedirs(self.cpath) except OSError: pass write_format_version(self.conf_path) write_rebuild_version(self.conf_path) if rw == True: self.txn_commit(ltxn) else: os.chdir(cwd) return def close(self): try: for txn in self.txns.keys(): self.txn_abort(txn) except AttributeError: return self.lcrepo.close() self.changesdb.close() self.branchmapdb.close() self.branchdb.close() self.staticdb.close() self.linforepo.close() self.repolistdb.close() self.contents.close() self.names.close() self.allnamesdb.close() if self.dbenv is not None: self.dbenv.txn_checkpoint() for lfile in self.dbenv.log_archive(): os.remove(path.join(self.dbenv.db_home, lfile)) self.dbenv.close() return def txn_begin(self): txn = self.dbenv.txn_begin() self.txns[txn] = 1 return txn def txn_commit(self, txn): txn.commit() del self.txns[txn] def txn_abort(self, txn): txn.abort() del self.txns[txn] class ServerHandler(ServerRepository): def __init__(self, config): self.socket = {} self.rs = RawServer(Event(), 100, 1000) self.nh = NetworkHandler(self) self.shutdown = Event() self.config = config self.post_commit = [] for pattern, action in self.config.items('post-commit'): try: self.post_commit.append((re.compile(pattern, re.I), action)) except re.error, msg: raise ServerError, 'Bad post-commit pattern \"%s\": %s' % \ (pattern, msg) pw_file = path.join(config.get('control', 'datadir'), 'passwd') self.passwd = Passwd(pw_file) def db_init(self, init=False): local = self.config.get('control', 'datadir') self._db_init(local, init=init) self.file_locks = {} return #def close(self): # # XXX: should send a shutdown response # #for s in self.socket.keys(): # # self._close(s) def external_connection_made(self, s): self.socket[s] = {'state': 0} def _srp_auth(self, s, msg): socket = self.socket[s] pw = socket['pw'] srp = socket['srp'] = {} srp['B'], srp['u'], srp['K'], srp['m'] = SRP.host_begin(msg['user'], msg['A'], pw['s'], pw['v']) srp['A'] = msg['A'] self._send_msg(s, {'s': pw['s'], 'B': srp['B'], 'u': srp['u']}) socket['state'] = 1 def _secret_auth(self, s): socket = self.socket[s] socket['salt'] = random_string(20) self._send_msg(s, {'salt': socket['salt']}) socket['state'] = 4 def message_came_in(self, s, data): socket = self.socket[s] try: msg = bdecode(data) except ValueError: self._send_error(s, None, 'garbage data') self._close(s) return if socket['state'] == 0: try: pw = socket['pw'] = self.passwd.get(msg['user']) except KeyError: self._send_error(s, None, 'Bad user') self._close(s) return socket['user'] = msg['user'] if msg['op'] == 'get hash': self._send_msg(s, {'hash': sha.new('public hash check' + pw['secret']).digest()}) socket['state'] = 3 elif msg['op'] == 'secret auth': self._secret_auth(s) elif msg['op'] == 'srp auth': self._srp_auth(s, msg) else: self._close(s) elif socket['state'] == 1: srp = socket['srp'] if srp['m'].digest() != msg['m']: self._send_error(s, None, 'Bad password') socket['state'] = 3 return auth = SRP.host_authenticator(srp['K'], srp['A'], srp['m'].digest()) self._send_msg(s, {'auth': auth.digest()}) self.nh.set_hmac(s, srp['m'], auth) socket['state'] = 2 elif socket['state'] == 2: srp = socket['srp'] if msg['op'] == 'get secret': secret = socket['pw']['secret'] esecret = crypt(secret, srp['K'])[0] self._send_msg(s, {'secret': esecret}) socket['state'] = 3 elif msg['op'] == 'set password': if socket['user'] == 'anonymous': self._send_error(s, None, 'operation not permitted') self._close(s) return v = string_to_long(crypt(msg['v'], srp['K'])[0]) self.passwd.define(socket['user'], v, msg['s']) self._send_msg(s, {'ok': 1}) self._close(s) elif socket['state'] == 3: if msg['op'] == 'secret auth': self._secret_auth(s) elif msg['op'] == 'srp auth': self._srp_auth(s, msg) else: self._close(s) elif socket['state'] == 4: pw = socket['pw'] if len(msg['salt']) < 20: self._send_error(s, None, 'Bad salt length') self._close(s) return if msg['salt'] + socket['salt'] == socket['salt'] + msg['salt']: self._send_error(s, None, 'Bad salt') self._close(s) return base = 'session key' + pw['secret'] + socket['salt'] + msg['salt'] key = sha.new(base).digest() socket['m_in'] = hmac.new(key, '', sha) base = 'session key' + pw['secret'] + msg['salt'] + socket['salt'] key = sha.new(base).digest() socket['m_out'] = hmac.new(key, '', sha) if msg['auth'] != socket['m_out'].digest(): self._send_error(s, None, 'Bad password') socket['state'] = 3 return self._send_msg(s, {'auth': socket['m_in'].digest()}) self.nh.set_hmac(s, socket['m_in'], socket['m_out']) self._req_mode(s, 1) self.socket[s] = [{}, {}, socket['user'], [], 1] else: self._close(s) def connection_flushed(self, s): queue = self.socket[s][Queue] socket = self.socket[s] socket[Flushed] = 1 while len(queue) and socket[Flushed] == 1: mid, msg = queue.pop(0) self._send_diff(s, mid, msg) def connection_lost(self, s, msg): if self.nh.get_req_mode(s): self._socket_cleanup(s) del self.socket[s] def bind(self, port): self.rs.bind(port, reuse=1) def listen_forever(self): self.rs.listen_forever(self.nh) self.shutdown.set() def request_came_in(self, s, mid, data): try: msg = bdecode(data) except ValueError: self._send_error(s, mid, 'garbage request') return try: self.request_handlers[msg['request']](self, s, mid, msg) except: print_exc() self._close(s) return def response_came_in(self, s, mid, data): try: msg = bdecode(data) except ValueError: self._close(s) return try: rstate = self.socket[s][Request][mid] except KeyError: self._close(s) return try: self.response_handlers[rstate['request']](self, s, mid, msg, rstate) except: print_exc() self._close(s) return # request handlers def _request_get_head(self, s, mid, msg): if not self.repolistdb.has_key(msg['repository']): self._send_error(s, mid, 'repository "' + msg['repository'] + '" does not exist') return resp = {'head': self.repolistdb.get(msg['repository'])} self._send_response(s, mid, resp) def _request_get_change(self, s, mid, msg): resp = {'changeset': self.lcrepo.get(msg['changenum'])} self._send_response(s, mid, resp) def _send_diff(self, s, mid, msg): diff = read_diff(self, msg['handle'], msg['changenum'], None) if diff is not None: self.socket[s][Flushed] = self._send_response(s, mid, {'diff': diff}) else: self._send_error(s, mid, 'No such diff', close=False) return def _request_get_diff(self, s, mid, msg): if self.socket[s][Flushed] == 1: self._send_diff(s, mid, msg) else: self.socket[s][Queue].append((mid, msg)) def _request_commit(self, s, mid, msg): socket = self.socket[s] if socket[User] == 'anonymous': self._send_error(s, mid, 'operation not permitted') return socket[Response][mid] = {'request': 'commit', 'repository': msg['repository'], 'head': msg['changenum'], 'count': 1, 'counts': {}, 'changes': {}, 'requests': {}, 'diffs': {}, 'reqq': [], 'req-outstanding': 0} if not self.repolistdb.has_key(msg['repository']): self._send_error(s, mid, 'repository "' + msg['repository'] + '" does not exist') return lstate = socket[Response][mid] lstate['txn'] = self.txn_begin() lstate['cur head'] = self.repolistdb.get(msg['repository']) if self.lcrepo.has_key(msg['changenum']): try: self._commit_phase_1(s, mid) self._commit_phase_2(s, mid) return except HistoryError, msg: self._commit_fail(s, mid, str(msg)) return rid = self._get_change(s, socket[Response][mid], msg['changenum']) socket[Request][rid] = {'request': 'get change', 'changenum': msg['changenum'], 'ref': mid} def _request_create_repository(self, s, mid, msg): if self.socket[s][User] == 'anonymous': self._send_error(s, mid, 'operation not permitted') return self._create_repo(s, mid, msg['repository']) def _request_destroy_repository(self, s, mid, msg): if self.socket[s][User] == 'anonymous': self._send_error(s, mid, 'operation not permitted') return self._remove_repo(s, mid, msg['repository']) def _request_list_repositories(self, s, mid, msg): self._send_response(s, mid, {'list': self.repolistdb.keys()}) request_handlers = {'get head': _request_get_head, 'get change': _request_get_change, 'get diff': _request_get_diff, 'commit': _request_commit, 'create repository': _request_create_repository, 'destroy repository': _request_destroy_repository, 'list repositories': _request_list_repositories} # response handlers def _response_get_change(self, s, mid, msg, rstate): lstate = self.socket[s][Response][rstate['ref']] changeset = msg['changeset'] del msg['changeset'] if sha.new(changeset).digest() != rstate['changenum']: self._close(s) return write_changeset(self, rstate['changenum'], changeset, lstate['txn']) changeset = bdecode(changeset) lstate['changes'][rstate['changenum']] = changeset for change in changeset['precursors']: if self.lcrepo.has_key(change): continue if lstate['changes'].has_key(change): continue if lstate['requests'].has_key(change): continue rid = self._get_change(s, lstate, change) self.socket[s][Request][rid] = {'request': 'get change', 'changenum': change, 'ref': rstate['ref']} lstate['count'] += 1 lstate['count'] -= 1 del self.socket[s][Request][mid] # record all the diffs we'll need to request diffs = lstate['diffs'] for handle, hinfo in changeset['handles'].items(): if not hinfo.has_key('hash'): continue if not diffs.has_key(handle): diffs[handle] = {} lstate['counts'][handle] = 0 diffs[handle][rstate['changenum']] = 1 lstate['counts'][handle] += 1 changeset = None try: if lstate['count'] == 0: self._commit_phase_1(s, rstate['ref']) if lstate['count'] == 0: self._commit_phase_2(s, rstate['ref']) except HistoryError, msg: self._commit_fail(s, rstate['ref'], str(msg)) return def _response_get_diff(self, s, mid, msg, rstate): lstate = self.socket[s][Response][rstate['ref']] # send out the next one lstate['req-outstanding'] -= 1 self._get_diff(s, rstate['ref']) handle = rstate['handle'] diffs = lstate['diffs'] diffs[handle][rstate['change']] = msg['diff'] del self.socket[s][Request][mid] lstate['counts'][handle] -= 1 if lstate['counts'][handle] == 0: lstate['count'] -= 1 # write out the diffs WD = WriteDiff(self, handle, lstate['txn']) for change, diff in diffs[handle].items(): WD.write(diff, change) WD.close() # XXX: suboptimal change = handle_last_modified(self, self.contents, handle, lstate['cur head'], lstate['txn']) if change is None or is_ancestor(self, change, lstate['head'], lstate['txn']): handle_contents_at_point(self, handle, lstate['head'], lstate['txn'], dcache=diffs[handle]) assert lstate['modified'].has_key(handle) lstate['modified'][handle] = 1 del diffs[handle] try: if lstate['count'] == 0: self._commit_phase_2(s, rstate['ref']) except HistoryError, msg: self._commit_fail(s, rstate['ref'], str(msg)) return response_handlers = {'get change': _response_get_change, 'get diff': _response_get_diff} # helpers def _req_mode(self, s, mode): self.nh.req_mode(s, mode) def _create_repo(self, s, mid, repo): if self.repolistdb.has_key(repo): self._send_error(s, mid, 'repository "' + repo + '" already exists') return txn = self.txn_begin() self.repolistdb.put(repo, rootnode, txn=txn) self.txn_commit(txn) self._send_response(s, mid, {}) def _remove_repo(self, s, mid, repo): if not self.repolistdb.has_key(repo): self._send_error(s, mid, 'repository "' + repo + '" does not exist') return txn = self.txn_begin() self.repolistdb.delete(repo, txn) self.txn_commit(txn) self._send_response(s, mid, {}) def _commit_phase_1(self, s, mid): request = self.socket[s][Response][mid] txn = request['txn'] head = request['cur head'] # if this change is already committed then we have nothing to do if self.branchmapdb.has_key(request['head']) and is_ancestor(self, request['head'], head, None): request['no phase 2'] = 1 return sync_history(self, request['head'], txn, cache=request['changes']) if self.config.getboolean('control', 'backup'): if not is_ancestor(self, request['cur head'], request['head'], txn): raise HistoryError, 'not an incremental backup' point = request['head'] while point != rootnode: if point == request['cur head']: break pinfo = bdecode(self.lcrepo.get(point, txn=txn)) if not clean_merge_point(pinfo): raise HistoryError, 'not an incremental backup' point = pinfo['precursors'][0] modified = handles_in_branch(self, [head], [request['head']], txn)[1] unlocked = self._lock_files(s, mid, modified) # bump the reference count by the locks we don't have request['count'] += len(unlocked) # mark all the diff requests which have to wait until we get the lock request['diff queue'] = {} for handle in unlocked: if request['diffs'].has_key(handle): request['diff queue'][handle] = 1 # request all the related file diffs for handle, changes in request['diffs'].items(): if request['diff queue'].has_key(handle): request['diff queue'][handle] = changes.keys() continue requested = 0 for change in changes.keys(): requested = 1 self._queue_diff(s, change, handle, mid) request['count'] += requested self._get_diff(s, mid) # denote the merge checks we have to do later rmodified = request['modified'] = {} for handle in modified: rmodified[handle] = 0 def _commit_phase_2(self, s, mid): request = self.socket[s][Response][mid] txn = request['txn'] head = request['cur head'] if request.has_key('no phase 2'): self.txn_commit(txn) del self.socket[s][Response][mid] self._send_response(s, mid, {}) return # backup servers don't create clean merge heads req_head = request['head'] if not self.config.getboolean('control', 'backup'): # create new clean merge head changeset = bencode({'precursors': [head, req_head], 'user': self.socket[s][User], 'time': int(time()), 'handles': {}}) new_head = request['new head'] = sha.new(changeset).digest() write_changeset(self, new_head, changeset, txn) else: new_head = request['head'] self.repolistdb.put(request['repository'], new_head, txn=txn) sync_history(self, new_head, txn, cache=request['changes']) del request['changes'] # validate all the files for which we already have the diffs locks = [] for handle, checked in request['modified'].items(): locks.append(handle) # did we already validate it? if checked: continue # if there are diffs then some other checkin verified them # we only need to make sure there aren't any implicit merges handle_merge_check(self, handle, new_head, txn) # complete everything and clean up self.txn_commit(txn) del self.socket[s][Response][mid] self._send_response(s, mid, {}) for handle in locks: self._unlock_file(s, handle) for pattern, action in self.post_commit: if not pattern.search(request['repository']): continue try: afd = os.popen(action, 'w') afd.write(dump_changeinfo(self, new_head, repo=request['repository'])) afd.close() except IOError, msg: print 'Command failed' print 'Command: ' + action print 'Error: ' + str(msg) def _commit_fail(self, s, mid, msg): lstate = self.socket[s][Response][mid] self.txn_abort(lstate['txn']) lstate['txn'] = None self._send_error(s, mid, msg) def _lock_files(self, s, mid, modified): retval = [] lock = (s, mid) for handle in modified: self.file_locks.setdefault(handle, []).append(lock) if len(self.file_locks[handle]) > 1: retval.append(handle) return retval def _force_unlock_files(self, s, mid, handles): lock = (s, mid) for handle in handles: index = self.file_locks[handle].index(lock) self.file_locks[handle].pop(index) if index == 0: self._unlock_wakeup(handle) return def _unlock_file(self, s, handle): lock = self.file_locks[handle].pop(0) assert lock[0] == s self._unlock_wakeup(handle) return def _unlock_wakeup(self, handle): if len(self.file_locks[handle]) == 0: return # make the requests on behalf of the next connection lock = self.file_locks[handle][0] rstate = self.socket[lock[0]][Request][lock[1]] rstate['count'] -= 1 if rstate['count'] == 0: # if it's a clean merge, go ahead and commit if rstate['diff queue'] == {}: try: self._commit_phase_2(lock[0], lock[1]) except HistoryError, msg: self._commit_fail(lock[0], lock[1], str(msg)) return # need the diffs, request them all for handle, changes in rstate['diff queue']: for change in changes: self._queue_diff(lock[0], change, handle, lock[1]) self._get_diff(lock[0], lock[1]) del rstate['diff queue'] return def _get_change(self, s, lstate, change): req = {'request': 'get change', 'changenum': change} lstate['requests'][change] = 1 return self._send_request(s, req) def _queue_diff(self, s, change, handle, mid): rstate = self.socket[s][Response][mid] rstate['reqq'].append((change, handle)) def _get_diff(self, s, mid): rstate = self.socket[s][Response][mid] while len(rstate['reqq']) and rstate['req-outstanding'] <= 20: change, handle = rstate['reqq'].pop(0) req = {'request': 'get diff', 'changenum': change, 'handle': handle} state = {'request': 'get diff', 'ref': mid, 'change': change, 'handle': handle} rid = self._send_request(s, req) self.socket[s][Request][rid] = state rstate['req-outstanding'] += 1 def _send_msg(self, s, msg): return self.nh.send_msg(s, bencode(msg)) def _send_error(self, s, mid, msg, close=True): retval = None if mid is None: retval = self._send_msg(s, {'error': msg}) else: retval = self._send_response(s, mid, {'error': msg}) if close: self._close(s) return retval def _send_request(self, s, data): return self.nh.send_request(s, bencode(data)) def _send_response(self, s, mid, data): return self.nh.send_response(s, mid, bencode(data)) def _socket_cleanup(self, s): for mid, response in self.socket[s][Response].items(): if response['request'] != 'commit': continue if response.has_key('modified'): self._force_unlock_files(s, mid, response['modified'].keys()) if response.has_key('txn') and response['txn'] is not None: self.txn_abort(response['txn']) def _close(self, s): if self.nh.get_req_mode(s): self._socket_cleanup(s) self.nh.close(s) del self.socket[s] #print 'closing socket:' #print_stack() Codeville-0.8.0/Codeville/testtest.py0000664000076400007640000000413710340212676017122 0ustar rcohenrcohen""" A much simpler testing framework than PyUnit tests a module by running all functions in it whose name starts with 'test' a test fails if it raises an exception, otherwise it passes functions are try_all and try_single """ # Written by Bram Cohen # see LICENSE.txt for license information import traceback import sys import types def try_all(excludes = [], excluded_paths=[]): """ tests all imported modules takes an optional list of module names and/or module objects to skip over. modules from files under under any of excluded_paths are also skipped. """ failed = [] for modulename, module in sys.modules.items(): # skip builtins if not hasattr(module, '__file__'): continue # skip modules under any of excluded_paths if [p for p in excluded_paths if module.__file__.startswith(p)]: continue if modulename not in excludes and module not in excludes: try_module(module, modulename, failed) print_failed(failed) def try_single(m): """ tests a single module accepts either a module object or a module name in string form """ if type(m) == types.StringType: modulename = m module = __import__(m) else: modulename = str(m) module = m failed = [] try_module(module, modulename, failed) print_failed(failed) def try_module(module, modulename, failed): if not hasattr(module, '__dict__'): return for n, func in module.__dict__.items(): if not callable(func) or n[:4] != 'test': continue name = modulename + '.' + n try: print 'trying ' + name func() print 'passed ' + name except: traceback.print_exc() failed.append(name) print 'failed ' + name def print_failed(failed): print if len(failed) == 0: print 'everything passed' else: print 'the following tests failed:' for i in failed: print i Codeville-0.8.0/Codeville/upgrade.py0000664000076400007640000002146610340212676016676 0ustar rcohenrcohenimport binascii from Codeville.bencode import bdecode, bencode from Codeville.client_helpers import create_handle, gen_diff from Codeville.DFS import DFS from Codeville.history import sync_history, write_changeset from Codeville.history import roothandle, rootnode from Codeville.history import read_diff, write_diff, write_index from Codeville.history import handle_contents_at_point from Codeville.history import handle_name_at_point from Codeville.history import HistoryError from Codeville.old.history import handle_contents_at_point as old_handle_contents_at_point from Codeville.old.history import handle_name_at_point as old_handle_name_at_point import copy import sha from sys import stdout import zlib class UpgradeRepository: def __init__(self, old_repo, new_repo, txn): self.point_map = {} self.handle_map = {} self.all_old_handles = {} self.old_repo = old_repo self.new_repo = new_repo self.txn = txn return def sort_history(self, handle_list): history_dfs = DFS(self._history_deps, [self.old_repo]) for point in handle_list: history_dfs.search(point) return history_dfs.result() def _history_deps(node, args): co = args[0] cset = bdecode(co.lcrepo.get(node)) cset['precursors'].reverse() return cset['precursors'] _history_deps = staticmethod(_history_deps) def sort_names(self, handles): name_dfs = DFS(self._name_deps, [handles]) for old_handle in handles.keys(): name_dfs.search(old_handle) return name_dfs.result() def _name_deps(node, args): handles = args[0] if handles.has_key(node) and handles[node].has_key('parent'): parent = handles[node]['parent'] if handles.has_key(parent) and handles[parent].has_key('name'): return [parent] return [] _name_deps = staticmethod(_name_deps) def clean_merges(self, UR, dagdb, point): clean_merges = {} handles = [] for handle in UR.all_old_handles.keys(): if not dagdb.has_key(handle + point): continue hinfo = bdecode(dagdb.get(handle + point)) if hinfo.has_key('handle'): continue if len(hinfo['precursors']) <= 1: continue clean_merges[handle] = 1 handles.append(handle) return clean_merges, handles def upgrade(old_repo, new_repo, changes, txn): UR = UpgradeRepository(old_repo, new_repo, txn) for old_handle in old_repo.staticdb.keys(): hinfo = bdecode(old_repo.staticdb.get(old_handle)) if hinfo['type'] == 'file': UR.all_old_handles[old_handle] = hinfo # sort the history ordering = UR.sort_history(changes) # sort again for better dag construction ordering.reverse() ordering = UR.sort_history(ordering) assert rootnode == ordering[0] print "%d changesets to convert" % (len(ordering), ) for point in ordering: new_point = convert_cset(UR, point) stdout.write('.') stdout.flush() return UR def convert_cset(UR, point): indices = {} old_cset = bdecode(UR.old_repo.lcrepo.get(point)) new_cset = {} new_cset['precursors'] = [UR.point_map[pre] for pre in old_cset['precursors']] if old_cset.has_key('time'): new_cset['time'] = old_cset['time'] if old_cset.has_key('user'): new_cset['user'] = old_cset['user'] # some heuristics for comments and whether this was a server change clean_merge = True force_new_cset = False if old_cset.has_key('comment'): clean_merge = False new_cset['comment'] = old_cset['comment'].rstrip() if len(new_cset['comment']): new_cset['comment'] = new_cset['comment'] + '\n' elif point == rootnode: pass elif old_cset['handles'] != {} or len(old_cset['precursors']) != 2: clean_merge = False new_cset['comment'] = '--- comment inserted by cdvupgrade ---\n' # sort the handles handle_list = UR.sort_names(old_cset['handles']) # find implicit clean content merges clean_merges, hl = UR.clean_merges(UR, UR.old_repo.contents.dagdb, point) handle_list.extend(hl) # find implicit clean name merges clean_nmerges, hl = UR.clean_merges(UR, UR.old_repo.names.dagdb, point) handle_list.extend(hl) new_cset['handles'] = handles = {} for old_handle in handle_list: old_hinfo = None try: old_hinfo = old_cset['handles'][old_handle] except KeyError: old_hinfo = {} # not much has changed new_hinfo = copy.copy(old_hinfo) new_handle = None if UR.handle_map.has_key(old_handle): new_handle = UR.handle_map[old_handle] # make name changes explicit if clean_nmerges.has_key(old_handle): name = old_handle_name_at_point(UR.old_repo, old_handle, point, None) new_hinfo['parent'] = name['parent'] new_hinfo['name'] = name['name'] # fixup the parent pointers if old_hinfo.has_key('parent'): new_hinfo['parent'] = UR.handle_map[old_hinfo['parent']] if old_hinfo.has_key('hash') or clean_merges.has_key(old_handle): # figure out what the file is supposed to look like now lines = old_handle_contents_at_point(UR.old_repo, old_handle, point, None)['lines'] # if the file is being added, there are no precursors precursors = [] if new_handle is not None and not old_hinfo.has_key('add'): precursors = new_cset['precursors'] # generate the diff against the new repo dinfo = gen_diff(UR.new_repo, new_handle, precursors, lines, UR.txn) if old_hinfo.has_key('add'): dinfo['add'] = 1 assert dinfo['matches'] == [] if dinfo is not None: diff = bencode(dinfo) new_hinfo['hash'] = sha.new(diff).digest() # if this used to be a clean merge, we have to replace it if not old_cset.has_key(old_handle) or not old_cset[old_handle].has_key('hash'): force_new_cset = True elif new_hinfo.has_key('hash'): del new_hinfo['hash'] # sanity check if new_handle is None: assert old_hinfo.has_key('add') assert old_hinfo['add']['type'] == 'file' # if the file is new, we have to create the handle before writing # the diff if old_hinfo.has_key('add'): nhandle = create_handle(new_cset['precursors'], new_hinfo) assert new_handle is None or new_handle == nhandle new_handle = nhandle UR.handle_map[old_handle] = new_handle # write out the new diff if new_hinfo.has_key('hash'): zdiff = zlib.compress(diff, 6) indices[new_handle] = write_diff(UR.new_repo, new_handle, zdiff, UR.txn) elif old_hinfo.has_key('add'): assert old_hinfo['add']['type'] == 'dir' nhandle = create_handle(new_cset['precursors'], new_hinfo) assert new_handle is None or new_handle == nhandle new_handle = nhandle UR.handle_map[old_handle] = new_handle if new_hinfo != {}: handles[new_handle] = new_hinfo # if it used to be a clean merge, preserve the line of clean merge heads index_point = None if clean_merge and force_new_cset: forced_cset = new_cset forced_cset['comment'] = '--- change created by cdvupgrade ---\n' bforced_cset = bencode(forced_cset) forced_point = sha.new(bforced_cset).digest() UR.new_repo.lcrepo.put(forced_point, bforced_cset, txn=UR.txn) index_point = forced_point new_cset = {'precursors': [forced_cset['precursors'][0], forced_point], 'user': forced_cset['user'], 'time': forced_cset['time'], 'handles': {}} # calculate the new point name and write it out bnew_cset = bencode(new_cset) new_point = sha.new(bnew_cset).digest() UR.new_repo.lcrepo.put(new_point, bnew_cset, txn=UR.txn) UR.point_map[point] = new_point if index_point is None: index_point = new_point # now that we know the new point name, write out the indices for new_handle, index in indices.items(): write_index(UR.new_repo, index_point, new_handle, index, UR.txn) # diff generation depends on history syncing named, modified = sync_history(UR.new_repo, new_point, UR.txn) for new_handle in modified: handle_contents_at_point(UR.new_repo, new_handle, new_point, UR.txn) return new_point Codeville-0.8.0/src/0000775000076400007640000000000010645744720013555 5ustar rcohenrcohenCodeville-0.8.0/src/winrand.c0000664000076400007640000000665010340212676015362 0ustar rcohenrcohen/* -*- C -*- */ /* * Uses Windows CryptoAPI CryptGenRandom to get random bytes * * Distribute and use freely; there are no restrictions on further * dissemination and usage except those imposed by the laws of your * country of residence. This software is provided "as is" without * warranty of fitness for use or suitability for any purpose, express * or implied. Use at your own risk or not at all. * */ /* Author: Mark Moraes */ #include "Python.h" #ifdef MS_WIN32 #define _WIN32_WINNT 0x400 #define WINSOCK #include #include LPVOID PrintWindowsError(char* msg, DWORD error) { LPVOID lpMsgBuf; if (!FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language (LPTSTR) &lpMsgBuf, 0, NULL )) { fprintf(stderr, "FormatMessage had an error when processing this error (%d) and this message (%s)!", error, msg); } fprintf(stderr, "%s: %s\n", msg, lpMsgBuf); LocalFree(lpMsgBuf); } static char winrandom__doc__[] = "winrandom(nbytes): Returns nbytes of random data from Windows CryptGenRandom," "a cryptographically strong pseudo-random generator using system entropy"; static PyObject * winrandom(PyObject *self, PyObject *args) { HCRYPTPROV hcp = 0; int n, nbytes; PyObject *res; char *buf; if (!PyArg_ParseTuple(args, "i", &n)) { return NULL; } /* Just in case char != BYTE */ nbytes = (n * sizeof(char)) / sizeof(BYTE); if (nbytes <= 0) { PyErr_SetString(PyExc_ValueError, "nbytes must be positive number"); return NULL; } if ((buf = (char *) PyMem_Malloc(nbytes)) == NULL) return PyErr_NoMemory(); if (! CryptAcquireContext(&hcp, NULL, NULL, PROV_RSA_FULL, 0)) { // If the last error was a bad keyset, then it might be // because we need to generate a keyset, so we call // CryptAcquireContext again in order to try to create // a key set this time. DWORD lastError = GetLastError(); if (lastError == NTE_BAD_KEYSET) { if (!CryptAcquireContext(&hcp, NULL, NULL, PROV_RSA_FULL, CRYPT_NEWKEYSET)) { lastError = GetLastError(); } else { lastError = 0; } } if (lastError != 0) { PyErr_Format(PyExc_SystemError, "CryptAcquireContext failed, error %i", lastError); PrintWindowsError("CryptAcquireContext failed", lastError); PyMem_Free(buf); return NULL; } } if (! CryptGenRandom(hcp, (DWORD) nbytes, (BYTE *) buf)) { PyErr_Format(PyExc_SystemError, "CryptGenRandom failed, error %i", GetLastError()); PyMem_Free(buf); (void) CryptReleaseContext(hcp, 0); return NULL; } if (! CryptReleaseContext(hcp, 0)) { PyErr_Format(PyExc_SystemError, "CryptReleaseContext failed, error %i", GetLastError()); return NULL; } res = PyString_FromStringAndSize(buf, n); PyMem_Free(buf); return res; } static PyMethodDef WRMethods[] = { {"winrandom", (PyCFunction) winrandom, METH_VARARGS, winrandom__doc__}, {NULL, NULL} /* Sentinel */ }; void initwinrandom() { (void) Py_InitModule("winrandom", WRMethods); } #endif /* MS_WIN32 */ Codeville-0.8.0/LICENSE.txt0000664000076400007640000000274310340212676014607 0ustar rcohenrcohenCopyright (C) 2002-2005 Bram Cohen and Ross Cohen All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Codeville nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Codeville-0.8.0/MANIFEST.in0000664000076400007640000000026010340212676014512 0ustar rcohenrcohenrecursive-include Codeville *.py *.pyd include cdv cdvserver cdvpasswd cdv-agent cdvupgrade LICENSE.txt include cdvserver.conf.sample include src/winrand.c include MANIFEST.in Codeville-0.8.0/cdv0000775000076400007640000004211710412020344013452 0ustar rcohenrcohen#!/usr/bin/env python # Written by Ross Cohen # See LICENSE.txt for license information. # Setup import path in case we can't find codeville in standard places try: import Codeville.db except ImportError: import sys, os.path from os.path import dirname, abspath, join pypath = "lib/python%d.%d/site-packages" % \ (sys.version_info[0], sys.version_info[1]) base = dirname(dirname(abspath(sys.argv[0]))) sys.path[0:0] = [ join(base, pypath) ] # first place to look import binascii import Codeville from Codeville.db import db from Codeville.db import VersionMismatchException from Codeville.db import check_format_version, check_rebuild_version from Codeville.client import cli_init, PathError from Codeville.client import add, delete, rename, revert, edit from Codeville.client import history, status, diff, describe, annotate from Codeville.client import update, commit, set_password, rebuild from Codeville.client import create_repo, remove_repo, list_repos from Codeville.client import find_co, server_to_tuple, Checkout from Codeville.client import cli_is_ancestor, cli_print_mini_dag, cli_print_dag from Codeville.client import cli_construct, cli_heads, cli_last_modified from Codeville.client import cli_handle_to_filename from Codeville.client import term_encoding, text_encoding from Codeville.history import long_id, short_id from Codeville.history import ChangeNotKnown, ChangeNotUnique, ChangeBadRepository from getopt import gnu_getopt, GetoptError import os from os import path from random import seed import signal from sys import argv, exit from time import time, strptime, mktime from traceback import print_exc def run_init(co, opts, args): return cli_init(args) def run_add(co, opts, args): return add(co, args) def run_remove(co, opts, args): return delete(co, args) def run_rename(co, opts, args): return rename(co, args[0], args[1]) def run_revert(co, opts, args): unmod_flag = 0 for (opt, arg) in opts: if opt == '-a': unmod_flag = 1 return revert(co, args, unmod_flag) def run_edit(co, opts, args): by_id = False for (opt, arg) in opts: if opt == '-i': by_id = True return edit(co, args, by_id=by_id) def run_history(co, opts, args): head, limit, skip = None, -1, 0 by_id = False v = 0 for (opt, arg) in opts: if opt == '-h': head = long_id(co, arg) elif opt == '-i': by_id = True elif opt == '-c': try: limit = int(arg) except ValueError: print 'Invalid count: ' + arg return 1 elif opt == '-s': try: skip = int(arg) except ValueError: print 'Invalid skip: ' + arg return 1 if skip < 0: print 'Invalid skip: ' + arg return 1 elif opt == '-v': v = 1 return history(co, head, limit, skip, v, by_id, args) def run_status(co, opts, args): verbose = 0 for (opt, arg) in opts: if opt == '-v': verbose = 1 return status(co, args, verbose) def run_heads(co, opts, args): return cli_heads(co) def run_last_mod(co, opts, args): uhead = None by_id = False for (opt, arg) in opts: if opt == '-h': uhead = arg elif opt == '-i': by_id = True return cli_last_modified(co, args[0], uhead, by_id) def run_diff(co, opts, args): print_new = False rev = [co.repo, 'local'] revnum = 0 for (opt, arg) in opts: if opt == '-r': if revnum == 2: print '-r cannot be used more than twice' return 1 rev[revnum] = arg revnum += 1 elif opt == '-N': print_new = True return diff(co, rev, args, print_new) def run_update(co, opts, args): merge = True for (opt, arg) in opts: if opt == '-d': merge = False if co.repo is None: print 'error - no server specified' return 1 remote = server_to_tuple(co.repo) if remote is None or remote[2] is None: print 'error - Invalid server: ' + co.repo return 1 return update(co, remote, merge=merge) def run_commit(co, opts, args): backup, tstamp, comment, noserver = False, None, None, 0 for (opt, arg) in opts: if opt == '-b': backup = True elif opt == '-D': tstamp = mktime(strptime(arg, '%Y/%m/%d %H:%M:%S %Z')) elif opt == '-m': comment = arg ucomment = comment.decode(term_encoding) comment = ucomment.encode('utf8') elif opt == '-M': try: fd = open(arg, "r") comment = fd.read() fd.close() except IOError, err: print 'error - Cannot read "%s": %s' % (arg, err[1]) return 1 try: ucomment = comment.decode(text_encoding) comment = ucomment.encode('utf8') except UnicodeDecodeError: print "error - Invalid %s characters in comment" % \ (text_encoding,) return 1 elif opt == '-n': noserver = 1 if noserver or co.repo is None: co.repo = None remote = None else: remote = server_to_tuple(co.repo) if remote is None or remote[2] is None: print 'error - Invalid server: ' + co.repo return 1 return commit(co, remote, comment, tstamp=tstamp, backup=backup, files=args) def run_construct(co, opts, args): return cli_construct(co, args[0]) def run_rebuild(co, opts, args): uheads = [] for (opt, arg) in opts: if opt == '-h': uheads.append(arg) return rebuild(co, uheads) def run_is_ancestor(co, opts, args): return cli_is_ancestor(co, args[0], args[1]) def run_print_dag(co, opts, args): return cli_print_dag(co, args) def run_print_mini_dag(co, opts, args): uheads = [] by_id = False for (opt, arg) in opts: if opt == '-h': uheads.append(arg) if opt == '-i': by_id = True return cli_print_mini_dag(co, args[0], uheads, by_id) def run_create(co, opts, args): remote = server_to_tuple(args[0]) if remote is None or remote[2] is None: print 'Invalid server: ' + args[0] return 1 return create_repo(co, remote) def run_destroy(co, opts, args): remote = server_to_tuple(args[0]) if remote is None or remote[2] is None: print 'Invalid server: ' + args[0] return 1 return remove_repo(co, remote) def run_list(co, opts, args): return list_repos(co) def run_set(co, opts, args): txn = co.txn_begin() co.varsdb.put(args[0], args[1], txn=txn) co.txn_commit(txn) print 'Variable set' return 0 def run_unset(co, opts, args): txn = co.txn_begin() try: co.varsdb.delete(args[0], txn=txn) except db.DBNotFoundError: print 'Variable was not set' txn.abort() return 1 co.txn_commit(txn) print 'Variable unset' return 0 def run_show_vars(co, opts, args): for item in co.varsdb.items(): print "%s:\t%s" % item return 0 def run_describe(co, opts, args): dodiff, short, xml = False, False, False for (opt, arg) in opts: if opt == '-d': dodiff = True elif opt == '-s': short = True elif opt == '-x': xml = True if dodiff and xml: print 'error - Only 1 of -x, -d is allowed' return 1 return describe(co, args[0], short, xml, dodiff, args[1:]) def run_password(co, opts, args): return set_password(co) def run_annotate(co, opts, args): rev = 'local' for (opt, arg) in opts: if opt == '-r': rev = arg return annotate(co, rev, args) def run_id_to_name(co, opts, args): head = 'local' for (opt, arg) in opts: if opt == '-h': head = arg return cli_handle_to_filename(co, head, args) options = { "unmodified": ('a', 0, "Only files which were not modified"), "backup": ('b', 0, "Backup existing changesets, don't create a new one"), "count": ('c', 1, " elements to display"), "date": ('D', 1, " indicating changeset creation time"), "diff": ('d', 0, "Display a diff"), "dont-merge": ('d', 0, "Don't merge changes into the workspace"), "head": ('h', 1, "Use as the head"), "id": ('i', 0, "Treat name as a file identifier"), "message": ('m', 1, "Use as the commit message"), "message-file": ('M', 1, "Get commit message from "), "no-network": ('n', 0, "Don't perform any network activity"), "new-files": ('N', 0, "Show diffs for new and deleted files"), "path": ('p', 1, " to client"), "revision": ('r', 1, " or "), "repository": ('R', 1, " for this operation"), "skip": ('s', 1, " number of elements"), "user": ('u', 1, " as whom to perform this operation"), "verbose": ('v', 0, ""), "version": ('V', 0, "Print version information"), "xml": ('x', 0, "Output XML") } # the format is (function, min args, max args, long opts) commands = { 'init': (run_init, 0, 0, []), 'add': (run_add, 1, None, []), 'rename': (run_rename, 2, 2, []), 'remove': (run_remove, 1, None, []), 'revert': (run_revert, 1, None, ["unmodified"]), 'edit': (run_edit, 1, None, ["id"]), 'annotate': (run_annotate, 0, None, ["revision"]), 'diff': (run_diff, 0, None, ["revision", "new-files"]), 'describe': (run_describe, 1, None, ["diff", "xml"]), 'history': (run_history, 0, None, ["head", "id", "count", "skip", "verbose"]), 'status': (run_status, 0, None, ["verbose"]), 'heads': (run_heads, 0, 0, []), 'last-modified': (run_last_mod, 1, 1, ["head", "id"]), 'update': (run_update, 0, 0, ["dont-merge"]), 'commit': (run_commit, 0, None, ["backup", "date", "message", "message-file", "no-network"]), 'construct': (run_construct, 1, 1, []), 'rebuild': (run_rebuild, 0, 0, ["head"]), 'is_ancestor': (run_is_ancestor, 2, 2, []), 'print_mini_dag': (run_print_mini_dag, 1, 1, ["head", "id"]), 'print_dag': (run_print_dag, 0, None, []), 'id_to_name': (run_id_to_name, 1, 1, ["head"]), 'create': (run_create, 1, 1, []), 'destroy': (run_destroy, 1, 1, []), 'list-repos': (run_list, 0, 0, []), 'set': (run_set, 2, 2, []), 'unset': (run_unset, 1, 1, []), 'show-vars': (run_show_vars, 0, 0, []), 'password': (run_password, 0, 0, []), } def run_help(): print 'Valid commands are:\n\t', coms = commands.keys() coms.sort() print '\n\t'.join(coms) def opt_help(opts): for lopt in opts: sopt = options[lopt][0] desc = options[lopt][2] print " -%s, --%s\t%s" % (sopt, lopt, desc) return def subcommand_help(cmd): print "Global options:" opt_help(["path", "repository", "user", "version"]) if cmd is None: return print "\nOptions for '%s':" % (cmd,) opt_help(commands[cmd][3]) return def aggregate_opts(commands): fopts, lfopts = [], [] for key, value in options.items(): if value[1] == 1: fopts.append(value[0] + ':') lfopts.append(key + '=') else: fopts.append(value[0]) lfopts.append(key) return ''.join(fopts), lfopts def convert_to_short_opts(optlist): new_optlist = [] for opt, arg in optlist: if options.has_key(opt[2:]): opt = '-' + options[opt[2:]][0] new_optlist.append((opt, arg)) return new_optlist def check_subcommand_opts(cmd, opts): known_opts = [] command = commands[cmd] for lo in command[3]: known_opts.append('--' + lo) if options.has_key(lo): known_opts.append('-' + options[lo][0]) for opt, arg in opts: if opt not in known_opts: raise GetoptError, "option %s not recognized" % (opt,) return def run(args): seed(time() + os.getpid()) local, ulocal, repo, user = None, None, None, None retval = 1 # get rid of nuisance traceback if os.name != 'nt': signal.signal(signal.SIGPIPE, signal.SIG_DFL) all_opts, all_lopts = aggregate_opts(commands) try: optlist, args = gnu_getopt(args, all_opts, all_lopts) except GetoptError, msg: print "error - %s\n" % (msg.args[0],) # maybe we can find something that looks like a subcommand cmd = None for arg in args: if arg not in commands: continue cmd = arg break subcommand_help(cmd) return 1 # parse the top level options optlist = convert_to_short_opts(optlist) optlist_unhandled = [] for (opt, arg) in optlist: if opt == '-p': ulocal = os.path.abspath(arg) elif opt == '-R': repo = arg elif opt == '-u': user = arg elif opt == '-V': print "cdv, version %s" % (Codeville.version,) return 0 else: optlist_unhandled.append((opt, arg)) optlist = optlist_unhandled # pull the subcommand if len(args) == 0: run_help() return 1 user_cname = args.pop(0) cname_list = [] for cname in commands.keys(): if cname.startswith(user_cname): cname_list.append(cname) if len(cname_list) > 1: print 'Command \'%s\' is not specific enough.' % (user_cname,) print 'Matching commands:\n\t', print '\n\t'.join(cname_list) return 1 elif len(cname_list) == 1: cname = cname_list[0] command = commands[cname] else: print '\'%s\' does not match any commands.' % (user_cname,) run_help() return 1 # check the options passed in against the subcommand try: check_subcommand_opts(cname, optlist) except GetoptError, msg: print 'error - %s\n' % (msg.args[0],) subcommand_help(cname) return 1 # check the proper number of arguments were passed if len(args) < command[2]: print 'Too few arguments to "%s" (requires %s).' % \ (cname, command[2]) return 1 elif command[3] is not None and len(args) > command[3]: print 'Too many arguments to "%s" (max %d).' % \ (cname, command[3]) return 1 # find the metadata directory if ulocal is None: ulocal = os.getcwd() try: local = find_co(ulocal) except PathError, msg: if cname != 'init': print msg return 1 if cname == 'init': if local != None: print 'error - found existing repository at %s' % (local,) return 1 return command[0](None, optlist, [ulocal]) metadata_dir = path.join(local, '.cdv') try: check_format_version(metadata_dir) except VersionMismatchException, versions: print "error - expected version %d does not match repository version %d, you probably need to run cdvupgrade." % versions.args return 1 if cname != 'rebuild': try: check_rebuild_version(metadata_dir) except VersionMismatchException, versions: print "error - auxiliary format %d does not match repository format %d, you need to run 'cdv rebuild'" % versions.args return 1 # run the user's command co = None try: # open the database co = Checkout(local) if repo is None: co.repo = co.varsdb.get('repository') else: co.repo = repo if user is not None: co.user = user retval = command[0](co, optlist, args) except KeyboardInterrupt: print 'Aborting...' except ChangeNotKnown, point: print 'error - %s is not a valid changeset' % (point,) except ChangeNotUnique, (point, changes): changes = [short_id(co, binascii.unhexlify(change)) for change in changes] print 'error - %s matches more than one changeset:\n\t%s' % \ (point, '\n\t'.join(changes)) except ChangeBadRepository, repo: print 'error - %s is not a known repository' % (repo,) except: print_exc() if co is not None: # clean up and shut down if co.txn is not None: co.txn_abort(co.txn) co.close() return retval if __name__ == '__main__': if 0: import hotshot, hotshot.stats prof = hotshot.Profile("cdv.prof") retval = prof.runcall(run, argv[1:]) prof.close() stats = hotshot.stats.load("cdv.prof") stats.strip_dirs() stats.sort_stats('time', 'calls') stats.print_stats() else: retval = run(argv[1:]) #retval = run(argv[1:]) exit(retval) Codeville-0.8.0/cdv-agent0000775000076400007640000001317110340212676014557 0ustar rcohenrcohen#!/usr/bin/env python # Written by Ross Cohen # See LICENSE.txt for license information. try: import Codeville.db except ImportError: import sys, os.path from os.path import dirname, abspath, join pypath = "lib/python%d.%d/site-packages" % \ (sys.version_info[0], sys.version_info[1]) base = dirname(dirname(abspath(sys.argv[0]))) sys.path[0:0] = [ join(base, pypath) ] # first place to look from Codeville.agent import Agent from getopt import getopt, GetoptError import os from os import path from resource import setrlimit, RLIMIT_CORE from signal import alarm, signal from signal import SIGALRM, SIGHUP, SIGPIPE, SIGINT, SIGTERM, SIG_IGN import socket import sys from tempfile import mkdtemp ppid = None agent = None def cleanup_handler(signum, foo): agent.cleanup() sys.exit(2) return def check_parent_exists(signum, foo): if ppid is not None: try: os.kill(ppid, 0) except OSError: cleanup_handler(signum, foo) alarm(10) return def usage(): stderr = sys.stderr print >> stderr, "Usage: %s [options] [command [args ...]]" % sys.argv[0] print >> stderr, "Options:" print >> stderr, " -c Generate C-shell commands on stdout." print >> stderr, " -s Generate Bourne shell commands on stdout" print >> stderr, " -k Kill the current agent." print >> stderr, " -d Debug mode." print >> stderr, " -a socket Bind agent socket to given name." print >> stderr, " -t life Default identity lifetime (seconds) (ignored)." sys.exit(1) def fork_stuff(agent, sock, args, c_flag): pid = os.fork() if pid != 0: sock.close() if len(args) == 0: if c_flag: print 'setenv %s %s' % ('CDV_AGENT_PID', pid) print 'setenv %s %s' % ('CDV_AUTH_SOCK', agent.auth_file) else: print '%s=%s; export %s;' % ('CDV_AGENT_PID', pid, 'CDV_AGENT_PID') print '%s=%s; export %s;' % ('CDV_AUTH_SOCK', agent.auth_file, 'CDV_AUTH_SOCK') print "echo Agent pid %d;" % (pid) sys.exit(0) os.environ['CDV_AGENT_PID'] = str(pid) os.environ['CDV_AUTH_SOCK'] = agent.auth_file try: os.execvp(args[0], args) except OSError, msg: print msg sys.exit(1) os.setsid() os.chdir('/') fd = open('/dev/null', 'r+') os.dup2(fd.fileno(), sys.stdin.fileno()) os.dup2(fd.fileno(), sys.stdout.fileno()) os.dup2(fd.fileno(), sys.stderr.fileno()) # set rlimit to prevent core dumping setrlimit(RLIMIT_CORE, (0, 0)) return def run(args): global ppid global agent auth_path = None auth_file = None os.setegid(os.getgid()) os.setgid(os.getgid()) try: optlist, args = getopt(args, 'a:cdkst:') except GetoptError: usage() c_flag, d_flag, k_flag, s_flag = 0, 0, 0, 0 for (opt, arg) in optlist: if opt == '-a': auth_file = arg elif opt == '-c': if s_flag: usage() c_flag += 1 elif opt == '-d': if d_flag: usage() d_flag += 1 elif opt == '-k': k_flag += 1 elif opt == '-s': if c_flag: usage() s_flag += 1 elif opt == '-t': # XXX: for compatibility pass if len(args) and (c_flag or k_flag or s_flag or d_flag): usage() if len(args) and not c_flag and not s_flag: try: shell = os.environ['SHELL'] except KeyError: pass else: if shell.endswith('csh'): c_flag = 1 if k_flag: try: kpid = int(os.environ['CDV_AGENT_PID']) except KeyError: 'CDV_AGENT_PID is not set, cannot kill agent' sys.exit(1) if kpid < 1: print 'CDV_AGENT_PID=%d, which is not a good PID' % (kpid) sys.exit(1) try: os.kill(kpid, SIGTERM) except OSError, msg: print msg sys.exit(1) if c_flag: print 'unsetenv %s;' % ('CDV_AUTH_SOCK') print 'unsetenv %s;' % ('CDV_AGENT_PID') else: print 'unset %s;' % ('CDV_AUTH_SOCK') print 'unset %s;' % ('CDV_AGENT_PID') print 'echo Agent pid %d killed;' % (kpid) sys.exit(0) agent = Agent() ppid = os.getpid() if auth_file is None: auth_path = mkdtemp('', 'cdv-') auth_file = path.join(auth_path, 'agent.' + str(ppid)) try: sock = agent.listen_sock(auth_path, auth_file) if sock is None: sys.exit(1) except socket.error, reason: print reason[1] sys.exit(1) if d_flag: if c_flag: print 'setenv %s %s' % ('CDV_AUTH_SOCK', agent.auth_file) else: print '%s=%s; export %s;' % ('CDV_AUTH_SOCK', agent.auth_file, 'CDV_AUTH_SOCK') print "echo Agent pid %d;" % (ppid) else: fork_stuff(agent, sock, args, c_flag) # periodically check that parent exists if len(args): signal(SIGALRM, check_parent_exists) alarm(10) if not d_flag: signal(SIGINT, SIG_IGN) signal(SIGPIPE, SIG_IGN) signal(SIGHUP, cleanup_handler) signal(SIGTERM, cleanup_handler) agent.listen() return if __name__ == '__main__': sys.exit(run(sys.argv[1:])) Codeville-0.8.0/cdvpasswd0000775000076400007640000000676010340212676014713 0ustar rcohenrcohen#!/usr/bin/env python # Written by Ross Cohen # See LICENSE.txt for license information try: import Codeville.db except ImportError: import sys, os.path from os.path import dirname, abspath, join pypath = "lib/python%d.%d/site-packages" % \ (sys.version_info[0], sys.version_info[1]) base = dirname(dirname(abspath(sys.argv[0]))) sys.path[0:0] = [ join(base, pypath) ] # first place to look from Codeville.passwd import Passwd from ConfigParser import ConfigParser from getopt import getopt, GetoptError from getpass import getpass from os import getcwd, path from sys import argv, exit, platform def print_help(): print "Valid options are:\n\t-c \n\t-f " print 'Valid commands are: add, set, delete' def run(args): try: optlist, args = getopt(args, 'c:f:') except GetoptError: print 'Bad options' print_help() return 1 noconfig = False if platform == 'win32': noconfig = True else: config_file = '/etc/cdvserver.conf' # do a first pass of the command line to pick up an alternate config for (opt, arg) in optlist: if opt == '-c': config_file = arg if opt == '-f': noconfig = True config = ConfigParser() config.add_section('control') if platform == 'win32': config.set('control', 'datadir', getcwd()) else: config.set('control', 'datadir', '/var/lib/cdvserver') if not noconfig: try: confighandle = open(config_file, 'rU') except IOError, msg: print 'Could not open config file: ' + str(msg) return 1 config.readfp(confighandle) confighandle.close() passwd_file = path.join(config.get('control', 'datadir'), 'passwd') for (opt, arg) in optlist: if opt == '-f': passwd_file = arg if len(args) == 0: print_help() return 1 try: pw = Passwd(passwd_file, create=1) except IOError, msg: print 'Could not read password file: ' + str(msg) return 1 if args[0] == 'add': if len(args) != 2: print 'add takes argument ' return 1 try: pw.get(args[1]) except KeyError: pass else: print 'User exists' return 1 password = getpass('Password: ') conf = getpass('Confirm password: ') if conf != password: print 'Confirmation failed' return 1 pw.add(args[1], password) print 'User added' elif args[0] == 'set': if len(args) != 2: print 'set takes argument ' return 1 try: pw.get(args[1]) except KeyError: print 'No such user' return 1 password = getpass('Password: ') conf = getpass('Confirm password: ') if conf != password: print 'Confirmation failed' return 1 pw.set(args[1], password) print 'Password set' elif args[0] == 'delete': if len(args) != 2: print 'delete takes argument ' return 1 try: pw.delete(args[1]) except ValueError, msg: print str(msg) return 1 print 'User deleted' return 0 else: print 'Unknown command' print_help() return 1 return 0 if __name__ == '__main__': exit(run(argv[1:])) Codeville-0.8.0/cdvserver0000775000076400007640000002015210377417235014717 0ustar rcohenrcohen#!/usr/bin/env python # Written by Ross Cohen # See LICENSE.TXT for license information. try: import Codeville.db except ImportError: import sys, os.path from os.path import dirname, abspath, join pypath = "lib/python%d.%d/site-packages" % \ (sys.version_info[0], sys.version_info[1]) base = dirname(dirname(abspath(sys.argv[0]))) sys.path[0:0] = [ join(base, pypath) ] # first place to look from Codeville.db import VersionMismatchException from Codeville.db import check_format_version, check_rebuild_version from Codeville.db import write_rebuild_version from Codeville.server import ServerHandler, ServerError from Codeville.history import rebuild_from_points, rootnode from ConfigParser import ConfigParser from getopt import getopt, GetoptError import os from os import path from random import seed import signal from sys import argv, exit import sys from time import time from traceback import print_exc sh = None keyboard_interrupt = None def signal_handler(signum, frame): print 'Shutting down...' sh.rs.doneflag.set() def run(args): seed(time() + os.getpid()) if sys.platform == 'win32': daemonize = False noconfig = True else: daemonize = True noconfig = False configfile = '/etc/cdvserver.conf' # parse command line arguments initialize = False rebuild = False try: optlist, args = getopt(args, 'bc:df:il:no:p:ru:') except GetoptError: print """Valid options are: -b -c -d -f -i -l -n -o -p -r -u """ return 1 # do a first pass of the command line to pick up an alternate config for (opt, arg) in optlist: if opt == '-c': configfile = arg elif opt == '-n': noconfig = True config = ConfigParser() # set the defaults config.add_section('control') config.set('control', 'pidfile', '/var/run/cdvserver.pid') config.set('control', 'backup', 'False') config.set('control', 'port', '6601') if sys.platform == 'win32': config.set('control', 'datadir', os.getcwd()) else: config.set('control', 'datadir', '/var/lib/cdvserver') config.set('control', 'logfile', '/var/log/cdvserver.log') config.add_section('post-commit') # next read the config file if not noconfig: try: confighandle = open(configfile, 'r') except IOError, msg: print 'Could not open config file: ' + str(msg) return 1 config.readfp(confighandle) confighandle.close() # finally, override everything else with command line options for (opt, arg) in optlist: if opt == '-b': config.set('control', 'backup', 'True') elif opt == '-d': daemonize = False elif opt == '-f': config.set('control', 'datadir', arg) elif opt == '-i': initialize = True elif opt == '-l': config.set('control', 'logfile', arg) elif opt == '-o': config.set('control', 'port', arg) elif opt == '-p': config.set('control', 'pidfile', arg) elif opt == '-r': rebuild = True elif opt == '-u': config.set('control', 'user', arg) if len(args) != 0: print 'Command takes no arguments' return 1 metadata_dir = path.join(config.get('control', 'datadir'), '.cdv') # create the handler object global sh try: sh = ServerHandler(config) except ServerError, msg: print 'Error - %s' % (msg,) return 1 # make sure we can read the repository format repo_marker = path.join(config.get('control', 'datadir'), 'codeville_repository') if initialize: if path.exists(repo_marker): print config.get('control', 'datadir') + " is already initialized as a repository" return 1 crp = open(repo_marker, 'w') crp.close() sh.db_init(init=initialize) return 0 else: if not path.exists(repo_marker): print config.get('control', 'datadir') + " is not a repository, start with -i to initalize one" return 1 try: check_format_version(metadata_dir) except VersionMismatchException, versions: print "error - expected version %d does not match repository version %d, you probably need to run cdvupgrade." % versions.args return 1 # sanity checks try: int(config.get('control', 'port')) except ValueError: print 'Invalid port %s' % config.get('control', 'port') return 1 # bind the port before forking so that we can report errors try: sh.bind(int(config.get('control', 'port'))) except Exception, msg: print 'Error - ' + str(msg) print 'Exiting...' sh.close() return 1 if daemonize == True: try: loghandle = open(config.get('control', 'logfile'), 'a') except IOError: print 'Could not open log file, exiting...' sh.close() return 1 # send all our output to the log sys.stdout.close() sys.stderr.close() sys.stdout = loghandle sys.stderr = loghandle # open the pid file before forking so we can report errors try: pidhandle = open(config.get('control', 'pidfile'), 'w') except IOError: print 'Could not write pid file, exiting...' sh.close() return 1 # switch to the specified user if config.has_option('control', 'user'): import pwd try: pwe = pwd.getpwnam(config.get('control', 'user')) except KeyError: print 'No such user ' + config.get('control', 'user') + ', exiting...' os.remove(config.get('control', 'pidfile')) sh.close() return 1 os.chown(config.get('control', 'pidfile'), pwe.pw_uid, pwe.pw_gid) os.chown(config.get('control', 'logfile'), pwe.pw_uid, pwe.pw_gid) try: os.setegid(pwe.pw_gid) os.seteuid(pwe.pw_uid) except OSError, msg: print 'Could not switch to user ' + pwe.pw_name + ', exiting...' os.remove(config.get('control', 'pidfile')) sh.close() return 1 sh.db_init() pid = os.fork() if pid: pidhandle.write(str(pid)) pidhandle.close() return 0 pidhandle.close() else: sh.db_init() if rebuild: print 'Rebuilding...' # Don't sync the same point twice, this includes the root node points = sh.repolistdb.values() unique_points = dict.fromkeys(points) if unique_points.has_key(rootnode): del unique_points[rootnode] points = unique_points.keys() points[:0] = [rootnode] txn = sh.txn_begin() rebuild_from_points(sh, points, txn) sh.txn_commit(txn) write_rebuild_version(metadata_dir) else: try: check_rebuild_version(metadata_dir) except VersionMismatchException, versions: print "error - auxiliary format %d does not match repository format %d, you need to start the server with -r to rebuild" % versions.args return 1 retval = 0 print 'Listening for clients...' signal.signal(signal.SIGHUP, signal_handler) signal.signal(signal.SIGPIPE, signal.SIG_IGN) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) try: sh.listen_forever() except: print_exc() retval = 1 sh.close() if daemonize == True: if config.has_option('control', 'user'): os.seteuid(os.getuid()) os.remove(config.get('control', 'pidfile')) return retval if __name__ == '__main__': #import profile #retval = profile.run('run(argv[1:])') retval = run(argv[1:]) exit(retval) Codeville-0.8.0/cdvserver.conf.sample0000664000076400007640000000045510340212676017114 0ustar rcohenrcohen[control] #datadir=/var/lib/cdvserver #pidfile=/var/run/cdvserver.pid #logfile=/var/log/cdvserver.log #backup=False #port=6601 user=cdv [post-commit] # keys are regular expression matching repository names # values are programs which will be sent XML through stdin #.*=cat >> /var/log/cdvcommits.log Codeville-0.8.0/cdvupgrade0000775000076400007640000001054310340212676015033 0ustar rcohenrcohen#!/usr/bin/env python # Written by Ross Cohen # see LICENSE.txt for license information try: import Codeville.db except ImportError: import sys, os.path from os.path import dirname, abspath, join pypath = "lib/python%d.%d/site-packages" % \ (sys.version_info[0], sys.version_info[1]) base = dirname(dirname(abspath(sys.argv[0]))) sys.path[0:0] = [ join(base, pypath) ] # first place to look from Codeville.bencode import bdecode, bencode from Codeville.client import find_co, Checkout, _rebuild_fndb, PathError from Codeville.server import ServerRepository from Codeville.upgrade import upgrade import os from os import path from sys import argv, exit, version_info assert version_info >= (2,3), "Python 2.3 or higher is required" def upgrade_client(repo_dir): print "Looks like we're upgrading a client." try: local = find_co(repo_dir, 'CVILLE') except PathError: print "Couldn't find checkout, aborting." return 1 old_repo = Checkout(local, metadata_dir='CVILLE', rw=False) # since we're creating a new metadata directory, we can upgrade in place new_repo = Checkout(local, init=True) txn = new_repo.txn_begin() UR = upgrade(old_repo, new_repo, old_repo.lcrepo.keys(), txn) # fix up all the client specific dbs for handle, modtime in old_repo.modtimesdb.items(): new_handle = handle if UR.handle_map.has_key(handle): new_handle = UR.handle_map[handle] new_repo.modtimesdb.put(new_handle, modtime, txn=txn) for handle, bhinfo in old_repo.editsdb.items(): hinfo = bdecode(bhinfo) if hinfo.has_key('parent'): hinfo['parent'] = UR.handle_map[hinfo['parent']] new_handle = handle if not hinfo.has_key('add'): new_handle = UR.handle_map[handle] else: old_sinfo = old_repo.staticdb.get(handle) new_repo.staticdb.put(new_handle, old_sinfo, txn=txn) new_repo.allnamesdb.put(hinfo['parent'] + hinfo['name'], new_handle, txn=txn) new_repo.editsdb.put(new_handle, bencode(hinfo), txn=txn) for key, value in old_repo.varsdb.items(): new_repo.varsdb.put(key, value, txn=txn) heads = bdecode(old_repo.linforepo.get('heads')) new_heads = [UR.point_map[point] for point in heads] new_repo.linforepo.put('heads', bencode(new_heads), txn=txn) _rebuild_fndb(new_repo, txn) for key, value in old_repo.linforepo.items(): if not key.startswith('cdv://'): continue new_repo.linforepo.put(key, UR.point_map[value], txn=txn) # all done! close everything down. old_repo.close() new_repo.txn_commit(txn) new_repo.close() print """ The client has been upgraded in place. If things are working, you should delete all the old metadata directory by doing: rm -rf CVILLE/ """ return 0 def upgrade_server(repo_dir): print "Looks like we're upgrading a server." old_repo = ServerRepository() old_repo._db_init(repo_dir, '', rw=False) # since we're creating a new metadata directory, we can upgrade in place new_repo = ServerRepository() new_repo._db_init(repo_dir, init=True) txn = new_repo.txn_begin() UR = upgrade(old_repo, new_repo, old_repo.repolistdb.values(), txn) # write the new repository heads for repo, head in old_repo.repolistdb.items(): new_repo.repolistdb.put(repo, UR.point_map[head], txn=txn) old_repo.close() new_repo.txn_commit(txn) new_repo.close() print """ The server has been upgraded in place. If things are working, you should delete all the old database files by doing: rm -rf *.db log.* contents/ """ return 0 def run(): retval = 0 repo_dir = None if len(argv) < 2: repo_dir = os.getcwd() else: repo_dir = path.abspath(argv[1]) if path.exists(path.join(repo_dir, 'codeville_repository')): retval = upgrade_server(repo_dir) else: retval = upgrade_client(repo_dir) return 0 if __name__ == '__main__': if 0: import hotshot, hotshot.stats prof = hotshot.Profile("cdvupgrade.prof") retval = prof.runcall(run) prof.close() stats = hotshot.stats.load("cdvupgrade.prof") stats.strip_dirs() stats.sort_stats('time', 'calls') stats.print_stats() else: retval = run() exit(retval) Codeville-0.8.0/setup.py0000664000076400007640000000270110645744653014505 0ustar rcohenrcohen#!/usr/bin/python # Written by Ross Cohen # see LICENSE.txt for license information import Codeville from distutils.core import setup import shutil import sys assert sys.version >= '2', "Install Python 2.0 or greater" scripts = ["cdv", "cdvserver", "cdvpasswd", "cdv-agent", "cdvupgrade"] plat_ext = [] data_files = [('share/doc/Codeville-' + Codeville.version, ['LICENSE.txt'])] if sys.platform == 'win32': from distutils.core import Extension if sys.version < '2.4': plat_ext = [Extension("Codeville.winrandom", libraries = ['ws2_32', 'advapi32'], sources = ["src/winrand.c"])] for i in xrange(len(scripts)): shutil.copy(scripts[i], scripts[i] + '.py') scripts[i] = scripts[i] + '.py' #os.copy('bin/winrandom.pyd', 'Codeville/winrandom.pyd') else: data_files[0][1].append('cdvserver.conf.sample') for arg in sys.argv: if arg.find('wininst') >= 0: data_files = [('', ['LICENSE.txt'])] for i in xrange(len(scripts)): shutil.copy(scripts[i], scripts[i] + '.py') scripts[i] = scripts[i] + '.py' setup( name = "Codeville", version = Codeville.version, author = "Ross Cohen", author_email = "rcohen@snurgle.org", url = "http://www.codeville.org/", license = "BSD", packages = ["Codeville", "Codeville/old"], ext_modules = plat_ext, scripts = scripts, data_files = data_files ) Codeville-0.8.0/PKG-INFO0000664000076400007640000000032310645744720014061 0ustar rcohenrcohenMetadata-Version: 1.0 Name: Codeville Version: 0.8.0 Summary: UNKNOWN Home-page: http://www.codeville.org/ Author: Ross Cohen Author-email: rcohen@snurgle.org License: BSD Description: UNKNOWN Platform: UNKNOWN