Commit 41ab46d5 authored by Qijiang Fan's avatar Qijiang Fan

Upstream version 1.4

parent 4d96cae6
repo: f2636cfed11500fdc47d1e3822d8e4a2bd636bf7
node: 0cbf9fd89672e73165e1bb4db1ec8f7f65b95c94
node: 07234759a3f750029ccaa001837d42fa12dd33ee
branch: default
tag: 1.3
tag: 1.4
......@@ -5,3 +5,4 @@
093ae2915b452539b44390ee4ea14987484e1eee 1.1.2
708234ad6c97fb52417e0b46a86c8373e25123a5 1.2
4bbc6bf947f56a92e95a04a27b94a9f72d5482d7 1.2.1
0cbf9fd89672e73165e1bb4db1ec8f7f65b95c94 1.3
include Makefile *.rst
include Makefile hgsubversion/help/*.rst
recursive-include tests *.py *.sh *.svndump *.txt
......@@ -17,11 +17,11 @@ check: check-demandimport check-subvertpy check-swig
check-demandimport:
# verify that hgsubversion loads properly without bindings, but fails
# when actually used
! LC_ALL=C HGSUBVERSION_BINDINGS=none \
! LC_ALL=C HGSUBVERSION_BINDINGS=none HGRCPATH=/dev/null \
hg --config extensions.hgsubversion=./hgsubversion \
version 2>&1 \
| egrep '(^abort:|failed to import extension)'
LC_ALL=C HGSUBVERSION_BINDINGS=none \
LC_ALL=C HGSUBVERSION_BINDINGS=none HGRCPATH=/dev/null \
hg --config extensions.hgsubversion=./hgsubversion \
version --svn 2>&1 \
| egrep '(^abort:|failed to import extension)'
......
......@@ -177,6 +177,8 @@ def extsetup():
def reposetup(ui, repo):
if repo.local():
svnrepo.generate_repo_class(ui, repo)
for tunnel in ui.configlist('hgsubversion', 'tunnels'):
hg.schemes['svn+' + tunnel] = svnrepo
_old_local = hg.schemes['file']
def _lookup(url):
......
......@@ -10,6 +10,18 @@ import svnwrap
import util
import svnexternals
class NeverClosingStringIO(object):
def __init__(self):
self._fp = cStringIO.StringIO()
def __getattr__(self, name):
return getattr(self._fp, name)
def close(self):
# svn 1.7 apply_delta driver now calls close() on passed file
# object which prevent us from calling getvalue() afterwards.
pass
class RevisionData(object):
__slots__ = [
......@@ -331,11 +343,11 @@ class HgEditor(svnwrap.Editor):
if self.current.file in self.current.missing:
return lambda x: None
base = self.current.files[self.current.file]
target = cStringIO.StringIO()
target = NeverClosingStringIO()
self.stream = target
handler = svnwrap.apply_txdelta(base, target)
if not callable(handler): #pragma: no cover
if not callable(handler): # pragma: no cover
raise hgutil.Abort('Error in Subversion bindings: '
'cannot call handler!')
def txdelt_window(window):
......@@ -346,12 +358,12 @@ class HgEditor(svnwrap.Editor):
# window being None means commit this file
if not window:
self.current.files[self.current.file] = target.getvalue()
except svnwrap.SubversionException, e: #pragma: no cover
except svnwrap.SubversionException, e: # pragma: no cover
if e.args[1] == svnwrap.ERR_INCOMPLETE_DATA:
self.current.missing.add(self.current.file)
else: #pragma: no cover
else: # pragma: no cover
raise hgutil.Abort(*e.args)
except: #pragma: no cover
except: # pragma: no cover
print len(base), self.current.file
self._exception_info = sys.exc_info()
raise
......
......@@ -306,6 +306,17 @@ settings:
Set the username or password for accessing Subversion repositories.
``hgsubversion.password_stores``
List of methods to use for storing passwords (similar to the option of the
same name in the subversion configuration files). Default is
``gnome_keyring,keychain,kwallet,windows``. Password stores can be disabled
completely by setting this to an empty value.
.. NOTE::
Password stores are only supported with the SWIG bindings.
``hgsubversion.stupid``
Setting this boolean option to true will force using a slower method for
pulling revisions from Subversion. This method is compatible with servers
......
......@@ -21,7 +21,7 @@ class AuthorMap(dict):
The ui argument is used to print diagnostic messages.
The path argument is the location of the backing store,
typically .hg/authormap.
typically .hg/svn/authors.
'''
self.ui = ui
self.path = path
......@@ -252,41 +252,54 @@ class RevMap(dict):
class FileMap(object):
def __init__(self, repo):
self.ui = repo.ui
VERSION = 1
def __init__(self, ui, path):
'''Initialise a new FileMap.
The ui argument is used to print diagnostic messages.
The path argument is the location of the backing store,
typically .hg/svn/filemap.
'''
self.ui = ui
self.path = path
self.include = {}
self.exclude = {}
filemap = repo.ui.config('hgsubversion', 'filemap')
if filemap and os.path.exists(filemap):
self.load(filemap)
if os.path.isfile(self.path):
self._load()
else:
self._write()
def _rpairs(self, name):
yield '.', name
e = len(name)
while e != -1:
yield name[:e], name[e+1:]
e = name.rfind('/', 0, e)
yield '.', name
def check(self, m, path):
m = getattr(self, m)
for pre, _suf in self._rpairs(path):
if pre not in m:
continue
return m[pre]
return None
if pre in m:
return m[pre]
return -1
def __contains__(self, path):
if len(self.include) and len(path):
if not len(path):
return True
if len(self.include):
inc = self.check('include', path)
elif not len(self.exclude):
return True
else:
inc = path
if len(self.exclude) and len(path):
inc = 0
if len(self.exclude):
exc = self.check('exclude', path)
else:
exc = None
if inc is None or exc is not None:
return False
return True
exc = -1
# respect rule order: newer rules override older
return inc > exc
# Needed so empty filemaps are false
def __len__(self):
......@@ -300,11 +313,20 @@ class FileMap(object):
return
bits = m.strip('e'), path
self.ui.debug('%sing %s\n' % bits)
mapping[path] = path
# respect rule order
mapping[path] = len(self)
if fn != self.path:
f = open(self.path, 'a')
f.write(m + ' ' + path + '\n')
f.close()
def load(self, fn):
self.ui.note('reading file map from %s\n' % fn)
f = open(fn, 'r')
self.load_fd(f, fn)
f.close()
def load_fd(self, f, fn):
for line in f:
if line.strip() == '' or line.strip()[0] == '#':
continue
......@@ -319,6 +341,20 @@ class FileMap(object):
except IndexError:
msg = 'ignoring bad line in filemap %s: %s\n'
self.ui.warn(msg % (fn, line.rstrip()))
def _load(self):
self.ui.note('reading in-repo file map from %s\n' % self.path)
f = open(self.path)
ver = int(f.readline())
if ver != self.VERSION:
print 'filemap too new -- please upgrade'
raise NotImplementedError
self.load_fd(f, self.path)
f.close()
def _write(self):
f = open(self.path, 'w')
f.write('%s\n' % self.VERSION)
f.close()
class BranchMap(dict):
......
......@@ -71,7 +71,7 @@ def convert_rev(ui, meta, svn, r, tbdelta, firstrun):
updateexternals(ui, meta, current)
if current.exception is not None: #pragma: no cover
if current.exception is not None: # pragma: no cover
traceback.print_exception(*current.exception)
raise ReplayException()
if current.missing:
......@@ -194,7 +194,7 @@ def convert_rev(ui, meta, svn, r, tbdelta, firstrun):
raise IOError(errno.ENOENT, 'deleting all files')
# True here meant nuke all files, shouldn't happen with branch closing
if current.emptybranches[branch]: #pragma: no cover
if current.emptybranches[branch]: # pragma: no cover
raise hgutil.Abort('Empty commit to an open branch attempted. '
'Please report this issue.')
......
......@@ -12,41 +12,114 @@ import svnwrap
import svnexternals
import util
# Here is a diff mixing content and property changes in svn >= 1.7
#
# Index: a
# ===================================================================
# --- a (revision 12)
# +++ a (working copy)
# @@ -1,2 +1,3 @@
# a
# a
# +a
#
# Property changes on: a
# ___________________________________________________________________
# Added: svn:executable
# ## -0,0 +1 ##
# +*
class ParseError(Exception):
pass
binary_file_re = re.compile(r'''Index: ([^\n]*)
index_header = r'''Index: ([^\n]*)
=*
Cannot display: file marked as a binary type.''')
property_exec_set_re = re.compile(r'''Property changes on: ([^\n]*)
_*
(?:Added|Name): svn:executable
\+''')
property_exec_removed_re = re.compile(r'''Property changes on: ([^\n]*)
_*
(?:Deleted|Name): svn:executable
-''')
empty_file_patch_wont_make_re = re.compile(r'''Index: ([^\n]*)\n=*\n(?=Index:)''')
any_file_re = re.compile(r'''^Index: ([^\n]*)\n=*\n''', re.MULTILINE)
property_special_set_re = re.compile(r'''Property changes on: ([^\n]*)
_*
(?:Added|Name): svn:special
\+''')
'''
property_special_removed_re = re.compile(r'''Property changes on: ([^\n]*)
property_header = r'''Property changes on: ([^\n]*)
_*
(?:Deleted|Name): svn:special
\-''')
'''
headers_re = re.compile('(?:' + '|'.join([
index_header,
property_header,
]) + ')')
property_special_added = r'''(?:Added|Name): (svn:special)
(?: \+|## -0,0 \+1 ##
\+)'''
property_special_deleted = r'''(?:Deleted|Name): (svn:special)
(?: \-|## -1 \+0,0 ##
\-)'''
property_exec_added = r'''(?:Added|Name): (svn:executable)
(?: \+|## -0,0 \+1 ##
\+)'''
property_exec_deleted = r'''(?:Deleted|Name): (svn:executable)
(?: \-|## -1 \+0,0 ##
\-)'''
properties_re = re.compile('(?:' + '|'.join([
property_special_added,
property_special_deleted,
property_exec_added,
property_exec_deleted,
]) + ')')
class filediff:
def __init__(self, name):
self.name = name
self.diff = None
self.binary = False
self.executable = None
self.symlink = None
self.hasprops = False
def isempty(self):
return (not self.diff and not self.binary and not self.hasprops)
def maybedir(self):
return (not self.diff and not self.binary and self.hasprops
and self.symlink is None and self.executable is None)
def parsediff(diff):
changes = {}
headers = headers_re.split(diff)[1:]
if (len(headers) % 3) != 0:
# headers should be a sequence of (index file, property file, data)
raise ParseError('unexpected diff format')
files = []
for i in xrange(len(headers)/3):
iname, pname, data = headers[3*i:3*i+3]
fname = iname or pname
if fname not in changes:
changes[fname] = filediff(fname)
files.append(changes[fname])
f = changes[fname]
if iname is not None:
if data.strip():
f.binary = data.lstrip().startswith(
'Cannot display: file marked as a binary type.')
if not f.binary and '@@' in data:
# Non-empty diff
f.diff = data
else:
f.hasprops = True
for m in properties_re.finditer(data):
p = m.group(1, 2, 3, 4)
if p[0] or p[1]:
f.symlink = bool(p[0])
elif p[2] or p[3]:
f.executable = bool(p[2])
return files
class BadPatchApply(Exception):
pass
def print_your_svn_is_old_message(ui): #pragma: no cover
def print_your_svn_is_old_message(ui): # pragma: no cover
ui.status("In light of that, I'll fall back and do diffs, but it won't do "
"as good a job. You should really upgrade your server.\n")
......@@ -215,17 +288,12 @@ def diff_branchrev(ui, svn, meta, branch, branchpath, r, parentctx):
if '\0' in d:
raise BadPatchApply('binary diffs are not supported')
files_data = {}
# we have to pull each binary file by hand as a fulltext,
# which sucks but we've got no choice
binary_files = set(binary_file_re.findall(d))
touched_files = set(binary_files)
d2 = empty_file_patch_wont_make_re.sub('', d)
d2 = property_exec_set_re.sub('', d2)
d2 = property_exec_removed_re.sub('', d2)
changed = parsediff(d)
# Here we ensure that all files, including the new empty ones
# are marked as touched. Content is loaded on demand.
touched_files.update(any_file_re.findall(d))
if d2.strip() and len(re.findall('\n[-+]', d2.strip())) > 0:
touched_files = set(f.name for f in changed)
d2 = '\n'.join(f.diff for f in changed if f.diff)
if changed:
files_data = patchrepo(ui, meta, parentctx, cStringIO.StringIO(d2))
for x in files_data.iterkeys():
ui.note('M %s\n' % x)
......@@ -233,23 +301,6 @@ def diff_branchrev(ui, svn, meta, branch, branchpath, r, parentctx):
ui.status('Not using patch for %s, diff had no hunks.\n' %
r.revnum)
exec_files = {}
for m in property_exec_removed_re.findall(d):
exec_files[m] = False
for m in property_exec_set_re.findall(d):
exec_files[m] = True
touched_files.update(exec_files)
link_files = {}
for m in property_special_set_re.findall(d):
# TODO(augie) when a symlink is removed, patching will fail.
# We're seeing that above - there's gotta be a better
# workaround than just bailing like that.
assert m in files_data
link_files[m] = True
for m in property_special_removed_re.findall(d):
assert m in files_data
link_files[m] = False
unknown_files = set()
for p in r.paths:
action = r.paths[p].action
......@@ -273,9 +324,35 @@ def diff_branchrev(ui, svn, meta, branch, branchpath, r, parentctx):
touched_files.update(files_data)
touched_files.update(unknown_files)
# As of svn 1.7, diff may contain a lot of property changes for
# directories. We do not what to include these in our touched
# files list so we try to filter them while minimizing the number
# of svn API calls.
property_files = set(f.name for f in changed if f.maybedir())
property_files.discard('.')
touched_files.discard('.')
branchprefix = (branchpath and branchpath + '/') or branchpath
for f in list(property_files):
if f in parentctx:
continue
# We can be smarter here by checking if f is a subcomponent
# of a know path in parentctx or touched_files. KISS for now.
kind = svn.checkpath(branchprefix + f, r.revnum)
if kind == 'd':
touched_files.discard(f)
copies = getcopies(svn, meta, branch, branchpath, r, touched_files,
parentctx)
# We note binary files because svn's diff format doesn't describe
# what changed, only that a change occurred. This means we'll have
# to pull them as fulltexts from the server outside the diff
# apply.
binary_files = set(f.name for f in changed if f.binary)
exec_files = dict((f.name, f.executable) for f in changed
if f.executable is not None)
link_files = dict((f.name, f.symlink) for f in changed
if f.symlink is not None)
def filectxfn(repo, memctx, path):
if path in files_data and files_data[path] is None:
raise IOError(errno.ENOENT, '%s is deleted' % path)
......@@ -638,7 +715,13 @@ def convert_rev(ui, meta, svn, r, tbdelta, firstrun):
deleted_branches[b] = parentctx.node()
continue
incremental = (meta.revmap.oldest > 0)
# The nullrev check might not be necessary in theory but svn <
# 1.7 failed to diff branch creation so the diff_branchrev()
# path does not support this case with svn >= 1.7. We can fix
# it, or we can force the existing fetch_branchrev() path. Do
# the latter for now.
incremental = (meta.revmap.oldest > 0 and
parentctx.rev() != node.nullrev)
if incremental:
try:
......
......@@ -43,13 +43,16 @@ def verify(ui, repo, args=None, **opts):
srev, branch, branchpath = meta.get_source_rev(ctx=ctx)
branchpath = branchpath[len(svn.subdir.lstrip('/')):]
branchurl = ('%s/%s' % (url, branchpath)).strip('/')
ui.write('verifying %s against r%i\n' % (ctx, srev))
ui.write('verifying %s against %s@%i\n' % (ctx, branchurl, srev))
svnfiles = set()
result = 0
for fn, type in svn.list_files(branchpath, srev):
svndata = svn.list_files(branchpath, srev)
for i, (fn, type) in enumerate(svndata):
util.progress(ui, 'verify', i)
if type != 'f':
continue
svnfiles.add(fn)
......@@ -57,7 +60,11 @@ def verify(ui, repo, args=None, **opts):
if branchpath:
fp = branchpath + '/' + fn
data, mode = svn.get_file(posixpath.normpath(fp), srev)
fctx = ctx[fn]
try:
fctx = ctx[fn]
except error.LookupError:
result = 1
continue
dmatch = fctx.data() == data
mmatch = fctx.flags() == mode
if not (dmatch and mmatch):
......@@ -66,8 +73,16 @@ def verify(ui, repo, args=None, **opts):
hgfiles = set(ctx) - util.ignoredfiles
if hgfiles != svnfiles:
missing = set(hgfiles).symmetric_difference(svnfiles)
ui.write('missing files: %s\n' % (', '.join(missing)))
unexpected = hgfiles - svnfiles
if unexpected:
ui.write('unexpected files:\n')
for f in sorted(unexpected):
ui.write(' %s\n' % f)
missing = svnfiles - hgfiles
if missing:
ui.write('missing files:\n')
for f in sorted(missing):
ui.write(' %s\n' % f)
result = 1
return result
......
......@@ -95,9 +95,9 @@ def diff(ext1, ext2):
class BadDefinition(Exception):
pass
re_defold = re.compile(r'^\s*(.*?)\s+(?:-r\s*(\d+|\{REV\})\s+)?([a-zA-Z]+://.*)\s*$')
re_defnew = re.compile(r'^\s*(?:-r\s*(\d+|\{REV\})\s+)?((?:[a-zA-Z]+://|\^/).*)\s+(\S+)\s*$')
re_scheme = re.compile(r'^[a-zA-Z]+://')
re_defold = re.compile(r'^\s*(.*?)\s+(?:-r\s*(\d+|\{REV\})\s+)?([a-zA-Z+]+://.*)\s*$')
re_defnew = re.compile(r'^\s*(?:-r\s*(\d+|\{REV\})\s+)?((?:[a-zA-Z+]+://|\^/).*)\s+(\S+)\s*$')
re_scheme = re.compile(r'^[a-zA-Z+]+://')
def parsedefinition(line):
"""Parse an external definition line, return a tuple (path, rev, source)
......@@ -443,3 +443,9 @@ if subrepo:
if self._state[1] == 'HEAD':
rev = 'HEAD'
return rev
def basestate(self):
# basestate() was introduced by bcb973abcc0b in 2.2
if self._state[1] == 'HEAD':
return 'HEAD'
return super(svnsubrepo, self).basestate()
......@@ -13,21 +13,20 @@ import maps
import editor
def pickle_atomic(data, file_path, dir=None):
def pickle_atomic(data, file_path):
"""pickle some data to a path atomically.
This is present because I kept corrupting my revmap by managing to hit ^C
during the pickle of that file.
"""
try:
f, path = tempfile.mkstemp(prefix='pickling', dir=dir)
f = os.fdopen(f, 'w')
pickle.dump(data, f)
f.close()
except: #pragma: no cover
raise
f = hgutil.atomictempfile(file_path, 'w+b', 0644)
pickle.dump(data, f)
# Older versions of hg have .rename() instead of .close on
# atomictempfile.
if getattr(hgutil.atomictempfile, 'rename', False):
f.rename()
else:
hgutil.rename(path, file_path)
f.close()
class SVNMeta(object):
......@@ -55,6 +54,7 @@ class SVNMeta(object):
'usebranchnames', True)
branchmap = self.ui.config('hgsubversion', 'branchmap')
tagmap = self.ui.config('hgsubversion', 'tagmap')
filemap = self.ui.config('hgsubversion', 'filemap')
self.branches = {}
if os.path.exists(self.branch_info_file):
......@@ -76,8 +76,7 @@ class SVNMeta(object):
self.repo.ui.setconfig('hgsubversion', 'layout', self._layout)
else:
self._layout = None
pickle_atomic(self.tag_locations, self.tag_locations_file,
self.meta_data_dir)
pickle_atomic(self.tag_locations, self.tag_locations_file)
# ensure nested paths are handled properly
self.tag_locations.sort()
self.tag_locations.reverse()
......@@ -94,8 +93,11 @@ class SVNMeta(object):
if tagmap:
self.tagmap.load(tagmap)
self.filemap = maps.FileMap(self.ui, self.filemap_file)
if filemap:
self.filemap.load(filemap)
self.lastdate = '1970-01-01 00:00:00 -0000'
self.filemap = maps.FileMap(repo)
self.addedtags = {}
self.deletedtags = {}
......@@ -191,6 +193,10 @@ class SVNMeta(object):
def authors_file(self):
return os.path.join(self.meta_data_dir, 'authors')
@property
def filemap_file(self):
return os.path.join(self.meta_data_dir, 'filemap')
@property
def branchmapfile(self):
return os.path.join(self.meta_data_dir, 'branchmap')
......@@ -217,7 +223,7 @@ class SVNMeta(object):
'''Save the Subversion metadata. This should really be called after
every revision is created.
'''
pickle_atomic(self.branches, self.branch_info_file, self.meta_data_dir)
pickle_atomic(self.branches, self.branch_info_file)
def localname(self, path):
"""Compute the local name for a branch located at path.
......
......@@ -21,6 +21,13 @@ from mercurial import util as hgutil
from mercurial import httprepo
import mercurial.repo
try:
from mercurial import phases
phases.public # defeat demand import
except ImportError:
phases = None
import re
import util
import wrappers
import svnwrap
......@@ -76,7 +83,11 @@ def generate_repo_class(ui, repo):
class svnlocalrepo(superclass):
def svn_commitctx(self, ctx):
"""Commits a ctx, but defeats manifest recycling introduced in hg 1.9."""
return self.commitctx(ctxctx(ctx))
hash = self.commitctx(ctxctx(ctx))
if phases is not None and getattr(self, 'pushkey', False):
# set phase to be public
self.pushkey('phases', self[hash].hex(), str(phases.draft), str(phases.public))
return hash
# TODO use newbranch to allow branch creation in Subversion?
@remotesvn
......@@ -107,6 +118,14 @@ class svnremoterepo(mercurial.repo.repository):
raise hgutil.Abort('no Subversion URL specified')
self.path = path
self.capabilities = set(['lookup', 'subversion'])
pws = self.ui.config('hgsubversion', 'password_stores', None)
if pws is not None:
# Split pws at comas and strip neighbouring whitespace (whitespace
# at the beginning and end of pws has already been removed by the
# config parser).
self.password_stores = re.split(r'\s*,\s*', pws)
else:
self.password_stores = None
@propertycache
def svnauth(self):
......@@ -127,7 +146,7 @@ class svnremoterepo(mercurial.repo.repository):
@propertycache
def svn(self):
try:
return svnwrap.SubversionRepo(*self.svnauth)
return svnwrap.SubversionRepo(*self.svnauth, password_stores=self.password_stores)
except svnwrap.SubversionConnectionException, e:
self.ui.traceback()
raise hgutil.Abort(e)
......
......@@ -31,7 +31,7 @@ except ImportError:
def _versionstr(v):
return '.'.join(str(d) for d in v)
if subvertpy.__version__ < subvertpy_required: #pragma: no cover
if subvertpy.__version__ < subvertpy_required: # pragma: no cover
raise ImportError('Subvertpy %s or later required, '
'but %s found'
% (_versionstr(subvertpy_required),
......@@ -165,8 +165,11 @@ class SubversionRepo(object):
This wrapper uses Subvertpy, an alternate set of bindings for Subversion
that's more pythonic and sucks less. See earlier in this file for version
requirements.
Note that password stores do not work, the parameter is only here
to ensure that the API is the same as for the SWIG wrapper.
"""
def __init__(self, url='', username='', password='', head=None):
def __init__(self, url='', username='', password='', head=None, password_stores=None):
parsed = common.parse_url(url, username, password)
# --username and --password override URL credentials
self.username = parsed[0]
......@@ -338,7 +341,7 @@ class SubversionRepo(object):