p4.py
402 lines
| 12.7 KiB
| text/x-python
|
PythonLexer
Frank Kingswood
|
r7823 | # Perforce source for convert extension. | ||
# | ||||
# Copyright 2009, Frank Kingswood <frank@kingswood-consulting.co.uk> | ||||
# | ||||
Martin Geisler
|
r8225 | # This software may be used and distributed according to the terms of the | ||
Matt Mackall
|
r10263 | # GNU General Public License version 2 or any later version. | ||
Frank Kingswood
|
r7823 | |||
timeless
|
r28371 | import marshal | ||
import re | ||||
Yuya Nishihara
|
r29205 | from mercurial.i18n import _ | ||
timeless
|
r28371 | from mercurial import ( | ||
error, | ||||
util, | ||||
) | ||||
Yuya Nishihara
|
r37102 | from mercurial.utils import ( | ||
dateutil, | ||||
Yuya Nishihara
|
r37138 | procutil, | ||
Yuya Nishihara
|
r37102 | stringutil, | ||
) | ||||
Frank Kingswood
|
r7823 | |||
timeless
|
r28371 | from . import common | ||
Frank Kingswood
|
r7823 | |||
Augie Fackler
|
r43346 | |||
Frank Kingswood
|
r7823 | def loaditer(f): | ||
Matt Harbison
|
r44226 | """Yield the dictionary objects generated by p4""" | ||
Frank Kingswood
|
r7823 | try: | ||
while True: | ||||
d = marshal.load(f) | ||||
if not d: | ||||
break | ||||
yield d | ||||
except EOFError: | ||||
pass | ||||
Augie Fackler
|
r43346 | |||
Eugene Baranov
|
r25788 | def decodefilename(filename): | ||
"""Perforce escapes special characters @, #, *, or % | ||||
with %40, %23, %2A, or %25 respectively | ||||
Yuya Nishihara
|
r34133 | >>> decodefilename(b'portable-net45%252Bnetcore45%252Bwp8%252BMonoAndroid') | ||
Eugene Baranov
|
r25788 | 'portable-net45%2Bnetcore45%2Bwp8%2BMonoAndroid' | ||
Yuya Nishihara
|
r34133 | >>> decodefilename(b'//Depot/Directory/%2525/%2523/%23%40.%2A') | ||
Eugene Baranov
|
r25788 | '//Depot/Directory/%25/%23/#@.*' | ||
""" | ||||
Augie Fackler
|
r43347 | replacements = [ | ||
(b'%2A', b'*'), | ||||
(b'%23', b'#'), | ||||
(b'%40', b'@'), | ||||
(b'%25', b'%'), | ||||
] | ||||
Eugene Baranov
|
r25788 | for k, v in replacements: | ||
filename = filename.replace(k, v) | ||||
return filename | ||||
Augie Fackler
|
r43346 | |||
timeless
|
r28371 | class p4_source(common.converter_source): | ||
Matt Harbison
|
r35168 | def __init__(self, ui, repotype, path, revs=None): | ||
Matt Harbison
|
r35141 | # avoid import cycle | ||
from . import convcmd | ||||
Matt Harbison
|
r35168 | super(p4_source, self).__init__(ui, repotype, path, revs=revs) | ||
Frank Kingswood
|
r7823 | |||
Augie Fackler
|
r43347 | if b"/" in path and not path.startswith(b'//'): | ||
Augie Fackler
|
r43346 | raise common.NoRepo( | ||
Augie Fackler
|
r43347 | _(b'%s does not look like a P4 repository') % path | ||
Augie Fackler
|
r43346 | ) | ||
Matt Mackall
|
r7973 | |||
Augie Fackler
|
r43347 | common.checktool(b'p4', abort=False) | ||
Frank Kingswood
|
r7823 | |||
David Soria Parra
|
r30601 | self.revmap = {} | ||
Augie Fackler
|
r43346 | self.encoding = self.ui.config( | ||
Augie Fackler
|
r43347 | b'convert', b'p4.encoding', convcmd.orig_encoding | ||
Augie Fackler
|
r43346 | ) | ||
Matt Mackall
|
r10282 | self.re_type = re.compile( | ||
Gregory Szorc
|
r41678 | br"([a-z]+)?(text|binary|symlink|apple|resource|unicode|utf\d+)" | ||
Augie Fackler
|
r43346 | br"(\+\w+)?$" | ||
) | ||||
Matt Mackall
|
r10282 | self.re_keywords = re.compile( | ||
Gregory Szorc
|
r41678 | br"\$(Id|Header|Date|DateTime|Change|File|Revision|Author)" | ||
Augie Fackler
|
r43346 | br":[^$\n]*\$" | ||
) | ||||
Gregory Szorc
|
r41678 | self.re_keywords_old = re.compile(br"\$(Id|Header):[^$\n]*\$") | ||
Frank Kingswood
|
r7823 | |||
Durham Goode
|
r25748 | if revs and len(revs) > 1: | ||
Augie Fackler
|
r43346 | raise error.Abort( | ||
Augie Fackler
|
r43347 | _( | ||
b"p4 source does not support specifying " | ||||
b"multiple revisions" | ||||
) | ||||
Augie Fackler
|
r43346 | ) | ||
Frank Kingswood
|
r7823 | |||
David Soria Parra
|
r30601 | def setrevmap(self, revmap): | ||
"""Sets the parsed revmap dictionary. | ||||
Revmap stores mappings from a source revision to a target revision. | ||||
It is set in convertcmd.convert and provided by the user as a file | ||||
on the commandline. | ||||
Revisions in the map are considered beeing present in the | ||||
repository and ignored during _parse(). This allows for incremental | ||||
imports if a revmap is provided. | ||||
""" | ||||
self.revmap = revmap | ||||
Frank Kingswood
|
r7823 | def _parse_view(self, path): | ||
Matt Harbison
|
r44226 | """Read changes affecting the path""" | ||
Augie Fackler
|
r43347 | cmd = b'p4 -G changes -s submitted %s' % procutil.shellquote(path) | ||
stdout = procutil.popen(cmd, mode=b'rb') | ||||
David Soria Parra
|
r30629 | p4changes = {} | ||
Frank Kingswood
|
r7823 | for d in loaditer(stdout): | ||
Augie Fackler
|
r43347 | c = d.get(b"change", None) | ||
Frank Kingswood
|
r7823 | if c: | ||
David Soria Parra
|
r30629 | p4changes[c] = True | ||
return p4changes | ||||
Frank Kingswood
|
r7823 | |||
def _parse(self, ui, path): | ||||
Matt Harbison
|
r44226 | """Prepare list of P4 filenames and revisions to import""" | ||
David Soria Parra
|
r30631 | p4changes = {} | ||
changeset = {} | ||||
files_map = {} | ||||
copies_map = {} | ||||
localname = {} | ||||
depotname = {} | ||||
heads = [] | ||||
Augie Fackler
|
r43347 | ui.status(_(b'reading p4 views\n')) | ||
Frank Kingswood
|
r7823 | |||
# read client spec or view | ||||
Augie Fackler
|
r43347 | if b"/" in path: | ||
David Soria Parra
|
r30631 | p4changes.update(self._parse_view(path)) | ||
Augie Fackler
|
r43347 | if path.startswith(b"//") and path.endswith(b"/..."): | ||
views = {path[:-3]: b""} | ||||
Frank Kingswood
|
r7823 | else: | ||
Augie Fackler
|
r43347 | views = {b"//": b""} | ||
Frank Kingswood
|
r7823 | else: | ||
Augie Fackler
|
r43347 | cmd = b'p4 -G client -o %s' % procutil.shellquote(path) | ||
clientspec = marshal.load(procutil.popen(cmd, mode=b'rb')) | ||||
Dirkjan Ochtman
|
r7869 | |||
Frank Kingswood
|
r7823 | views = {} | ||
for client in clientspec: | ||||
Augie Fackler
|
r43347 | if client.startswith(b"View"): | ||
Frank Kingswood
|
r7823 | sview, cview = clientspec[client].split() | ||
David Soria Parra
|
r30631 | p4changes.update(self._parse_view(sview)) | ||
Augie Fackler
|
r43347 | if sview.endswith(b"...") and cview.endswith(b"..."): | ||
Frank Kingswood
|
r7823 | sview = sview[:-3] | ||
cview = cview[:-3] | ||||
cview = cview[2:] | ||||
Augie Fackler
|
r43347 | cview = cview[cview.find(b"/") + 1 :] | ||
Frank Kingswood
|
r7823 | views[sview] = cview | ||
# list of changes that affect our source files | ||||
Nate Skulic
|
r47899 | p4changes = sorted(p4changes.keys(), key=int) | ||
Frank Kingswood
|
r7823 | |||
# list with depot pathnames, longest first | ||||
Nate Skulic
|
r47899 | vieworder = sorted(views.keys(), key=len, reverse=True) | ||
Frank Kingswood
|
r7823 | |||
# handle revision limiting | ||||
Augie Fackler
|
r43347 | startrev = self.ui.config(b'convert', b'p4.startrev') | ||
Frank Kingswood
|
r7823 | |||
# now read the full changelists to get the list of file revisions | ||||
Augie Fackler
|
r43347 | ui.status(_(b'collecting p4 changelists\n')) | ||
Frank Kingswood
|
r7823 | lastid = None | ||
David Soria Parra
|
r30631 | for change in p4changes: | ||
David Soria Parra
|
r30597 | if startrev and int(change) < int(startrev): | ||
continue | ||||
if self.revs and int(change) > int(self.revs[0]): | ||||
continue | ||||
David Soria Parra
|
r30601 | if change in self.revmap: | ||
# Ignore already present revisions, but set the parent pointer. | ||||
lastid = change | ||||
continue | ||||
David Soria Parra
|
r30597 | |||
Frank Kingswood
|
r7823 | if lastid: | ||
parents = [lastid] | ||||
else: | ||||
parents = [] | ||||
Dirkjan Ochtman
|
r7869 | |||
David Soria Parra
|
r30603 | d = self._fetch_revision(change) | ||
c = self._construct_commit(d, parents) | ||||
David Soria Parra
|
r31590 | descarr = c.desc.splitlines(True) | ||
if len(descarr) > 0: | ||||
Augie Fackler
|
r43347 | shortdesc = descarr[0].rstrip(b'\r\n') | ||
David Soria Parra
|
r31590 | else: | ||
Augie Fackler
|
r43347 | shortdesc = b'**empty changelist description**' | ||
David Soria Parra
|
r31590 | |||
Nate Skulic
|
r47899 | t = b'%s %s' % (c.rev, shortdesc) | ||
Augie Fackler
|
r43347 | ui.status(stringutil.ellipsis(t, 80) + b'\n') | ||
Frank Kingswood
|
r7823 | |||
files = [] | ||||
Eugene Baranov
|
r25751 | copies = {} | ||
copiedfiles = [] | ||||
Frank Kingswood
|
r7823 | i = 0 | ||
Augie Fackler
|
r43347 | while (b"depotFile%d" % i) in d and (b"rev%d" % i) in d: | ||
oldname = d[b"depotFile%d" % i] | ||||
Frank Kingswood
|
r7823 | filename = None | ||
for v in vieworder: | ||||
Eugene Baranov
|
r25776 | if oldname.lower().startswith(v.lower()): | ||
Augie Fackler
|
r43346 | filename = decodefilename(views[v] + oldname[len(v) :]) | ||
Frank Kingswood
|
r7823 | break | ||
if filename: | ||||
Augie Fackler
|
r43347 | files.append((filename, d[b"rev%d" % i])) | ||
David Soria Parra
|
r30631 | depotname[filename] = oldname | ||
Augie Fackler
|
r43347 | if d.get(b"action%d" % i) == b"move/add": | ||
Eugene Baranov
|
r25751 | copiedfiles.append(filename) | ||
David Soria Parra
|
r30630 | localname[oldname] = filename | ||
Frank Kingswood
|
r7823 | i += 1 | ||
Eugene Baranov
|
r25751 | |||
# Collect information about copied files | ||||
for filename in copiedfiles: | ||||
David Soria Parra
|
r30631 | oldname = depotname[filename] | ||
Eugene Baranov
|
r25751 | |||
Augie Fackler
|
r43347 | flcmd = b'p4 -G filelog %s' % procutil.shellquote(oldname) | ||
flstdout = procutil.popen(flcmd, mode=b'rb') | ||||
Eugene Baranov
|
r25751 | |||
copiedfilename = None | ||||
for d in loaditer(flstdout): | ||||
copiedoldname = None | ||||
i = 0 | ||||
Augie Fackler
|
r43347 | while (b"change%d" % i) in d: | ||
Augie Fackler
|
r43346 | if ( | ||
Augie Fackler
|
r43347 | d[b"change%d" % i] == change | ||
and d[b"action%d" % i] == b"move/add" | ||||
Augie Fackler
|
r43346 | ): | ||
Eugene Baranov
|
r25751 | j = 0 | ||
Augie Fackler
|
r43347 | while (b"file%d,%d" % (i, j)) in d: | ||
if d[b"how%d,%d" % (i, j)] == b"moved from": | ||||
copiedoldname = d[b"file%d,%d" % (i, j)] | ||||
Eugene Baranov
|
r25751 | break | ||
j += 1 | ||||
i += 1 | ||||
David Soria Parra
|
r30630 | if copiedoldname and copiedoldname in localname: | ||
copiedfilename = localname[copiedoldname] | ||||
Eugene Baranov
|
r25751 | break | ||
if copiedfilename: | ||||
copies[filename] = copiedfilename | ||||
else: | ||||
Augie Fackler
|
r43346 | ui.warn( | ||
Augie Fackler
|
r43347 | _(b"cannot find source for copied file: %s@%s\n") | ||
Augie Fackler
|
r43346 | % (filename, change) | ||
) | ||||
Eugene Baranov
|
r25751 | |||
David Soria Parra
|
r30631 | changeset[change] = c | ||
files_map[change] = files | ||||
copies_map[change] = copies | ||||
Frank Kingswood
|
r7823 | lastid = change | ||
Dirkjan Ochtman
|
r7869 | |||
David Soria Parra
|
r30631 | if lastid and len(changeset) > 0: | ||
heads = [lastid] | ||||
return { | ||||
Augie Fackler
|
r43347 | b'changeset': changeset, | ||
b'files': files_map, | ||||
b'copies': copies_map, | ||||
b'heads': heads, | ||||
b'depotname': depotname, | ||||
David Soria Parra
|
r30631 | } | ||
David Soria Parra
|
r30632 | @util.propertycache | ||
def _parse_once(self): | ||||
return self._parse(self.ui, self.path) | ||||
@util.propertycache | ||||
def copies(self): | ||||
Augie Fackler
|
r43347 | return self._parse_once[b'copies'] | ||
David Soria Parra
|
r30632 | |||
@util.propertycache | ||||
def files(self): | ||||
Augie Fackler
|
r43347 | return self._parse_once[b'files'] | ||
David Soria Parra
|
r30632 | |||
@util.propertycache | ||||
def changeset(self): | ||||
Augie Fackler
|
r43347 | return self._parse_once[b'changeset'] | ||
David Soria Parra
|
r30632 | |||
@util.propertycache | ||||
def heads(self): | ||||
Augie Fackler
|
r43347 | return self._parse_once[b'heads'] | ||
David Soria Parra
|
r30632 | |||
@util.propertycache | ||||
def depotname(self): | ||||
Augie Fackler
|
r43347 | return self._parse_once[b'depotname'] | ||
Frank Kingswood
|
r7823 | |||
def getheads(self): | ||||
return self.heads | ||||
def getfile(self, name, rev): | ||||
Augie Fackler
|
r43347 | cmd = b'p4 -G print %s' % procutil.shellquote( | ||
b"%s#%s" % (self.depotname[name], rev) | ||||
Augie Fackler
|
r43346 | ) | ||
Frank Kingswood
|
r7823 | |||
Eugene Baranov
|
r25775 | lasterror = None | ||
while True: | ||||
Augie Fackler
|
r43347 | stdout = procutil.popen(cmd, mode=b'rb') | ||
Eugene Baranov
|
r25775 | |||
mode = None | ||||
Eugene Baranov
|
r25882 | contents = [] | ||
Eugene Baranov
|
r25775 | keywords = None | ||
Frank Kingswood
|
r7823 | |||
Eugene Baranov
|
r25775 | for d in loaditer(stdout): | ||
Augie Fackler
|
r43347 | code = d[b"code"] | ||
data = d.get(b"data") | ||||
Frank Kingswood
|
r8829 | |||
Augie Fackler
|
r43347 | if code == b"error": | ||
Eugene Baranov
|
r25775 | # if this is the first time error happened | ||
# re-attempt getting the file | ||||
if not lasterror: | ||||
Augie Fackler
|
r43347 | lasterror = IOError(d[b"generic"], data) | ||
Eugene Baranov
|
r25775 | # this will exit inner-most for-loop | ||
break | ||||
else: | ||||
raise lasterror | ||||
Dirkjan Ochtman
|
r8843 | |||
Augie Fackler
|
r43347 | elif code == b"stat": | ||
action = d.get(b"action") | ||||
if action in [b"purge", b"delete", b"move/delete"]: | ||||
Eugene Baranov
|
r25775 | return None, None | ||
Augie Fackler
|
r43347 | p4type = self.re_type.match(d[b"type"]) | ||
Eugene Baranov
|
r25775 | if p4type: | ||
Augie Fackler
|
r43347 | mode = b"" | ||
flags = (p4type.group(1) or b"") + ( | ||||
p4type.group(3) or b"" | ||||
Augie Fackler
|
r43346 | ) | ||
Augie Fackler
|
r43347 | if b"x" in flags: | ||
mode = b"x" | ||||
if p4type.group(2) == b"symlink": | ||||
mode = b"l" | ||||
if b"ko" in flags: | ||||
Eugene Baranov
|
r25775 | keywords = self.re_keywords_old | ||
Augie Fackler
|
r43347 | elif b"k" in flags: | ||
Eugene Baranov
|
r25775 | keywords = self.re_keywords | ||
Dirkjan Ochtman
|
r8843 | |||
Augie Fackler
|
r43347 | elif code == b"text" or code == b"binary": | ||
Eugene Baranov
|
r25882 | contents.append(data) | ||
Eugene Baranov
|
r25775 | |||
lasterror = None | ||||
if not lasterror: | ||||
break | ||||
Frank Kingswood
|
r7823 | |||
if mode is None: | ||||
Mads Kiilerich
|
r22296 | return None, None | ||
Frank Kingswood
|
r7823 | |||
Augie Fackler
|
r43347 | contents = b''.join(contents) | ||
Eugene Baranov
|
r25882 | |||
Frank Kingswood
|
r8829 | if keywords: | ||
Augie Fackler
|
r43347 | contents = keywords.sub(b"$\\1$", contents) | ||
if mode == b"l" and contents.endswith(b"\n"): | ||||
Frank Kingswood
|
r8829 | contents = contents[:-1] | ||
Patrick Mezard
|
r11134 | return contents, mode | ||
Frank Kingswood
|
r7823 | |||
Mads Kiilerich
|
r22300 | def getchanges(self, rev, full): | ||
if full: | ||||
Augie Fackler
|
r43347 | raise error.Abort(_(b"convert from p4 does not support --full")) | ||
Eugene Baranov
|
r25751 | return self.files[rev], self.copies[rev], set() | ||
Frank Kingswood
|
r7823 | |||
David Soria Parra
|
r30603 | def _construct_commit(self, obj, parents=None): | ||
""" | ||||
Constructs a common.commit object from an unmarshalled | ||||
`p4 describe` output | ||||
""" | ||||
Augie Fackler
|
r43347 | desc = self.recode(obj.get(b"desc", b"")) | ||
date = (int(obj[b"time"]), 0) # timezone not set | ||||
David Soria Parra
|
r30603 | if parents is None: | ||
parents = [] | ||||
Augie Fackler
|
r43346 | return common.commit( | ||
Augie Fackler
|
r43347 | author=self.recode(obj[b"user"]), | ||
date=dateutil.datestr(date, b'%Y-%m-%d %H:%M:%S %1%2'), | ||||
Augie Fackler
|
r43346 | parents=parents, | ||
desc=desc, | ||||
branch=None, | ||||
Augie Fackler
|
r43347 | rev=obj[b'change'], | ||
extra={b"p4": obj[b'change'], b"convert_revision": obj[b'change']}, | ||||
Augie Fackler
|
r43346 | ) | ||
David Soria Parra
|
r30603 | |||
def _fetch_revision(self, rev): | ||||
"""Return an output of `p4 describe` including author, commit date as | ||||
a dictionary.""" | ||||
Augie Fackler
|
r43347 | cmd = b"p4 -G describe -s %s" % rev | ||
stdout = procutil.popen(cmd, mode=b'rb') | ||||
David Soria Parra
|
r30603 | return marshal.load(stdout) | ||
Frank Kingswood
|
r7823 | def getcommit(self, rev): | ||
David Soria Parra
|
r30604 | if rev in self.changeset: | ||
return self.changeset[rev] | ||||
elif rev in self.revmap: | ||||
d = self._fetch_revision(rev) | ||||
return self._construct_commit(d, parents=None) | ||||
raise error.Abort( | ||||
Augie Fackler
|
r43347 | _(b"cannot find %s in the revmap or parsed changesets") % rev | ||
Augie Fackler
|
r43346 | ) | ||
Frank Kingswood
|
r7823 | |||
def gettags(self): | ||||
David Soria Parra
|
r30599 | return {} | ||
Frank Kingswood
|
r7823 | |||
def getchangedfiles(self, rev, i): | ||||
Matt Mackall
|
r8209 | return sorted([x[0] for x in self.files[rev]]) | ||