The open source repository for the action RPG game in development by Sig Productions titled 'Adventures in Lestoria'!
https://forums.lestoria.net
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
4005 lines
162 KiB
4005 lines
162 KiB
#!/usr/bin/env python3
|
|
|
|
"""
|
|
git-filter-repo filters git repositories, similar to git filter-branch, BFG
|
|
repo cleaner, and others. The basic idea is that it works by running
|
|
git fast-export <options> | filter | git fast-import <options>
|
|
where this program not only launches the whole pipeline but also serves as
|
|
the 'filter' in the middle. It does a few additional things on top as well
|
|
in order to make it into a well-rounded filtering tool.
|
|
|
|
git-filter-repo can also be used as a library for more involved filtering
|
|
operations; however:
|
|
***** API BACKWARD COMPATIBILITY CAVEAT *****
|
|
Programs using git-filter-repo as a library can reach pretty far into its
|
|
internals, but I am not prepared to guarantee backward compatibility of
|
|
all APIs. I suspect changes will be rare, but I reserve the right to
|
|
change any API. Since it is assumed that repository filtering is
|
|
something one would do very rarely, and in particular that it's a
|
|
one-shot operation, this should not be a problem in practice for anyone.
|
|
However, if you want to re-use a program you have written that uses
|
|
git-filter-repo as a library (or makes use of one of its --*-callback
|
|
arguments), you should either make sure you are using the same version of
|
|
git and git-filter-repo, or make sure to re-test it.
|
|
|
|
If there are particular pieces of the API you are concerned about, and
|
|
there is not already a testcase for it in t9391-lib-usage.sh or
|
|
t9392-python-callback.sh, please contribute a testcase. That will not
|
|
prevent me from changing the API, but it will allow you to look at the
|
|
history of a testcase to see whether and how the API changed.
|
|
***** END API BACKWARD COMPATIBILITY CAVEAT *****
|
|
"""
|
|
|
|
import argparse
|
|
import collections
|
|
import fnmatch
|
|
import gettext
|
|
import io
|
|
import os
|
|
import platform
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import textwrap
|
|
|
|
from datetime import tzinfo, timedelta, datetime
|
|
|
|
__all__ = ["Blob", "Reset", "FileChange", "Commit", "Tag", "Progress",
|
|
"Checkpoint", "FastExportParser", "ProgressWriter",
|
|
"string_to_date", "date_to_string",
|
|
"record_id_rename", "GitUtils", "FilteringOptions", "RepoFilter"]
|
|
|
|
deleted_hash = b'0'*40
|
|
write_marks = True
|
|
date_format_permissive = True
|
|
|
|
def gettext_poison(msg):
|
|
if "GIT_TEST_GETTEXT_POISON" in os.environ: # pragma: no cover
|
|
return "# GETTEXT POISON #"
|
|
return gettext.gettext(msg)
|
|
|
|
_ = gettext_poison
|
|
|
|
def setup_gettext():
|
|
TEXTDOMAIN="git-filter-repo"
|
|
podir = os.environ.get("GIT_TEXTDOMAINDIR") or "@@LOCALEDIR@@"
|
|
if not os.path.isdir(podir): # pragma: no cover
|
|
podir = None # Python has its own fallback; use that
|
|
|
|
## This looks like the most straightforward translation of the relevant
|
|
## code in git.git:gettext.c and git.git:perl/Git/I18n.pm:
|
|
#import locale
|
|
#locale.setlocale(locale.LC_MESSAGES, "");
|
|
#locale.setlocale(locale.LC_TIME, "");
|
|
#locale.textdomain(TEXTDOMAIN);
|
|
#locale.bindtextdomain(TEXTDOMAIN, podir);
|
|
## but the python docs suggest using the gettext module (which doesn't
|
|
## have setlocale()) instead, so:
|
|
gettext.textdomain(TEXTDOMAIN);
|
|
gettext.bindtextdomain(TEXTDOMAIN, podir);
|
|
|
|
def _timedelta_to_seconds(delta):
|
|
"""
|
|
Converts timedelta to seconds
|
|
"""
|
|
offset = delta.days*86400 + delta.seconds + (delta.microseconds+0.0)/1000000
|
|
return round(offset)
|
|
|
|
class FixedTimeZone(tzinfo):
|
|
"""
|
|
Fixed offset in minutes east from UTC.
|
|
"""
|
|
|
|
tz_re = re.compile(br'^([-+]?)(\d\d)(\d\d)$')
|
|
|
|
def __init__(self, offset_string):
|
|
tzinfo.__init__(self)
|
|
sign, hh, mm = FixedTimeZone.tz_re.match(offset_string).groups()
|
|
factor = -1 if (sign and sign == b'-') else 1
|
|
self._offset = timedelta(minutes = factor*(60*int(hh) + int(mm)))
|
|
self._offset_string = offset_string
|
|
|
|
def utcoffset(self, dt):
|
|
return self._offset
|
|
|
|
def tzname(self, dt):
|
|
return self._offset_string
|
|
|
|
def dst(self, dt):
|
|
return timedelta(0)
|
|
|
|
def string_to_date(datestring):
|
|
(unix_timestamp, tz_offset) = datestring.split()
|
|
return datetime.fromtimestamp(int(unix_timestamp),
|
|
FixedTimeZone(tz_offset))
|
|
|
|
def date_to_string(dateobj):
|
|
epoch = datetime.fromtimestamp(0, dateobj.tzinfo)
|
|
return(b'%d %s' % (int(_timedelta_to_seconds(dateobj - epoch)),
|
|
dateobj.tzinfo.tzname(0)))
|
|
|
|
def decode(bytestr):
|
|
'Try to convert bytestr to utf-8 for outputting as an error message.'
|
|
return bytestr.decode('utf-8', 'backslashreplace')
|
|
|
|
def glob_to_regex(glob_bytestr):
|
|
'Translate glob_bytestr into a regex on bytestrings'
|
|
|
|
# fnmatch.translate is idiotic and won't accept bytestrings
|
|
if (decode(glob_bytestr).encode() != glob_bytestr): # pragma: no cover
|
|
raise SystemExit(_("Error: Cannot handle glob %s").format(glob_bytestr))
|
|
|
|
# Create regex operating on string
|
|
regex = fnmatch.translate(decode(glob_bytestr))
|
|
|
|
# FIXME: This is an ugly hack...
|
|
# fnmatch.translate tries to do multi-line matching and wants the glob to
|
|
# match up to the end of the input, which isn't relevant for us, so we
|
|
# have to modify the regex. fnmatch.translate has used different regex
|
|
# constructs to achieve this with different python versions, so we have
|
|
# to check for each of them and then fix it up. It would be much better
|
|
# if fnmatch.translate could just take some flags to allow us to specify
|
|
# what we want rather than employing this hackery, but since it
|
|
# doesn't...
|
|
if regex.endswith(r'\Z(?ms)'): # pragma: no cover
|
|
regex = regex[0:-7]
|
|
elif regex.startswith(r'(?s:') and regex.endswith(r')\Z'): # pragma: no cover
|
|
regex = regex[4:-3]
|
|
|
|
# Finally, convert back to regex operating on bytestr
|
|
return regex.encode()
|
|
|
|
class PathQuoting:
|
|
_unescape = {b'a': b'\a',
|
|
b'b': b'\b',
|
|
b'f': b'\f',
|
|
b'n': b'\n',
|
|
b'r': b'\r',
|
|
b't': b'\t',
|
|
b'v': b'\v',
|
|
b'"': b'"',
|
|
b'\\':b'\\'}
|
|
_unescape_re = re.compile(br'\\([a-z"\\]|[0-9]{3})')
|
|
_escape = [bytes([x]) for x in range(127)]+[
|
|
b'\\'+bytes(ord(c) for c in oct(x)[2:]) for x in range(127,256)]
|
|
_reverse = dict(map(reversed, _unescape.items()))
|
|
for x in _reverse:
|
|
_escape[ord(x)] = b'\\'+_reverse[x]
|
|
_special_chars = [len(x) > 1 for x in _escape]
|
|
|
|
@staticmethod
|
|
def unescape_sequence(orig):
|
|
seq = orig.group(1)
|
|
return PathQuoting._unescape[seq] if len(seq) == 1 else bytes([int(seq, 8)])
|
|
|
|
@staticmethod
|
|
def dequote(quoted_string):
|
|
if quoted_string.startswith(b'"'):
|
|
assert quoted_string.endswith(b'"')
|
|
return PathQuoting._unescape_re.sub(PathQuoting.unescape_sequence,
|
|
quoted_string[1:-1])
|
|
return quoted_string
|
|
|
|
@staticmethod
|
|
def enquote(unquoted_string):
|
|
# Option 1: Quoting when fast-export would:
|
|
# pqsc = PathQuoting._special_chars
|
|
# if any(pqsc[x] for x in set(unquoted_string)):
|
|
# Option 2, perf hack: do minimal amount of quoting required by fast-import
|
|
if unquoted_string.startswith(b'"') or b'\n' in unquoted_string:
|
|
pqe = PathQuoting._escape
|
|
return b'"' + b''.join(pqe[x] for x in unquoted_string) + b'"'
|
|
return unquoted_string
|
|
|
|
class AncestryGraph(object):
|
|
"""
|
|
A class that maintains a direct acycle graph of commits for the purpose of
|
|
determining if one commit is the ancestor of another.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.cur_value = 0
|
|
|
|
# A mapping from the external identifers given to us to the simple integers
|
|
# we use in self.graph
|
|
self.value = {}
|
|
|
|
# A tuple of (depth, list-of-ancestors). Values and keys in this graph are
|
|
# all integers from the self.value dict. The depth of a commit is one more
|
|
# than the max depth of any of its ancestors.
|
|
self.graph = {}
|
|
|
|
# Cached results from previous calls to is_ancestor().
|
|
self._cached_is_ancestor = {}
|
|
|
|
def record_external_commits(self, external_commits):
|
|
"""
|
|
Record in graph that each commit in external_commits exists, and is
|
|
treated as a root commit with no parents.
|
|
"""
|
|
for c in external_commits:
|
|
if c not in self.value:
|
|
self.cur_value += 1
|
|
self.value[c] = self.cur_value
|
|
self.graph[self.cur_value] = (1, [])
|
|
|
|
def add_commit_and_parents(self, commit, parents):
|
|
"""
|
|
Record in graph that commit has the given parents. parents _MUST_ have
|
|
been first recorded. commit _MUST_ not have been recorded yet.
|
|
"""
|
|
assert all(p in self.value for p in parents)
|
|
assert commit not in self.value
|
|
|
|
# Get values for commit and parents
|
|
self.cur_value += 1
|
|
self.value[commit] = self.cur_value
|
|
graph_parents = [self.value[x] for x in parents]
|
|
|
|
# Determine depth for commit, then insert the info into the graph
|
|
depth = 1
|
|
if parents:
|
|
depth += max(self.graph[p][0] for p in graph_parents)
|
|
self.graph[self.cur_value] = (depth, graph_parents)
|
|
|
|
def is_ancestor(self, possible_ancestor, check):
|
|
"""
|
|
Return whether possible_ancestor is an ancestor of check
|
|
"""
|
|
a, b = self.value[possible_ancestor], self.value[check]
|
|
original_pair = (a,b)
|
|
a_depth = self.graph[a][0]
|
|
ancestors = [b]
|
|
visited = set()
|
|
while ancestors:
|
|
ancestor = ancestors.pop()
|
|
prev_pair = (a, ancestor)
|
|
if prev_pair in self._cached_is_ancestor:
|
|
if not self._cached_is_ancestor[prev_pair]:
|
|
continue
|
|
self._cached_is_ancestor[original_pair] = True
|
|
return True
|
|
if ancestor in visited:
|
|
continue
|
|
visited.add(ancestor)
|
|
depth, more_ancestors = self.graph[ancestor]
|
|
if ancestor == a:
|
|
self._cached_is_ancestor[original_pair] = True
|
|
return True
|
|
elif depth <= a_depth:
|
|
continue
|
|
ancestors.extend(more_ancestors)
|
|
self._cached_is_ancestor[original_pair] = False
|
|
return False
|
|
|
|
class MailmapInfo(object):
|
|
def __init__(self, filename):
|
|
self.changes = {}
|
|
self._parse_file(filename)
|
|
|
|
def _parse_file(self, filename):
|
|
name_and_email_re = re.compile(br'(.*?)\s*<([^>]*)>\s*')
|
|
comment_re = re.compile(br'\s*#.*')
|
|
if not os.access(filename, os.R_OK):
|
|
raise SystemExit(_("Cannot read %s") % decode(filename))
|
|
with open(filename, 'br') as f:
|
|
count = 0
|
|
for line in f:
|
|
count += 1
|
|
err = "Unparseable mailmap file: line #{} is bad: {}".format(count, line)
|
|
# Remove comments
|
|
line = comment_re.sub(b'', line)
|
|
# Remove leading and trailing whitespace
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
|
|
m = name_and_email_re.match(line)
|
|
if not m:
|
|
raise SystemExit(err)
|
|
proper_name, proper_email = m.groups()
|
|
if len(line) == m.end():
|
|
self.changes[(None, proper_email)] = (proper_name, proper_email)
|
|
continue
|
|
rest = line[m.end():]
|
|
m = name_and_email_re.match(rest)
|
|
if m:
|
|
commit_name, commit_email = m.groups()
|
|
if len(rest) != m.end():
|
|
raise SystemExit(err)
|
|
else:
|
|
commit_name, commit_email = rest, None
|
|
self.changes[(commit_name, commit_email)] = (proper_name, proper_email)
|
|
|
|
def translate(self, name, email):
|
|
''' Given a name and email, return the expected new name and email from the
|
|
mailmap if there is a translation rule for it, otherwise just return
|
|
the given name and email.'''
|
|
for old, new in self.changes.items():
|
|
old_name, old_email = old
|
|
new_name, new_email = new
|
|
if (old_email is None or email.lower() == old_email.lower()) and (
|
|
name == old_name or not old_name):
|
|
return (new_name or name, new_email or email)
|
|
return (name, email)
|
|
|
|
class ProgressWriter(object):
|
|
def __init__(self):
|
|
self._last_progress_update = time.time()
|
|
self._last_message = None
|
|
|
|
def show(self, msg):
|
|
self._last_message = msg
|
|
now = time.time()
|
|
if now - self._last_progress_update > .1:
|
|
self._last_progress_update = now
|
|
sys.stdout.write("\r{}".format(msg))
|
|
sys.stdout.flush()
|
|
|
|
def finish(self):
|
|
self._last_progress_update = 0
|
|
if self._last_message:
|
|
self.show(self._last_message)
|
|
sys.stdout.write("\n")
|
|
|
|
class _IDs(object):
|
|
"""
|
|
A class that maintains the 'name domain' of all the 'marks' (short int
|
|
id for a blob/commit git object). The reason this mechanism is necessary
|
|
is because the text of fast-export may refer to an object using a different
|
|
mark than the mark that was assigned to that object using IDS.new(). This
|
|
class allows you to translate the fast-export marks (old) to the marks
|
|
assigned from IDS.new() (new).
|
|
|
|
Note that there are two reasons why the marks may differ: (1) The
|
|
user manually creates Blob or Commit objects (for insertion into the
|
|
stream) (2) We're reading the data from two different repositories
|
|
and trying to combine the data (git fast-export will number ids from
|
|
1...n, and having two 1's, two 2's, two 3's, causes issues).
|
|
"""
|
|
|
|
def __init__(self):
|
|
"""
|
|
Init
|
|
"""
|
|
# The id for the next created blob/commit object
|
|
self._next_id = 1
|
|
|
|
# A map of old-ids to new-ids (1:1 map)
|
|
self._translation = {}
|
|
|
|
# A map of new-ids to every old-id that points to the new-id (1:N map)
|
|
self._reverse_translation = {}
|
|
|
|
def has_renames(self):
|
|
"""
|
|
Return whether there have been ids remapped to new values
|
|
"""
|
|
return bool(self._translation)
|
|
|
|
def new(self):
|
|
"""
|
|
Should be called whenever a new blob or commit object is created. The
|
|
returned value should be used as the id/mark for that object.
|
|
"""
|
|
rv = self._next_id
|
|
self._next_id += 1
|
|
return rv
|
|
|
|
def record_rename(self, old_id, new_id, handle_transitivity = False):
|
|
"""
|
|
Record that old_id is being renamed to new_id.
|
|
"""
|
|
if old_id != new_id:
|
|
# old_id -> new_id
|
|
self._translation[old_id] = new_id
|
|
|
|
# Transitivity will be needed if new commits are being inserted mid-way
|
|
# through a branch.
|
|
if handle_transitivity:
|
|
# Anything that points to old_id should point to new_id
|
|
if old_id in self._reverse_translation:
|
|
for id_ in self._reverse_translation[old_id]:
|
|
self._translation[id_] = new_id
|
|
|
|
# Record that new_id is pointed to by old_id
|
|
if new_id not in self._reverse_translation:
|
|
self._reverse_translation[new_id] = []
|
|
self._reverse_translation[new_id].append(old_id)
|
|
|
|
def translate(self, old_id):
|
|
"""
|
|
If old_id has been mapped to an alternate id, return the alternate id.
|
|
"""
|
|
if old_id in self._translation:
|
|
return self._translation[old_id]
|
|
else:
|
|
return old_id
|
|
|
|
def __str__(self):
|
|
"""
|
|
Convert IDs to string; used for debugging
|
|
"""
|
|
rv = "Current count: %d\nTranslation:\n" % self._next_id
|
|
for k in sorted(self._translation):
|
|
rv += " %d -> %s\n" % (k, self._translation[k])
|
|
|
|
rv += "Reverse translation:\n"
|
|
for k in sorted(self._reverse_translation):
|
|
rv += " " + str(k) + " -> " + str(self._reverse_translation[k]) + "\n"
|
|
|
|
return rv
|
|
|
|
class _GitElement(object):
|
|
"""
|
|
The base class for all git elements that we create.
|
|
"""
|
|
|
|
def __init__(self):
|
|
# A string that describes what type of Git element this is
|
|
self.type = None
|
|
|
|
# A flag telling us if this Git element has been dumped
|
|
# (i.e. printed) or skipped. Typically elements that have been
|
|
# dumped or skipped will not be dumped again.
|
|
self.dumped = 0
|
|
|
|
def dump(self, file_):
|
|
"""
|
|
This version should never be called. Derived classes need to
|
|
override! We should note that subclasses should implement this
|
|
method such that the output would match the format produced by
|
|
fast-export.
|
|
"""
|
|
raise SystemExit(_("Unimplemented function: %s") % type(self).__name__
|
|
+".dump()") # pragma: no cover
|
|
|
|
def __bytes__(self):
|
|
"""
|
|
Convert GitElement to bytestring; used for debugging
|
|
"""
|
|
old_dumped = self.dumped
|
|
writeme = io.BytesIO()
|
|
self.dump(writeme)
|
|
output_lines = writeme.getvalue().splitlines()
|
|
writeme.close()
|
|
self.dumped = old_dumped
|
|
return b"%s:\n %s" % (type(self).__name__.encode(),
|
|
b"\n ".join(output_lines))
|
|
|
|
def skip(self, new_id=None):
|
|
"""
|
|
Ensures this element will not be written to output
|
|
"""
|
|
self.dumped = 2
|
|
|
|
class _GitElementWithId(_GitElement):
|
|
"""
|
|
The base class for Git elements that have IDs (commits and blobs)
|
|
"""
|
|
|
|
def __init__(self):
|
|
_GitElement.__init__(self)
|
|
|
|
# The mark (short, portable id) for this element
|
|
self.id = _IDS.new()
|
|
|
|
# The previous mark for this element
|
|
self.old_id = None
|
|
|
|
def skip(self, new_id=None):
|
|
"""
|
|
This element will no longer be automatically written to output. When a
|
|
commit gets skipped, it's ID will need to be translated to that of its
|
|
parent.
|
|
"""
|
|
self.dumped = 2
|
|
|
|
_IDS.record_rename(self.old_id or self.id, new_id)
|
|
|
|
class Blob(_GitElementWithId):
|
|
"""
|
|
This class defines our representation of git blob elements (i.e. our
|
|
way of representing file contents).
|
|
"""
|
|
|
|
def __init__(self, data, original_id = None):
|
|
_GitElementWithId.__init__(self)
|
|
|
|
# Denote that this is a blob
|
|
self.type = 'blob'
|
|
|
|
# Record original id
|
|
self.original_id = original_id
|
|
|
|
# Stores the blob's data
|
|
assert(type(data) == bytes)
|
|
self.data = data
|
|
|
|
def dump(self, file_):
|
|
"""
|
|
Write this blob element to a file.
|
|
"""
|
|
self.dumped = 1
|
|
HASH_TO_ID[self.original_id] = self.id
|
|
ID_TO_HASH[self.id] = self.original_id
|
|
|
|
file_.write(b'blob\n')
|
|
file_.write(b'mark :%d\n' % self.id)
|
|
file_.write(b'data %d\n%s' % (len(self.data), self.data))
|
|
file_.write(b'\n')
|
|
|
|
|
|
class Reset(_GitElement):
|
|
"""
|
|
This class defines our representation of git reset elements. A reset
|
|
event is the creation (or recreation) of a named branch, optionally
|
|
starting from a specific revision).
|
|
"""
|
|
|
|
def __init__(self, ref, from_ref = None):
|
|
_GitElement.__init__(self)
|
|
|
|
# Denote that this is a reset
|
|
self.type = 'reset'
|
|
|
|
# The name of the branch being (re)created
|
|
self.ref = ref
|
|
|
|
# Some reference to the branch/commit we are resetting from
|
|
self.from_ref = from_ref
|
|
|
|
def dump(self, file_):
|
|
"""
|
|
Write this reset element to a file
|
|
"""
|
|
self.dumped = 1
|
|
|
|
file_.write(b'reset %s\n' % self.ref)
|
|
if self.from_ref:
|
|
if isinstance(self.from_ref, int):
|
|
file_.write(b'from :%d\n' % self.from_ref)
|
|
else:
|
|
file_.write(b'from %s\n' % self.from_ref)
|
|
file_.write(b'\n')
|
|
|
|
class FileChange(_GitElement):
|
|
"""
|
|
This class defines our representation of file change elements. File change
|
|
elements are components within a Commit element.
|
|
"""
|
|
|
|
def __init__(self, type_, filename = None, id_ = None, mode = None):
|
|
_GitElement.__init__(self)
|
|
|
|
# Denote the type of file-change (b'M' for modify, b'D' for delete, etc)
|
|
# We could
|
|
# assert(type(type_) == bytes)
|
|
# here but I don't just due to worries about performance overhead...
|
|
self.type = type_
|
|
|
|
# Record the name of the file being changed
|
|
self.filename = filename
|
|
|
|
# Record the mode (mode describes type of file entry (non-executable,
|
|
# executable, or symlink)).
|
|
self.mode = mode
|
|
|
|
# blob_id is the id (mark) of the affected blob
|
|
self.blob_id = id_
|
|
|
|
if type_ == b'DELETEALL':
|
|
assert filename is None and id_ is None and mode is None
|
|
self.filename = b'' # Just so PathQuoting.enquote doesn't die
|
|
else:
|
|
assert filename is not None
|
|
|
|
if type_ == b'M':
|
|
assert id_ is not None and mode is not None
|
|
elif type_ == b'D':
|
|
assert id_ is None and mode is None
|
|
elif type_ == b'R': # pragma: no cover (now avoid fast-export renames)
|
|
assert mode is None
|
|
if id_ is None:
|
|
raise SystemExit(_("new name needed for rename of %s") % filename)
|
|
self.filename = (self.filename, id_)
|
|
self.blob_id = None
|
|
|
|
def dump(self, file_):
|
|
"""
|
|
Write this file-change element to a file
|
|
"""
|
|
skipped_blob = (self.type == b'M' and self.blob_id is None)
|
|
if skipped_blob: return
|
|
self.dumped = 1
|
|
|
|
quoted_filename = PathQuoting.enquote(self.filename)
|
|
if self.type == b'M' and isinstance(self.blob_id, int):
|
|
file_.write(b'M %s :%d %s\n' % (self.mode, self.blob_id, quoted_filename))
|
|
elif self.type == b'M':
|
|
file_.write(b'M %s %s %s\n' % (self.mode, self.blob_id, quoted_filename))
|
|
elif self.type == b'D':
|
|
file_.write(b'D %s\n' % quoted_filename)
|
|
elif self.type == b'DELETEALL':
|
|
file_.write(b'deleteall\n')
|
|
else:
|
|
raise SystemExit(_("Unhandled filechange type: %s") % self.type) # pragma: no cover
|
|
|
|
class Commit(_GitElementWithId):
|
|
"""
|
|
This class defines our representation of commit elements. Commit elements
|
|
contain all the information associated with a commit.
|
|
"""
|
|
|
|
def __init__(self, branch,
|
|
author_name, author_email, author_date,
|
|
committer_name, committer_email, committer_date,
|
|
message,
|
|
file_changes,
|
|
parents,
|
|
original_id = None,
|
|
encoding = None, # encoding for message; None implies UTF-8
|
|
**kwargs):
|
|
_GitElementWithId.__init__(self)
|
|
self.old_id = self.id
|
|
|
|
# Denote that this is a commit element
|
|
self.type = 'commit'
|
|
|
|
# Record the affected branch
|
|
self.branch = branch
|
|
|
|
# Record original id
|
|
self.original_id = original_id
|
|
|
|
# Record author's name
|
|
self.author_name = author_name
|
|
|
|
# Record author's email
|
|
self.author_email = author_email
|
|
|
|
# Record date of authoring
|
|
self.author_date = author_date
|
|
|
|
# Record committer's name
|
|
self.committer_name = committer_name
|
|
|
|
# Record committer's email
|
|
self.committer_email = committer_email
|
|
|
|
# Record date the commit was made
|
|
self.committer_date = committer_date
|
|
|
|
# Record commit message and its encoding
|
|
self.encoding = encoding
|
|
self.message = message
|
|
|
|
# List of file-changes associated with this commit. Note that file-changes
|
|
# are also represented as git elements
|
|
self.file_changes = file_changes
|
|
|
|
self.parents = parents
|
|
|
|
def dump(self, file_):
|
|
"""
|
|
Write this commit element to a file.
|
|
"""
|
|
self.dumped = 1
|
|
HASH_TO_ID[self.original_id] = self.id
|
|
ID_TO_HASH[self.id] = self.original_id
|
|
|
|
# Make output to fast-import slightly easier for humans to read if the
|
|
# message has no trailing newline of its own; cosmetic, but a nice touch...
|
|
extra_newline = b'\n'
|
|
if self.message.endswith(b'\n') or not (self.parents or self.file_changes):
|
|
extra_newline = b''
|
|
|
|
if not self.parents:
|
|
file_.write(b'reset %s\n' % self.branch)
|
|
file_.write((b'commit %s\n'
|
|
b'mark :%d\n'
|
|
b'author %s <%s> %s\n'
|
|
b'committer %s <%s> %s\n'
|
|
) % (
|
|
self.branch, self.id,
|
|
self.author_name, self.author_email, self.author_date,
|
|
self.committer_name, self.committer_email, self.committer_date
|
|
))
|
|
if self.encoding:
|
|
file_.write(b'encoding %s\n' % self.encoding)
|
|
file_.write(b'data %d\n%s%s' %
|
|
(len(self.message), self.message, extra_newline))
|
|
for i, parent in enumerate(self.parents):
|
|
file_.write(b'from ' if i==0 else b'merge ')
|
|
if isinstance(parent, int):
|
|
file_.write(b':%d\n' % parent)
|
|
else:
|
|
file_.write(b'%s\n' % parent)
|
|
for change in self.file_changes:
|
|
change.dump(file_)
|
|
if not self.parents and not self.file_changes:
|
|
# Workaround a bug in pre-git-2.22 versions of fast-import with
|
|
# the get-mark directive.
|
|
file_.write(b'\n')
|
|
file_.write(b'\n')
|
|
|
|
def first_parent(self):
|
|
"""
|
|
Return first parent commit
|
|
"""
|
|
if self.parents:
|
|
return self.parents[0]
|
|
return None
|
|
|
|
def skip(self, new_id=None):
|
|
_SKIPPED_COMMITS.add(self.old_id or self.id)
|
|
_GitElementWithId.skip(self, new_id)
|
|
|
|
class Tag(_GitElementWithId):
|
|
"""
|
|
This class defines our representation of annotated tag elements.
|
|
"""
|
|
|
|
def __init__(self, ref, from_ref,
|
|
tagger_name, tagger_email, tagger_date, tag_msg,
|
|
original_id = None):
|
|
_GitElementWithId.__init__(self)
|
|
self.old_id = self.id
|
|
|
|
# Denote that this is a tag element
|
|
self.type = 'tag'
|
|
|
|
# Store the name of the tag
|
|
self.ref = ref
|
|
|
|
# Store the entity being tagged (this should be a commit)
|
|
self.from_ref = from_ref
|
|
|
|
# Record original id
|
|
self.original_id = original_id
|
|
|
|
# Store the name of the tagger
|
|
self.tagger_name = tagger_name
|
|
|
|
# Store the email of the tagger
|
|
self.tagger_email = tagger_email
|
|
|
|
# Store the date
|
|
self.tagger_date = tagger_date
|
|
|
|
# Store the tag message
|
|
self.message = tag_msg
|
|
|
|
def dump(self, file_):
|
|
"""
|
|
Write this tag element to a file
|
|
"""
|
|
|
|
self.dumped = 1
|
|
HASH_TO_ID[self.original_id] = self.id
|
|
ID_TO_HASH[self.id] = self.original_id
|
|
|
|
file_.write(b'tag %s\n' % self.ref)
|
|
if (write_marks and self.id):
|
|
file_.write(b'mark :%d\n' % self.id)
|
|
markfmt = b'from :%d\n' if isinstance(self.from_ref, int) else b'from %s\n'
|
|
file_.write(markfmt % self.from_ref)
|
|
if self.tagger_name:
|
|
file_.write(b'tagger %s <%s> ' % (self.tagger_name, self.tagger_email))
|
|
file_.write(self.tagger_date)
|
|
file_.write(b'\n')
|
|
file_.write(b'data %d\n%s' % (len(self.message), self.message))
|
|
file_.write(b'\n')
|
|
|
|
class Progress(_GitElement):
|
|
"""
|
|
This class defines our representation of progress elements. The progress
|
|
element only contains a progress message, which is printed by fast-import
|
|
when it processes the progress output.
|
|
"""
|
|
|
|
def __init__(self, message):
|
|
_GitElement.__init__(self)
|
|
|
|
# Denote that this is a progress element
|
|
self.type = 'progress'
|
|
|
|
# Store the progress message
|
|
self.message = message
|
|
|
|
def dump(self, file_):
|
|
"""
|
|
Write this progress element to a file
|
|
"""
|
|
self.dumped = 1
|
|
|
|
file_.write(b'progress %s\n' % self.message)
|
|
file_.write(b'\n')
|
|
|
|
class Checkpoint(_GitElement):
|
|
"""
|
|
This class defines our representation of checkpoint elements. These
|
|
elements represent events which force fast-import to close the current
|
|
packfile, start a new one, and to save out all current branch refs, tags
|
|
and marks.
|
|
"""
|
|
|
|
def __init__(self):
|
|
_GitElement.__init__(self)
|
|
|
|
# Denote that this is a checkpoint element
|
|
self.type = 'checkpoint'
|
|
|
|
def dump(self, file_):
|
|
"""
|
|
Write this checkpoint element to a file
|
|
"""
|
|
self.dumped = 1
|
|
|
|
file_.write(b'checkpoint\n')
|
|
file_.write(b'\n')
|
|
|
|
class LiteralCommand(_GitElement):
|
|
"""
|
|
This class defines our representation of commands. The literal command
|
|
includes only a single line, and is not processed in any special way.
|
|
"""
|
|
|
|
def __init__(self, line):
|
|
_GitElement.__init__(self)
|
|
|
|
# Denote that this is a literal element
|
|
self.type = 'literal'
|
|
|
|
# Store the command
|
|
self.line = line
|
|
|
|
def dump(self, file_):
|
|
"""
|
|
Write this progress element to a file
|
|
"""
|
|
self.dumped = 1
|
|
|
|
file_.write(self.line)
|
|
|
|
class Alias(_GitElement):
|
|
"""
|
|
This class defines our representation of fast-import alias elements. An
|
|
alias element is the setting of one mark to the same sha1sum as another,
|
|
usually because the newer mark corresponded to a pruned commit.
|
|
"""
|
|
|
|
def __init__(self, ref, to_ref):
|
|
_GitElement.__init__(self)
|
|
# Denote that this is a reset
|
|
self.type = 'alias'
|
|
|
|
self.ref = ref
|
|
self.to_ref = to_ref
|
|
|
|
def dump(self, file_):
|
|
"""
|
|
Write this reset element to a file
|
|
"""
|
|
self.dumped = 1
|
|
|
|
file_.write(b'alias\nmark :%d\nto :%d\n\n' % (self.ref, self.to_ref))
|
|
|
|
class FastExportParser(object):
|
|
"""
|
|
A class for parsing and handling the output from fast-export. This
|
|
class allows the user to register callbacks when various types of
|
|
data are encountered in the fast-export output. The basic idea is that,
|
|
FastExportParser takes fast-export output, creates the various objects
|
|
as it encounters them, the user gets to use/modify these objects via
|
|
callbacks, and finally FastExportParser outputs the modified objects
|
|
in fast-import format (presumably so they can be used to create a new
|
|
repo).
|
|
"""
|
|
|
|
def __init__(self,
|
|
tag_callback = None, commit_callback = None,
|
|
blob_callback = None, progress_callback = None,
|
|
reset_callback = None, checkpoint_callback = None,
|
|
done_callback = None):
|
|
# Members below simply store callback functions for the various git
|
|
# elements
|
|
self._tag_callback = tag_callback
|
|
self._blob_callback = blob_callback
|
|
self._reset_callback = reset_callback
|
|
self._commit_callback = commit_callback
|
|
self._progress_callback = progress_callback
|
|
self._checkpoint_callback = checkpoint_callback
|
|
self._done_callback = done_callback
|
|
|
|
# Keep track of which refs appear from the export, and which make it to
|
|
# the import (pruning of empty commits, renaming of refs, and creating
|
|
# new manual objects and inserting them can cause these to differ).
|
|
self._exported_refs = set()
|
|
self._imported_refs = set()
|
|
|
|
# A list of the branches we've seen, plus the last known commit they
|
|
# pointed to. An entry in latest_*commit will be deleted if we get a
|
|
# reset for that branch. These are used because of fast-import's weird
|
|
# decision to allow having an implicit parent via naming the branch
|
|
# instead of requiring branches to be specified via 'from' directives.
|
|
self._latest_commit = {}
|
|
self._latest_orig_commit = {}
|
|
|
|
# A handle to the input source for the fast-export data
|
|
self._input = None
|
|
|
|
# A handle to the output file for the output we generate (we call dump
|
|
# on many of the git elements we create).
|
|
self._output = None
|
|
|
|
# Stores the contents of the current line of input being parsed
|
|
self._currentline = ''
|
|
|
|
# Compile some regexes and cache those
|
|
self._mark_re = re.compile(br'mark :(\d+)\n$')
|
|
self._parent_regexes = {}
|
|
parent_regex_rules = (br' :(\d+)\n$', br' ([0-9a-f]{40})\n')
|
|
for parent_refname in (b'from', b'merge'):
|
|
ans = [re.compile(parent_refname+x) for x in parent_regex_rules]
|
|
self._parent_regexes[parent_refname] = ans
|
|
self._quoted_string_re = re.compile(br'"(?:[^"\\]|\\.)*"')
|
|
self._refline_regexes = {}
|
|
for refline_name in (b'reset', b'commit', b'tag', b'progress'):
|
|
self._refline_regexes[refline_name] = re.compile(refline_name+b' (.*)\n$')
|
|
self._user_regexes = {}
|
|
for user in (b'author', b'committer', b'tagger'):
|
|
self._user_regexes[user] = re.compile(user + b' (.*?) <(.*?)> (.*)\n$')
|
|
|
|
def _advance_currentline(self):
|
|
"""
|
|
Grab the next line of input
|
|
"""
|
|
self._currentline = self._input.readline()
|
|
|
|
def _parse_optional_mark(self):
|
|
"""
|
|
If the current line contains a mark, parse it and advance to the
|
|
next line; return None otherwise
|
|
"""
|
|
mark = None
|
|
matches = self._mark_re.match(self._currentline)
|
|
if matches:
|
|
mark = int(matches.group(1))
|
|
self._advance_currentline()
|
|
return mark
|
|
|
|
def _parse_optional_parent_ref(self, refname):
|
|
"""
|
|
If the current line contains a reference to a parent commit, then
|
|
parse it and advance the current line; otherwise return None. Note
|
|
that the name of the reference ('from', 'merge') must match the
|
|
refname arg.
|
|
"""
|
|
orig_baseref, baseref = None, None
|
|
rule, altrule = self._parent_regexes[refname]
|
|
matches = rule.match(self._currentline)
|
|
if matches:
|
|
orig_baseref = int(matches.group(1))
|
|
# We translate the parent commit mark to what it needs to be in
|
|
# our mark namespace
|
|
baseref = _IDS.translate(orig_baseref)
|
|
self._advance_currentline()
|
|
else:
|
|
matches = altrule.match(self._currentline)
|
|
if matches:
|
|
orig_baseref = matches.group(1)
|
|
baseref = orig_baseref
|
|
self._advance_currentline()
|
|
return orig_baseref, baseref
|
|
|
|
def _parse_optional_filechange(self):
|
|
"""
|
|
If the current line contains a file-change object, then parse it
|
|
and advance the current line; otherwise return None. We only care
|
|
about file changes of type b'M' and b'D' (these are the only types
|
|
of file-changes that fast-export will provide).
|
|
"""
|
|
filechange = None
|
|
changetype = self._currentline[0:1]
|
|
if changetype == b'M':
|
|
(changetype, mode, idnum, path) = self._currentline.split(None, 3)
|
|
if idnum[0:1] == b':':
|
|
idnum = idnum[1:]
|
|
path = path.rstrip(b'\n')
|
|
# We translate the idnum to our id system
|
|
if len(idnum) != 40:
|
|
idnum = _IDS.translate( int(idnum) )
|
|
if idnum is not None:
|
|
if path.startswith(b'"'):
|
|
path = PathQuoting.dequote(path)
|
|
filechange = FileChange(b'M', path, idnum, mode)
|
|
else:
|
|
filechange = b'skipped'
|
|
self._advance_currentline()
|
|
elif changetype == b'D':
|
|
(changetype, path) = self._currentline.split(None, 1)
|
|
path = path.rstrip(b'\n')
|
|
if path.startswith(b'"'):
|
|
path = PathQuoting.dequote(path)
|
|
filechange = FileChange(b'D', path)
|
|
self._advance_currentline()
|
|
elif changetype == b'R': # pragma: no cover (now avoid fast-export renames)
|
|
rest = self._currentline[2:-1]
|
|
if rest.startswith(b'"'):
|
|
m = self._quoted_string_re.match(rest)
|
|
if not m:
|
|
raise SystemExit(_("Couldn't parse rename source"))
|
|
orig = PathQuoting.dequote(m.group(0))
|
|
new = rest[m.end()+1:]
|
|
else:
|
|
orig, new = rest.split(b' ', 1)
|
|
if new.startswith(b'"'):
|
|
new = PathQuoting.dequote(new)
|
|
filechange = FileChange(b'R', orig, new)
|
|
self._advance_currentline()
|
|
return filechange
|
|
|
|
def _parse_original_id(self):
|
|
original_id = self._currentline[len(b'original-oid '):].rstrip()
|
|
self._advance_currentline()
|
|
return original_id
|
|
|
|
def _parse_encoding(self):
|
|
encoding = self._currentline[len(b'encoding '):].rstrip()
|
|
self._advance_currentline()
|
|
return encoding
|
|
|
|
def _parse_ref_line(self, refname):
|
|
"""
|
|
Parses string data (often a branch name) from current-line. The name of
|
|
the string data must match the refname arg. The program will crash if
|
|
current-line does not match, so current-line will always be advanced if
|
|
this method returns.
|
|
"""
|
|
matches = self._refline_regexes[refname].match(self._currentline)
|
|
if not matches:
|
|
raise SystemExit(_("Malformed %(refname)s line: '%(line)s'") %
|
|
({'refname': refname, 'line':self._currentline})
|
|
) # pragma: no cover
|
|
ref = matches.group(1)
|
|
self._advance_currentline()
|
|
return ref
|
|
|
|
def _parse_user(self, usertype):
|
|
"""
|
|
Get user name, email, datestamp from current-line. Current-line will
|
|
be advanced.
|
|
"""
|
|
user_regex = self._user_regexes[usertype]
|
|
(name, email, when) = user_regex.match(self._currentline).groups()
|
|
|
|
self._advance_currentline()
|
|
return (name, email, when)
|
|
|
|
def _parse_data(self):
|
|
"""
|
|
Reads data from _input. Current-line will be advanced until it is beyond
|
|
the data.
|
|
"""
|
|
fields = self._currentline.split()
|
|
assert fields[0] == b'data'
|
|
size = int(fields[1])
|
|
data = self._input.read(size)
|
|
self._advance_currentline()
|
|
if self._currentline == b'\n':
|
|
self._advance_currentline()
|
|
return data
|
|
|
|
def _parse_blob(self):
|
|
"""
|
|
Parse input data into a Blob object. Once the Blob has been created, it
|
|
will be handed off to the appropriate callbacks. Current-line will be
|
|
advanced until it is beyond this blob's data. The Blob will be dumped
|
|
to _output once everything else is done (unless it has been skipped by
|
|
the callback).
|
|
"""
|
|
# Parse the Blob
|
|
self._advance_currentline()
|
|
id_ = self._parse_optional_mark()
|
|
|
|
original_id = None
|
|
if self._currentline.startswith(b'original-oid'):
|
|
original_id = self._parse_original_id();
|
|
|
|
data = self._parse_data()
|
|
if self._currentline == b'\n':
|
|
self._advance_currentline()
|
|
|
|
# Create the blob
|
|
blob = Blob(data, original_id)
|
|
|
|
# If fast-export text had a mark for this blob, need to make sure this
|
|
# mark translates to the blob's true id.
|
|
if id_:
|
|
blob.old_id = id_
|
|
_IDS.record_rename(id_, blob.id)
|
|
|
|
# Call any user callback to allow them to use/modify the blob
|
|
if self._blob_callback:
|
|
self._blob_callback(blob)
|
|
|
|
# Now print the resulting blob
|
|
if not blob.dumped:
|
|
blob.dump(self._output)
|
|
|
|
def _parse_reset(self):
|
|
"""
|
|
Parse input data into a Reset object. Once the Reset has been created,
|
|
it will be handed off to the appropriate callbacks. Current-line will
|
|
be advanced until it is beyond the reset data. The Reset will be dumped
|
|
to _output once everything else is done (unless it has been skipped by
|
|
the callback).
|
|
"""
|
|
# Parse the Reset
|
|
ref = self._parse_ref_line(b'reset')
|
|
self._exported_refs.add(ref)
|
|
ignoreme, from_ref = self._parse_optional_parent_ref(b'from')
|
|
if self._currentline == b'\n':
|
|
self._advance_currentline()
|
|
|
|
# fast-export likes to print extraneous resets that serve no purpose.
|
|
# While we could continue processing such resets, that is a waste of
|
|
# resources. Also, we want to avoid recording that this ref was
|
|
# seen in such cases, since this ref could be rewritten to nothing.
|
|
if not from_ref:
|
|
self._latest_commit.pop(ref, None)
|
|
self._latest_orig_commit.pop(ref, None)
|
|
return
|
|
|
|
# Create the reset
|
|
reset = Reset(ref, from_ref)
|
|
|
|
# Call any user callback to allow them to modify the reset
|
|
if self._reset_callback:
|
|
self._reset_callback(reset)
|
|
|
|
# Update metadata
|
|
self._latest_commit[reset.ref] = reset.from_ref
|
|
self._latest_orig_commit[reset.ref] = reset.from_ref
|
|
|
|
# Now print the resulting reset
|
|
if not reset.dumped:
|
|
self._imported_refs.add(reset.ref)
|
|
reset.dump(self._output)
|
|
|
|
def _parse_commit(self):
|
|
"""
|
|
Parse input data into a Commit object. Once the Commit has been created,
|
|
it will be handed off to the appropriate callbacks. Current-line will
|
|
be advanced until it is beyond the commit data. The Commit will be dumped
|
|
to _output once everything else is done (unless it has been skipped by
|
|
the callback OR the callback has removed all file-changes from the commit).
|
|
"""
|
|
# Parse the Commit. This may look involved, but it's pretty simple; it only
|
|
# looks bad because a commit object contains many pieces of data.
|
|
branch = self._parse_ref_line(b'commit')
|
|
self._exported_refs.add(branch)
|
|
id_ = self._parse_optional_mark()
|
|
|
|
original_id = None
|
|
if self._currentline.startswith(b'original-oid'):
|
|
original_id = self._parse_original_id();
|
|
|
|
author_name = None
|
|
author_email = None
|
|
if self._currentline.startswith(b'author'):
|
|
(author_name, author_email, author_date) = self._parse_user(b'author')
|
|
|
|
(committer_name, committer_email, committer_date) = \
|
|
self._parse_user(b'committer')
|
|
|
|
if not author_name and not author_email:
|
|
(author_name, author_email, author_date) = \
|
|
(committer_name, committer_email, committer_date)
|
|
|
|
encoding = None
|
|
if self._currentline.startswith(b'encoding '):
|
|
encoding = self._parse_encoding()
|
|
|
|
commit_msg = self._parse_data()
|
|
|
|
pinfo = [self._parse_optional_parent_ref(b'from')]
|
|
# Due to empty pruning, we can have real 'from' and 'merge' lines that
|
|
# due to commit rewriting map to a parent of None. We need to record
|
|
# 'from' if its non-None, and we need to parse all 'merge' lines.
|
|
while self._currentline.startswith(b'merge '):
|
|
pinfo.append(self._parse_optional_parent_ref(b'merge'))
|
|
orig_parents, parents = [list(tmp) for tmp in zip(*pinfo)]
|
|
|
|
# No parents is oddly represented as [None] instead of [], due to the
|
|
# special 'from' handling. Convert it here to a more canonical form.
|
|
if parents == [None]:
|
|
parents = []
|
|
if orig_parents == [None]:
|
|
orig_parents = []
|
|
|
|
# fast-import format is kinda stupid in that it allows implicit parents
|
|
# based on the branch name instead of requiring them to be specified by
|
|
# 'from' directives. The only way to get no parent is by using a reset
|
|
# directive first, which clears the latest_commit_for_this_branch tracking.
|
|
if not orig_parents and self._latest_commit.get(branch):
|
|
parents = [self._latest_commit[branch]]
|
|
if not orig_parents and self._latest_orig_commit.get(branch):
|
|
orig_parents = [self._latest_orig_commit[branch]]
|
|
|
|
# Get the list of file changes
|
|
file_changes = []
|
|
file_change = self._parse_optional_filechange()
|
|
had_file_changes = file_change is not None
|
|
while file_change:
|
|
if not (type(file_change) == bytes and file_change == b'skipped'):
|
|
file_changes.append(file_change)
|
|
file_change = self._parse_optional_filechange()
|
|
if self._currentline == b'\n':
|
|
self._advance_currentline()
|
|
|
|
# Okay, now we can finally create the Commit object
|
|
commit = Commit(branch,
|
|
author_name, author_email, author_date,
|
|
committer_name, committer_email, committer_date,
|
|
commit_msg, file_changes, parents, original_id, encoding)
|
|
|
|
# If fast-export text had a mark for this commit, need to make sure this
|
|
# mark translates to the commit's true id.
|
|
if id_:
|
|
commit.old_id = id_
|
|
_IDS.record_rename(id_, commit.id)
|
|
|
|
# Call any user callback to allow them to modify the commit
|
|
aux_info = {'orig_parents': orig_parents,
|
|
'had_file_changes': had_file_changes}
|
|
if self._commit_callback:
|
|
self._commit_callback(commit, aux_info)
|
|
|
|
# Now print the resulting commit, or if prunable skip it
|
|
self._latest_orig_commit[branch] = commit.id
|
|
if not (commit.old_id or commit.id) in _SKIPPED_COMMITS:
|
|
self._latest_commit[branch] = commit.id
|
|
if not commit.dumped:
|
|
self._imported_refs.add(commit.branch)
|
|
commit.dump(self._output)
|
|
|
|
def _parse_tag(self):
|
|
"""
|
|
Parse input data into a Tag object. Once the Tag has been created,
|
|
it will be handed off to the appropriate callbacks. Current-line will
|
|
be advanced until it is beyond the tag data. The Tag will be dumped
|
|
to _output once everything else is done (unless it has been skipped by
|
|
the callback).
|
|
"""
|
|
# Parse the Tag
|
|
tag = self._parse_ref_line(b'tag')
|
|
self._exported_refs.add(b'refs/tags/'+tag)
|
|
id_ = self._parse_optional_mark()
|
|
ignoreme, from_ref = self._parse_optional_parent_ref(b'from')
|
|
|
|
original_id = None
|
|
if self._currentline.startswith(b'original-oid'):
|
|
original_id = self._parse_original_id();
|
|
|
|
tagger_name, tagger_email, tagger_date = None, None, None
|
|
if self._currentline.startswith(b'tagger'):
|
|
(tagger_name, tagger_email, tagger_date) = self._parse_user(b'tagger')
|
|
tag_msg = self._parse_data()
|
|
if self._currentline == b'\n':
|
|
self._advance_currentline()
|
|
|
|
# Create the tag
|
|
tag = Tag(tag, from_ref,
|
|
tagger_name, tagger_email, tagger_date, tag_msg,
|
|
original_id)
|
|
|
|
# If fast-export text had a mark for this tag, need to make sure this
|
|
# mark translates to the tag's true id.
|
|
if id_:
|
|
tag.old_id = id_
|
|
_IDS.record_rename(id_, tag.id)
|
|
|
|
# Call any user callback to allow them to modify the tag
|
|
if self._tag_callback:
|
|
self._tag_callback(tag)
|
|
|
|
# The tag might not point at anything that still exists (self.from_ref
|
|
# will be None if the commit it pointed to and all its ancestors were
|
|
# pruned due to being empty)
|
|
if tag.from_ref:
|
|
# Print out this tag's information
|
|
if not tag.dumped:
|
|
self._imported_refs.add(b'refs/tags/'+tag.ref)
|
|
tag.dump(self._output)
|
|
else:
|
|
tag.skip()
|
|
|
|
def _parse_progress(self):
|
|
"""
|
|
Parse input data into a Progress object. Once the Progress has
|
|
been created, it will be handed off to the appropriate
|
|
callbacks. Current-line will be advanced until it is beyond the
|
|
progress data. The Progress will be dumped to _output once
|
|
everything else is done (unless it has been skipped by the callback).
|
|
"""
|
|
# Parse the Progress
|
|
message = self._parse_ref_line(b'progress')
|
|
if self._currentline == b'\n':
|
|
self._advance_currentline()
|
|
|
|
# Create the progress message
|
|
progress = Progress(message)
|
|
|
|
# Call any user callback to allow them to modify the progress messsage
|
|
if self._progress_callback:
|
|
self._progress_callback(progress)
|
|
|
|
# NOTE: By default, we do NOT print the progress message; git
|
|
# fast-import would write it to fast_import_pipes which could mess with
|
|
# our parsing of output from the 'ls' and 'get-mark' directives we send
|
|
# to fast-import. If users want these messages, they need to process
|
|
# and handle them in the appropriate callback above.
|
|
|
|
def _parse_checkpoint(self):
|
|
"""
|
|
Parse input data into a Checkpoint object. Once the Checkpoint has
|
|
been created, it will be handed off to the appropriate
|
|
callbacks. Current-line will be advanced until it is beyond the
|
|
checkpoint data. The Checkpoint will be dumped to _output once
|
|
everything else is done (unless it has been skipped by the callback).
|
|
"""
|
|
# Parse the Checkpoint
|
|
self._advance_currentline()
|
|
if self._currentline == b'\n':
|
|
self._advance_currentline()
|
|
|
|
# Create the checkpoint
|
|
checkpoint = Checkpoint()
|
|
|
|
# Call any user callback to allow them to drop the checkpoint
|
|
if self._checkpoint_callback:
|
|
self._checkpoint_callback(checkpoint)
|
|
|
|
# NOTE: By default, we do NOT print the checkpoint message; although it
|
|
# we would only realistically get them with --stdin, the fact that we
|
|
# are filtering makes me think the checkpointing is less likely to be
|
|
# reasonable. In fact, I don't think it's necessary in general. If
|
|
# users do want it, they should process it in the checkpoint_callback.
|
|
|
|
def _parse_literal_command(self):
|
|
"""
|
|
Parse literal command. Then just dump the line as is.
|
|
"""
|
|
# Create the literal command object
|
|
command = LiteralCommand(self._currentline)
|
|
self._advance_currentline()
|
|
|
|
# Now print the resulting literal command
|
|
if not command.dumped:
|
|
command.dump(self._output)
|
|
|
|
def insert(self, obj):
|
|
assert not obj.dumped
|
|
obj.dump(self._output)
|
|
if type(obj) == Commit:
|
|
self._imported_refs.add(obj.branch)
|
|
elif type(obj) in (Reset, Tag):
|
|
self._imported_refs.add(obj.ref)
|
|
|
|
def run(self, input, output):
|
|
"""
|
|
This method filters fast export output.
|
|
"""
|
|
# Set input. If no args provided, use stdin.
|
|
self._input = input
|
|
self._output = output
|
|
|
|
# Run over the input and do the filtering
|
|
self._advance_currentline()
|
|
while self._currentline:
|
|
if self._currentline.startswith(b'blob'):
|
|
self._parse_blob()
|
|
elif self._currentline.startswith(b'reset'):
|
|
self._parse_reset()
|
|
elif self._currentline.startswith(b'commit'):
|
|
self._parse_commit()
|
|
elif self._currentline.startswith(b'tag'):
|
|
self._parse_tag()
|
|
elif self._currentline.startswith(b'progress'):
|
|
self._parse_progress()
|
|
elif self._currentline.startswith(b'checkpoint'):
|
|
self._parse_checkpoint()
|
|
elif self._currentline.startswith(b'feature'):
|
|
self._parse_literal_command()
|
|
elif self._currentline.startswith(b'option'):
|
|
self._parse_literal_command()
|
|
elif self._currentline.startswith(b'done'):
|
|
if self._done_callback:
|
|
self._done_callback()
|
|
self._parse_literal_command()
|
|
# Prevent confusion from others writing additional stuff that'll just
|
|
# be ignored
|
|
self._output.close()
|
|
elif self._currentline.startswith(b'#'):
|
|
self._parse_literal_command()
|
|
elif self._currentline.startswith(b'get-mark') or \
|
|
self._currentline.startswith(b'cat-blob') or \
|
|
self._currentline.startswith(b'ls'):
|
|
raise SystemExit(_("Unsupported command: '%s'") % self._currentline)
|
|
else:
|
|
raise SystemExit(_("Could not parse line: '%s'") % self._currentline)
|
|
|
|
def get_exported_and_imported_refs(self):
|
|
return self._exported_refs, self._imported_refs
|
|
|
|
def record_id_rename(old_id, new_id):
|
|
"""
|
|
Register a new translation
|
|
"""
|
|
handle_transitivity = True
|
|
_IDS.record_rename(old_id, new_id, handle_transitivity)
|
|
|
|
# Internal globals
|
|
_IDS = _IDs()
|
|
_SKIPPED_COMMITS = set()
|
|
HASH_TO_ID = {}
|
|
ID_TO_HASH = {}
|
|
|
|
class SubprocessWrapper(object):
|
|
@staticmethod
|
|
def decodify(args):
|
|
if type(args) == str:
|
|
return args
|
|
else:
|
|
assert type(args) == list
|
|
return [decode(x) if type(x)==bytes else x for x in args]
|
|
|
|
@staticmethod
|
|
def call(*args, **kwargs):
|
|
if 'cwd' in kwargs:
|
|
kwargs['cwd'] = decode(kwargs['cwd'])
|
|
return subprocess.call(SubprocessWrapper.decodify(*args), **kwargs)
|
|
|
|
@staticmethod
|
|
def check_output(*args, **kwargs):
|
|
if 'cwd' in kwargs:
|
|
kwargs['cwd'] = decode(kwargs['cwd'])
|
|
return subprocess.check_output(SubprocessWrapper.decodify(*args), **kwargs)
|
|
|
|
@staticmethod
|
|
def check_call(*args, **kwargs): # pragma: no cover # used by filter-lamely
|
|
if 'cwd' in kwargs:
|
|
kwargs['cwd'] = decode(kwargs['cwd'])
|
|
return subprocess.check_call(SubprocessWrapper.decodify(*args), **kwargs)
|
|
|
|
@staticmethod
|
|
def Popen(*args, **kwargs):
|
|
if 'cwd' in kwargs:
|
|
kwargs['cwd'] = decode(kwargs['cwd'])
|
|
return subprocess.Popen(SubprocessWrapper.decodify(*args), **kwargs)
|
|
|
|
subproc = subprocess
|
|
if platform.system() == 'Windows' or 'PRETEND_UNICODE_ARGS' in os.environ:
|
|
subproc = SubprocessWrapper
|
|
|
|
class GitUtils(object):
|
|
@staticmethod
|
|
def get_commit_count(repo, *args):
|
|
"""
|
|
Return the number of commits that have been made on repo.
|
|
"""
|
|
if not args:
|
|
args = ['--all']
|
|
if len(args) == 1 and isinstance(args[0], list):
|
|
args = args[0]
|
|
p = subproc.Popen(["git", "rev-list", "--count"] + args,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
cwd=repo)
|
|
if p.wait() != 0:
|
|
raise SystemExit(_("%s does not appear to be a valid git repository")
|
|
% decode(repo))
|
|
return int(p.stdout.read())
|
|
|
|
@staticmethod
|
|
def get_total_objects(repo):
|
|
"""
|
|
Return the number of objects (both packed and unpacked)
|
|
"""
|
|
p1 = subproc.Popen(["git", "count-objects", "-v"],
|
|
stdout=subprocess.PIPE, cwd=repo)
|
|
lines = p1.stdout.read().splitlines()
|
|
# Return unpacked objects + packed-objects
|
|
return int(lines[0].split()[1]) + int(lines[2].split()[1])
|
|
|
|
@staticmethod
|
|
def is_repository_bare(repo_working_dir):
|
|
out = subproc.check_output('git rev-parse --is-bare-repository'.split(),
|
|
cwd=repo_working_dir)
|
|
return (out.strip() == b'true')
|
|
|
|
@staticmethod
|
|
def determine_git_dir(repo_working_dir):
|
|
d = subproc.check_output('git rev-parse --git-dir'.split(),
|
|
cwd=repo_working_dir).strip()
|
|
if repo_working_dir==b'.' or d.startswith(b'/'):
|
|
return d
|
|
return os.path.join(repo_working_dir, d)
|
|
|
|
@staticmethod
|
|
def get_refs(repo_working_dir):
|
|
try:
|
|
output = subproc.check_output('git show-ref'.split(),
|
|
cwd=repo_working_dir)
|
|
except subprocess.CalledProcessError as e:
|
|
# If error code is 1, there just aren't any refs; i.e. new repo.
|
|
# If error code is other than 1, some other error (e.g. not a git repo)
|
|
if e.returncode != 1:
|
|
raise SystemExit('fatal: {}'.format(e))
|
|
output = ''
|
|
return dict(reversed(x.split()) for x in output.splitlines())
|
|
|
|
@staticmethod
|
|
def get_blob_sizes(quiet = False):
|
|
blob_size_progress = ProgressWriter()
|
|
num_blobs = 0
|
|
processed_blobs_msg = _("Processed %d blob sizes")
|
|
|
|
# Get sizes of blobs by sha1
|
|
cmd = '--batch-check=%(objectname) %(objecttype) ' + \
|
|
'%(objectsize) %(objectsize:disk)'
|
|
cf = subproc.Popen(['git', 'cat-file', '--batch-all-objects', cmd],
|
|
bufsize = -1,
|
|
stdout = subprocess.PIPE)
|
|
unpacked_size = {}
|
|
packed_size = {}
|
|
for line in cf.stdout:
|
|
sha, objtype, objsize, objdisksize = line.split()
|
|
objsize, objdisksize = int(objsize), int(objdisksize)
|
|
if objtype == b'blob':
|
|
unpacked_size[sha] = objsize
|
|
packed_size[sha] = objdisksize
|
|
num_blobs += 1
|
|
if not quiet:
|
|
blob_size_progress.show(processed_blobs_msg % num_blobs)
|
|
cf.wait()
|
|
if not quiet:
|
|
blob_size_progress.finish()
|
|
return unpacked_size, packed_size
|
|
|
|
@staticmethod
|
|
def get_file_changes(repo, parent_hash, commit_hash):
|
|
"""
|
|
Return a FileChanges list with the differences between parent_hash
|
|
and commit_hash
|
|
"""
|
|
file_changes = []
|
|
|
|
cmd = ["git", "diff-tree", "-r", parent_hash, commit_hash]
|
|
output = subproc.check_output(cmd, cwd=repo)
|
|
for line in output.splitlines():
|
|
fileinfo, path = line.split(b'\t', 1)
|
|
if path.startswith(b'"'):
|
|
path = PathQuoting.dequote(path)
|
|
oldmode, mode, oldhash, newhash, changetype = fileinfo.split()
|
|
if changetype == b'D':
|
|
file_changes.append(FileChange(b'D', path))
|
|
elif changetype in (b'A', b'M', b'T'):
|
|
identifier = HASH_TO_ID.get(newhash, newhash)
|
|
file_changes.append(FileChange(b'M', path, identifier, mode))
|
|
else: # pragma: no cover
|
|
raise SystemExit("Unknown change type for line {}".format(line))
|
|
|
|
return file_changes
|
|
|
|
@staticmethod
|
|
def print_my_version():
|
|
with open(__file__, 'br') as f:
|
|
contents = f.read()
|
|
# If people replaced @@LOCALEDIR@@ string to point at their local
|
|
# directory, undo it so we can get original source version.
|
|
contents = re.sub(br'\A#\!.*',
|
|
br'#!/usr/bin/env python3', contents)
|
|
contents = re.sub(br'(\("GIT_TEXTDOMAINDIR"\) or ").*"',
|
|
br'\1@@LOCALEDIR@@"', contents)
|
|
|
|
cmd = 'git hash-object --stdin'.split()
|
|
version = subproc.check_output(cmd, input=contents).strip()
|
|
print(decode(version[0:12]))
|
|
|
|
class FilteringOptions(object):
|
|
default_replace_text = b'***REMOVED***'
|
|
class AppendFilter(argparse.Action):
|
|
def __call__(self, parser, namespace, values, option_string=None):
|
|
user_path = values
|
|
suffix = option_string[len('--path-'):] or 'match'
|
|
if suffix.startswith('rename'):
|
|
mod_type = 'rename'
|
|
match_type = option_string[len('--path-rename-'):] or 'match'
|
|
values = values.split(b':')
|
|
if len(values) != 2:
|
|
raise SystemExit(_("Error: --path-rename expects one colon in its"
|
|
" argument: <old_name:new_name>."))
|
|
if values[0] and values[1] and not (
|
|
values[0].endswith(b'/') == values[1].endswith(b'/')):
|
|
raise SystemExit(_("Error: With --path-rename, if OLD_NAME and "
|
|
"NEW_NAME are both non-empty and either ends "
|
|
"with a slash then both must."))
|
|
if any(v.startswith(b'/') for v in values):
|
|
raise SystemExit(_("Error: Pathnames cannot begin with a '/'"))
|
|
components = values[0].split(b'/') + values[1].split(b'/')
|
|
else:
|
|
mod_type = 'filter'
|
|
match_type = suffix
|
|
components = values.split(b'/')
|
|
if values.startswith(b'/'):
|
|
raise SystemExit(_("Error: Pathnames cannot begin with a '/'"))
|
|
for illegal_path in [b'.', b'..']:
|
|
if illegal_path in components:
|
|
raise SystemExit(_("Error: Invalid path component '%s' found in '%s'")
|
|
% (decode(illegal_path), decode(user_path)))
|
|
if match_type == 'regex':
|
|
values = re.compile(values)
|
|
items = getattr(namespace, self.dest, []) or []
|
|
items.append((mod_type, match_type, values))
|
|
if (match_type, mod_type) == ('glob', 'filter'):
|
|
if not values.endswith(b'*'):
|
|
extension = b'*' if values.endswith(b'/') else b'/*'
|
|
items.append((mod_type, match_type, values+extension))
|
|
setattr(namespace, self.dest, items)
|
|
|
|
class HelperFilter(argparse.Action):
|
|
def __call__(self, parser, namespace, values, option_string=None):
|
|
af = FilteringOptions.AppendFilter(dest='path_changes',
|
|
option_strings=None)
|
|
dirname = values if values[-1:] == b'/' else values+b'/'
|
|
if option_string == '--subdirectory-filter':
|
|
af(parser, namespace, dirname, '--path-match')
|
|
af(parser, namespace, dirname+b':', '--path-rename')
|
|
elif option_string == '--to-subdirectory-filter':
|
|
af(parser, namespace, b':'+dirname, '--path-rename')
|
|
else:
|
|
raise SystemExit(_("Error: HelperFilter given invalid option_string: %s")
|
|
% option_string) # pragma: no cover
|
|
|
|
class FileWithPathsFilter(argparse.Action):
|
|
def __call__(self, parser, namespace, values, option_string=None):
|
|
if not namespace.path_changes:
|
|
namespace.path_changes = []
|
|
namespace.path_changes += FilteringOptions.get_paths_from_file(values)
|
|
|
|
@staticmethod
|
|
def create_arg_parser():
|
|
# Include usage in the summary, so we can put the description first
|
|
summary = _('''Rewrite (or analyze) repository history
|
|
|
|
git-filter-repo destructively rewrites history (unless --analyze or
|
|
--dry-run are given) according to specified rules. It refuses to do any
|
|
rewriting unless either run from a clean fresh clone, or --force was
|
|
given.
|
|
|
|
Basic Usage:
|
|
git-filter-repo --analyze
|
|
git-filter-repo [FILTER/RENAME/CONTROL OPTIONS]
|
|
|
|
See EXAMPLES section for details.
|
|
''').rstrip()
|
|
|
|
# Provide a long helpful examples section
|
|
example_text = _('''CALLBACKS
|
|
|
|
All callback functions are of the same general format. For a command line
|
|
argument like
|
|
--foo-callback 'BODY'
|
|
|
|
the following code will be compiled and called:
|
|
def foo_callback(foo):
|
|
BODY
|
|
|
|
Thus, to replace 'Jon' with 'John' in author/committer/tagger names:
|
|
git filter-repo --name-callback 'return name.replace(b"Jon", b"John")'
|
|
|
|
To remove all 'Tested-by' tags in commit (or tag) messages:
|
|
git filter-repo --message-callback 'return re.sub(br"\\nTested-by:.*", "", message)'
|
|
|
|
To remove all .DS_Store files:
|
|
git filter-repo --filename-callback 'return None if os.path.basename(filename) == b".DS_Store" else filename'
|
|
|
|
Note that if BODY resolves to a filename, then the contents of that file
|
|
will be used as the BODY in the callback function.
|
|
|
|
For more detailed examples and explanations AND caveats, see
|
|
https://htmlpreview.github.io/?https://github.com/newren/git-filter-repo/blob/docs/html/git-filter-repo.html#CALLBACKS
|
|
|
|
EXAMPLES
|
|
|
|
To get a bunch of reports mentioning renames that have occurred in
|
|
your repo and listing sizes of objects aggregated by any of path,
|
|
directory, extension, or blob-id:
|
|
git filter-repo --analyze
|
|
|
|
(These reports can help you choose how to filter your repo; it can
|
|
be useful to re-run this command after filtering to regenerate the
|
|
report and verify the changes look correct.)
|
|
|
|
To extract the history that touched just 'guides' and 'tools/releases':
|
|
git filter-repo --path guides/ --path tools/releases
|
|
|
|
To remove foo.zip and bar/baz/zips from every revision in history:
|
|
git filter-repo --path foo.zip --path bar/baz/zips/ --invert-paths
|
|
|
|
To replace the text 'password' with 'p455w0rd':
|
|
git filter-repo --replace-text <(echo "password==>p455w0rd")
|
|
|
|
To use the current version of the .mailmap file to update authors,
|
|
committers, and taggers throughout history and make it permanent:
|
|
git filter-repo --use-mailmap
|
|
|
|
To extract the history of 'src/', rename all files to have a new leading
|
|
directory 'my-module' (e.g. src/foo.java -> my-module/src/foo.java), and
|
|
add a 'my-module-' prefix to all tags:
|
|
git filter-repo --path src/ --to-subdirectory-filter my-module --tag-rename '':'my-module-'
|
|
|
|
For more detailed examples and explanations, see
|
|
https://htmlpreview.github.io/?https://github.com/newren/git-filter-repo/blob/docs/html/git-filter-repo.html#EXAMPLES''')
|
|
|
|
# Create the basic parser
|
|
parser = argparse.ArgumentParser(description=summary,
|
|
usage = argparse.SUPPRESS,
|
|
add_help = False,
|
|
epilog = example_text,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
|
|
analyze = parser.add_argument_group(title=_("Analysis"))
|
|
analyze.add_argument('--analyze', action='store_true',
|
|
help=_("Analyze repository history and create a report that may be "
|
|
"useful in determining what to filter in a subsequent run. "
|
|
"Will not modify your repo."))
|
|
analyze.add_argument('--report-dir',
|
|
metavar='DIR_OR_FILE',
|
|
type=os.fsencode,
|
|
dest='report_dir',
|
|
help=_("Directory to write report, defaults to GIT_DIR/filter_repo/analysis,"
|
|
"refuses to run if exists, --force delete existing dir first."))
|
|
|
|
path = parser.add_argument_group(title=_("Filtering based on paths "
|
|
"(see also --filename-callback)"),
|
|
description=textwrap.dedent(_("""
|
|
These options specify the paths to select. Note that much like git
|
|
itself, renames are NOT followed so you may need to specify multiple
|
|
paths, e.g. `--path olddir/ --path newdir/`
|
|
"""[1:])))
|
|
|
|
path.add_argument('--invert-paths', action='store_false', dest='inclusive',
|
|
help=_("Invert the selection of files from the specified "
|
|
"--path-{match,glob,regex} options below, i.e. only select "
|
|
"files matching none of those options."))
|
|
|
|
path.add_argument('--path-match', '--path', metavar='DIR_OR_FILE',
|
|
type=os.fsencode,
|
|
action=FilteringOptions.AppendFilter, dest='path_changes',
|
|
help=_("Exact paths (files or directories) to include in filtered "
|
|
"history. Multiple --path options can be specified to get "
|
|
"a union of paths."))
|
|
path.add_argument('--path-glob', metavar='GLOB', type=os.fsencode,
|
|
action=FilteringOptions.AppendFilter, dest='path_changes',
|
|
help=_("Glob of paths to include in filtered history. Multiple "
|
|
"--path-glob options can be specified to get a union of "
|
|
"paths."))
|
|
path.add_argument('--path-regex', metavar='REGEX', type=os.fsencode,
|
|
action=FilteringOptions.AppendFilter, dest='path_changes',
|
|
help=_("Regex of paths to include in filtered history. Multiple "
|
|
"--path-regex options can be specified to get a union of "
|
|
"paths"))
|
|
path.add_argument('--use-base-name', action='store_true',
|
|
help=_("Match on file base name instead of full path from the top "
|
|
"of the repo. Incompatible with --path-rename, and "
|
|
"incompatible with matching against directory names."))
|
|
|
|
rename = parser.add_argument_group(title=_("Renaming based on paths "
|
|
"(see also --filename-callback)"))
|
|
rename.add_argument('--path-rename', '--path-rename-match',
|
|
metavar='OLD_NAME:NEW_NAME', dest='path_changes', type=os.fsencode,
|
|
action=FilteringOptions.AppendFilter,
|
|
help=_("Path to rename; if filename or directory matches OLD_NAME "
|
|
"rename to NEW_NAME. Multiple --path-rename options can be "
|
|
"specified. NOTE: If you combine filtering options with "
|
|
"renaming ones, do not rely on a rename argument to select "
|
|
"paths; you also need a filter to select them."))
|
|
|
|
helpers = parser.add_argument_group(title=_("Path shortcuts"))
|
|
helpers.add_argument('--paths-from-file', metavar='FILENAME',
|
|
type=os.fsencode,
|
|
action=FilteringOptions.FileWithPathsFilter, dest='path_changes',
|
|
help=_("Specify several path filtering and renaming directives, one "
|
|
"per line. Lines with '==>' in them specify path renames, "
|
|
"and lines can begin with 'literal:' (the default), 'glob:', "
|
|
"or 'regex:' to specify different matching styles. Blank "
|
|
"lines and lines starting with a '#' are ignored."))
|
|
helpers.add_argument('--subdirectory-filter', metavar='DIRECTORY',
|
|
action=FilteringOptions.HelperFilter, type=os.fsencode,
|
|
help=_("Only look at history that touches the given subdirectory "
|
|
"and treat that directory as the project root. Equivalent "
|
|
"to using '--path DIRECTORY/ --path-rename DIRECTORY/:'"))
|
|
helpers.add_argument('--to-subdirectory-filter', metavar='DIRECTORY',
|
|
action=FilteringOptions.HelperFilter, type=os.fsencode,
|
|
help=_("Treat the project root as instead being under DIRECTORY. "
|
|
"Equivalent to using '--path-rename :DIRECTORY/'"))
|
|
|
|
contents = parser.add_argument_group(title=_("Content editing filters "
|
|
"(see also --blob-callback)"))
|
|
contents.add_argument('--replace-text', metavar='EXPRESSIONS_FILE',
|
|
help=_("A file with expressions that, if found, will be replaced. "
|
|
"By default, each expression is treated as literal text, "
|
|
"but 'regex:' and 'glob:' prefixes are supported. You can "
|
|
"end the line with '==>' and some replacement text to "
|
|
"choose a replacement choice other than the default of '{}'."
|
|
.format(decode(FilteringOptions.default_replace_text))))
|
|
contents.add_argument('--strip-blobs-bigger-than', metavar='SIZE',
|
|
dest='max_blob_size', default=0,
|
|
help=_("Strip blobs (files) bigger than specified size (e.g. '5M', "
|
|
"'2G', etc)"))
|
|
contents.add_argument('--strip-blobs-with-ids', metavar='BLOB-ID-FILENAME',
|
|
help=_("Read git object ids from each line of the given file, and "
|
|
"strip all of them from history"))
|
|
|
|
refrename = parser.add_argument_group(title=_("Renaming of refs "
|
|
"(see also --refname-callback)"))
|
|
refrename.add_argument('--tag-rename', metavar='OLD:NEW', type=os.fsencode,
|
|
help=_("Rename tags starting with OLD to start with NEW. For "
|
|
"example, --tag-rename foo:bar will rename tag foo-1.2.3 "
|
|
"to bar-1.2.3; either OLD or NEW can be empty."))
|
|
|
|
messages = parser.add_argument_group(title=_("Filtering of commit messages "
|
|
"(see also --message-callback)"))
|
|
messages.add_argument('--replace-message', metavar='EXPRESSIONS_FILE',
|
|
help=_("A file with expressions that, if found in commit messages, "
|
|
"will be replaced. This file uses the same syntax as "
|
|
"--replace-text."))
|
|
messages.add_argument('--preserve-commit-hashes', action='store_true',
|
|
help=_("By default, since commits are rewritten and thus gain new "
|
|
"hashes, references to old commit hashes in commit messages "
|
|
"are replaced with new commit hashes (abbreviated to the same "
|
|
"length as the old reference). Use this flag to turn off "
|
|
"updating commit hashes in commit messages."))
|
|
messages.add_argument('--preserve-commit-encoding', action='store_true',
|
|
help=_("Do not reencode commit messages into UTF-8. By default, if "
|
|
"the commit object specifies an encoding for the commit "
|
|
"message, the message is re-encoded into UTF-8."))
|
|
|
|
people = parser.add_argument_group(title=_("Filtering of names & emails "
|
|
"(see also --name-callback "
|
|
"and --email-callback)"))
|
|
people.add_argument('--mailmap', dest='mailmap', metavar='FILENAME',
|
|
type=os.fsencode,
|
|
help=_("Use specified mailmap file (see git-shortlog(1) for "
|
|
"details on the format) when rewriting author, committer, "
|
|
"and tagger names and emails. If the specified file is "
|
|
"part of git history, historical versions of the file will "
|
|
"be ignored; only the current contents are consulted."))
|
|
people.add_argument('--use-mailmap', dest='mailmap',
|
|
action='store_const', const=b'.mailmap',
|
|
help=_("Same as: '--mailmap .mailmap' "))
|
|
|
|
parents = parser.add_argument_group(title=_("Parent rewriting"))
|
|
parents.add_argument('--replace-refs', default=None,
|
|
choices=['delete-no-add', 'delete-and-add',
|
|
'update-no-add', 'update-or-add',
|
|
'update-and-add'],
|
|
help=_("Replace refs (see git-replace(1)) are used to rewrite "
|
|
"parents (unless turned off by the usual git mechanism); this "
|
|
"flag specifies what do do with those refs afterward. "
|
|
"Replace refs can either be deleted or updated to point at new "
|
|
"commit hashes. Also, new replace refs can be added for each "
|
|
"commit rewrite. With 'update-or-add', new replace refs are "
|
|
"only added for commit rewrites that aren't used to update an "
|
|
"existing replace ref. default is 'update-and-add' if "
|
|
"$GIT_DIR/filter-repo/already_ran does not exist; "
|
|
"'update-or-add' otherwise."))
|
|
parents.add_argument('--prune-empty', default='auto',
|
|
choices=['always', 'auto', 'never'],
|
|
help=_("Whether to prune empty commits. 'auto' (the default) means "
|
|
"only prune commits which become empty (not commits which were "
|
|
"empty in the original repo, unless their parent was pruned). "
|
|
"When the parent of a commit is pruned, the first non-pruned "
|
|
"ancestor becomes the new parent."))
|
|
parents.add_argument('--prune-degenerate', default='auto',
|
|
choices=['always', 'auto', 'never'],
|
|
help=_("Since merge commits are needed for history topology, they "
|
|
"are typically exempt from pruning. However, they can become "
|
|
"degenerate with the pruning of other commits (having fewer "
|
|
"than two parents, having one commit serve as both parents, or "
|
|
"having one parent as the ancestor of the other.) If such "
|
|
"merge commits have no file changes, they can be pruned. The "
|
|
"default ('auto') is to only prune empty merge commits which "
|
|
"become degenerate (not which started as such)."))
|
|
parents.add_argument('--no-ff', action='store_true',
|
|
help=_("Even if the first parent is or becomes an ancestor of another "
|
|
"parent, do not prune it. This modifies how "
|
|
"--prune-degenerate behaves, and may be useful in projects who "
|
|
"always use merge --no-ff."))
|
|
|
|
callback = parser.add_argument_group(title=_("Generic callback code snippets"))
|
|
callback.add_argument('--filename-callback', metavar="FUNCTION_BODY_OR_FILE",
|
|
help=_("Python code body for processing filenames; see CALLBACKS "
|
|
"sections below."))
|
|
callback.add_argument('--message-callback', metavar="FUNCTION_BODY_OR_FILE",
|
|
help=_("Python code body for processing messages (both commit "
|
|
"messages and tag messages); see CALLBACKS section below."))
|
|
callback.add_argument('--name-callback', metavar="FUNCTION_BODY_OR_FILE",
|
|
help=_("Python code body for processing names of people; see "
|
|
"CALLBACKS section below."))
|
|
callback.add_argument('--email-callback', metavar="FUNCTION_BODY_OR_FILE",
|
|
help=_("Python code body for processing emails addresses; see "
|
|
"CALLBACKS section below."))
|
|
callback.add_argument('--refname-callback', metavar="FUNCTION_BODY_OR_FILE",
|
|
help=_("Python code body for processing refnames; see CALLBACKS "
|
|
"section below."))
|
|
|
|
callback.add_argument('--blob-callback', metavar="FUNCTION_BODY_OR_FILE",
|
|
help=_("Python code body for processing blob objects; see "
|
|
"CALLBACKS section below."))
|
|
callback.add_argument('--commit-callback', metavar="FUNCTION_BODY_OR_FILE",
|
|
help=_("Python code body for processing commit objects; see "
|
|
"CALLBACKS section below."))
|
|
callback.add_argument('--tag-callback', metavar="FUNCTION_BODY_OR_FILE",
|
|
help=_("Python code body for processing tag objects; see CALLBACKS "
|
|
"section below."))
|
|
callback.add_argument('--reset-callback', metavar="FUNCTION_BODY_OR_FILE",
|
|
help=_("Python code body for processing reset objects; see "
|
|
"CALLBACKS section below."))
|
|
|
|
desc = _(
|
|
"Specifying alternate source or target locations implies --partial,\n"
|
|
"except that the normal default for --replace-refs is used. However,\n"
|
|
"unlike normal uses of --partial, this doesn't risk mixing old and new\n"
|
|
"history since the old and new histories are in different repositories.")
|
|
location = parser.add_argument_group(title=_("Location to filter from/to"),
|
|
description=desc)
|
|
location.add_argument('--source', type=os.fsencode,
|
|
help=_("Git repository to read from"))
|
|
location.add_argument('--target', type=os.fsencode,
|
|
help=_("Git repository to overwrite with filtered history"))
|
|
|
|
misc = parser.add_argument_group(title=_("Miscellaneous options"))
|
|
misc.add_argument('--help', '-h', action='store_true',
|
|
help=_("Show this help message and exit."))
|
|
misc.add_argument('--version', action='store_true',
|
|
help=_("Display filter-repo's version and exit."))
|
|
misc.add_argument('--force', '-f', action='store_true',
|
|
help=_("Rewrite repository history even if the current repo does not "
|
|
"look like a fresh clone. History rewriting is irreversible "
|
|
"(and includes immediate pruning of reflogs and old objects), "
|
|
"so be cautious about using this flag."))
|
|
misc.add_argument('--partial', action='store_true',
|
|
help=_("Do a partial history rewrite, resulting in the mixture of "
|
|
"old and new history. This implies a default of "
|
|
"update-no-add for --replace-refs, disables rewriting "
|
|
"refs/remotes/origin/* to refs/heads/*, disables removing "
|
|
"of the 'origin' remote, disables removing unexported refs, "
|
|
"disables expiring the reflog, and disables the automatic "
|
|
"post-filter gc. Also, this modifies --tag-rename and "
|
|
"--refname-callback options such that instead of replacing "
|
|
"old refs with new refnames, it will instead create new "
|
|
"refs and keep the old ones around. Use with caution."))
|
|
# WARNING: --refs presents a problem with become-degenerate pruning:
|
|
# * Excluding a commit also excludes its ancestors so when some other
|
|
# commit has an excluded ancestor as a parent we have no way of
|
|
# knowing what it is an ancestor of without doing a special
|
|
# full-graph walk.
|
|
misc.add_argument('--refs', nargs='+',
|
|
help=_("Limit history rewriting to the specified refs. Implies "
|
|
"--partial. In addition to the normal caveats of --partial "
|
|
"(mixing old and new history, no automatic remapping of "
|
|
"refs/remotes/origin/* to refs/heads/*, etc.), this also may "
|
|
"cause problems for pruning of degenerate empty merge "
|
|
"commits when negative revisions are specified."))
|
|
|
|
misc.add_argument('--dry-run', action='store_true',
|
|
help=_("Do not change the repository. Run `git fast-export` and "
|
|
"filter its output, and save both the original and the "
|
|
"filtered version for comparison. This also disables "
|
|
"rewriting commit messages due to not knowing new commit "
|
|
"IDs and disables filtering of some empty commits due to "
|
|
"inability to query the fast-import backend." ))
|
|
misc.add_argument('--debug', action='store_true',
|
|
help=_("Print additional information about operations being "
|
|
"performed and commands being run. When used together "
|
|
"with --dry-run, also show extra information about what "
|
|
"would be run."))
|
|
# WARNING: --state-branch has some problems:
|
|
# * It does not work well with manually inserted objects (user creating
|
|
# Blob() or Commit() or Tag() objects and calling
|
|
# RepoFilter.insert(obj) on them).
|
|
# * It does not work well with multiple source or multiple target repos
|
|
# * It doesn't work so well with pruning become-empty commits (though
|
|
# --refs doesn't work so well with it either)
|
|
# These are probably fixable, given some work (e.g. re-importing the
|
|
# graph at the beginning to get the AncestryGraph right, doing our own
|
|
# export of marks instead of using fast-export --export-marks, etc.), but
|
|
# for now just hide the option.
|
|
misc.add_argument('--state-branch',
|
|
#help=_("Enable incremental filtering by saving the mapping of old "
|
|
# "to new objects to the specified branch upon exit, and"
|
|
# "loading that mapping from that branch (if it exists) "
|
|
# "upon startup."))
|
|
help=argparse.SUPPRESS)
|
|
misc.add_argument('--stdin', action='store_true',
|
|
help=_("Instead of running `git fast-export` and filtering its "
|
|
"output, filter the fast-export stream from stdin. The "
|
|
"stdin must be in the expected input format (e.g. it needs "
|
|
"to include original-oid directives)."))
|
|
misc.add_argument('--quiet', action='store_true',
|
|
help=_("Pass --quiet to other git commands called"))
|
|
return parser
|
|
|
|
@staticmethod
|
|
def sanity_check_args(args):
|
|
if args.analyze and args.path_changes:
|
|
raise SystemExit(_("Error: --analyze is incompatible with --path* flags; "
|
|
"it's a read-only operation."))
|
|
if args.analyze and args.stdin:
|
|
raise SystemExit(_("Error: --analyze is incompatible with --stdin."))
|
|
# If no path_changes are found, initialize with empty list but mark as
|
|
# not inclusive so that all files match
|
|
if args.path_changes == None:
|
|
args.path_changes = []
|
|
args.inclusive = False
|
|
else:
|
|
# Similarly, if we have no filtering paths, then no path should be
|
|
# filtered out. Based on how newname() works, the easiest way to
|
|
# achieve that is setting args.inclusive to False.
|
|
if not any(x[0] == 'filter' for x in args.path_changes):
|
|
args.inclusive = False
|
|
# Also check for incompatible --use-base-name and --path-rename flags.
|
|
if args.use_base_name:
|
|
if any(x[0] == 'rename' for x in args.path_changes):
|
|
raise SystemExit(_("Error: --use-base-name and --path-rename are "
|
|
"incompatible."))
|
|
# Also throw some sanity checks on git version here;
|
|
# PERF: remove these checks once new enough git versions are common
|
|
p = subproc.Popen('git fast-export -h'.split(),
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
output = p.stdout.read()
|
|
if b'--anonymize-map' not in output: # pragma: no cover
|
|
global date_format_permissive
|
|
date_format_permissive = False
|
|
if b'--mark-tags' not in output: # pragma: no cover
|
|
global write_marks
|
|
write_marks = False
|
|
if args.state_branch:
|
|
# We need a version of git-fast-export with --mark-tags
|
|
raise SystemExit(_("Error: need git >= 2.24.0"))
|
|
if b'--reencode' not in output: # pragma: no cover
|
|
if args.preserve_commit_encoding:
|
|
# We need a version of git-fast-export with --reencode
|
|
raise SystemExit(_("Error: need git >= 2.23.0"))
|
|
else:
|
|
# Set args.preserve_commit_encoding to None which we'll check for later
|
|
# to avoid passing --reencode=yes to fast-export (that option was the
|
|
# default prior to git-2.23)
|
|
args.preserve_commit_encoding = None
|
|
# If we don't have fast-exoprt --reencode, we may also be missing
|
|
# diff-tree --combined-all-paths, which is even more important...
|
|
p = subproc.Popen('git diff-tree -h'.split(),
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
output = p.stdout.read()
|
|
if b'--combined-all-paths' not in output:
|
|
# We need a version of git-diff-tree with --combined-all-paths
|
|
raise SystemExit(_("Error: need git >= 2.22.0"))
|
|
# End of sanity checks on git version
|
|
if args.max_blob_size:
|
|
suffix = args.max_blob_size[-1]
|
|
if suffix not in '1234567890':
|
|
mult = {'K': 1024, 'M': 1024**2, 'G': 1024**3}
|
|
if suffix not in mult:
|
|
raise SystemExit(_("Error: could not parse --strip-blobs-bigger-than"
|
|
" argument %s")
|
|
% args.max_blob_size)
|
|
args.max_blob_size = int(args.max_blob_size[0:-1]) * mult[suffix]
|
|
else:
|
|
args.max_blob_size = int(args.max_blob_size)
|
|
|
|
@staticmethod
|
|
def get_replace_text(filename):
|
|
replace_literals = []
|
|
replace_regexes = []
|
|
with open(filename, 'br') as f:
|
|
for line in f:
|
|
line = line.rstrip(b'\r\n')
|
|
|
|
# Determine the replacement
|
|
replacement = FilteringOptions.default_replace_text
|
|
if b'==>' in line:
|
|
line, replacement = line.rsplit(b'==>', 1)
|
|
|
|
# See if we need to match via regex
|
|
regex = None
|
|
if line.startswith(b'regex:'):
|
|
regex = line[6:]
|
|
elif line.startswith(b'glob:'):
|
|
regex = glob_to_regex(line[5:])
|
|
if regex:
|
|
replace_regexes.append((re.compile(regex), replacement))
|
|
else:
|
|
# Otherwise, find the literal we need to replace
|
|
if line.startswith(b'literal:'):
|
|
line = line[8:]
|
|
if not line:
|
|
continue
|
|
replace_literals.append((line, replacement))
|
|
return {'literals': replace_literals, 'regexes': replace_regexes}
|
|
|
|
@staticmethod
|
|
def get_paths_from_file(filename):
|
|
new_path_changes = []
|
|
with open(filename, 'br') as f:
|
|
for line in f:
|
|
line = line.rstrip(b'\r\n')
|
|
|
|
# Skip blank lines
|
|
if not line:
|
|
continue
|
|
# Skip comment lines
|
|
if line.startswith(b'#'):
|
|
continue
|
|
|
|
# Determine the replacement
|
|
match_type, repl = 'literal', None
|
|
if b'==>' in line:
|
|
line, repl = line.rsplit(b'==>', 1)
|
|
|
|
# See if we need to match via regex
|
|
match_type = 'match' # a.k.a. 'literal'
|
|
if line.startswith(b'regex:'):
|
|
match_type = 'regex'
|
|
match = re.compile(line[6:])
|
|
elif line.startswith(b'glob:'):
|
|
match_type = 'glob'
|
|
match = line[5:]
|
|
if repl:
|
|
raise SystemExit(_("Error: In %s, 'glob:' and '==>' are incompatible (renaming globs makes no sense)" % decode(filename)))
|
|
else:
|
|
if line.startswith(b'literal:'):
|
|
match = line[8:]
|
|
else:
|
|
match = line
|
|
if repl is not None:
|
|
if match and repl and match.endswith(b'/') != repl.endswith(b'/'):
|
|
raise SystemExit(_("Error: When rename directories, if OLDNAME "
|
|
"and NEW_NAME are both non-empty and either "
|
|
"ends with a slash then both must."))
|
|
|
|
# Record the filter or rename
|
|
if repl is not None:
|
|
new_path_changes.append(['rename', match_type, (match, repl)])
|
|
else:
|
|
new_path_changes.append(['filter', match_type, match])
|
|
if match_type == 'glob' and not match.endswith(b'*'):
|
|
extension = b'*' if match.endswith(b'/') else b'/*'
|
|
new_path_changes.append(['filter', match_type, match+extension])
|
|
return new_path_changes
|
|
|
|
@staticmethod
|
|
def default_options():
|
|
return FilteringOptions.parse_args([], error_on_empty = False)
|
|
|
|
@staticmethod
|
|
def parse_args(input_args, error_on_empty = True):
|
|
parser = FilteringOptions.create_arg_parser()
|
|
if not input_args and error_on_empty:
|
|
parser.print_usage()
|
|
raise SystemExit(_("No arguments specified."))
|
|
args = parser.parse_args(input_args)
|
|
if args.help:
|
|
parser.print_help()
|
|
raise SystemExit()
|
|
if args.version:
|
|
GitUtils.print_my_version()
|
|
raise SystemExit()
|
|
FilteringOptions.sanity_check_args(args)
|
|
if args.mailmap:
|
|
args.mailmap = MailmapInfo(args.mailmap)
|
|
if args.replace_text:
|
|
args.replace_text = FilteringOptions.get_replace_text(args.replace_text)
|
|
if args.replace_message:
|
|
args.replace_message = FilteringOptions.get_replace_text(args.replace_message)
|
|
if args.strip_blobs_with_ids:
|
|
with open(args.strip_blobs_with_ids, 'br') as f:
|
|
args.strip_blobs_with_ids = set(f.read().split())
|
|
else:
|
|
args.strip_blobs_with_ids = set()
|
|
if (args.partial or args.refs) and not args.replace_refs:
|
|
args.replace_refs = 'update-no-add'
|
|
args.repack = not (args.partial or args.refs)
|
|
if args.refs or args.source or args.target:
|
|
args.partial = True
|
|
if not args.refs:
|
|
args.refs = ['--all']
|
|
return args
|
|
|
|
class RepoAnalyze(object):
|
|
|
|
# First, several helper functions for analyze_commit()
|
|
|
|
@staticmethod
|
|
def equiv_class(stats, filename):
|
|
return stats['equivalence'].get(filename, (filename,))
|
|
|
|
@staticmethod
|
|
def setup_equivalence_for_rename(stats, oldname, newname):
|
|
# if A is renamed to B and B is renamed to C, then the user thinks of
|
|
# A, B, and C as all being different names for the same 'file'. We record
|
|
# this as an equivalence class:
|
|
# stats['equivalence'][name] = (A,B,C)
|
|
# for name being each of A, B, and C.
|
|
old_tuple = stats['equivalence'].get(oldname, ())
|
|
if newname in old_tuple:
|
|
return
|
|
elif old_tuple:
|
|
new_tuple = tuple(list(old_tuple)+[newname])
|
|
else:
|
|
new_tuple = (oldname, newname)
|
|
for f in new_tuple:
|
|
stats['equivalence'][f] = new_tuple
|
|
|
|
@staticmethod
|
|
def setup_or_update_rename_history(stats, commit, oldname, newname):
|
|
rename_commits = stats['rename_history'].get(oldname, set())
|
|
rename_commits.add(commit)
|
|
stats['rename_history'][oldname] = rename_commits
|
|
|
|
@staticmethod
|
|
def handle_renames(stats, commit, change_types, filenames):
|
|
for index, change_type in enumerate(change_types):
|
|
if change_type == ord(b'R'):
|
|
oldname, newname = filenames[index], filenames[-1]
|
|
RepoAnalyze.setup_equivalence_for_rename(stats, oldname, newname)
|
|
RepoAnalyze.setup_or_update_rename_history(stats, commit,
|
|
oldname, newname)
|
|
|
|
@staticmethod
|
|
def handle_file(stats, graph, commit, modes, shas, filenames):
|
|
mode, sha, filename = modes[-1], shas[-1], filenames[-1]
|
|
|
|
# Figure out kind of deletions to undo for this file, and update lists
|
|
# of all-names-by-sha and all-filenames
|
|
delmode = 'tree_deletions'
|
|
if mode != b'040000':
|
|
delmode = 'file_deletions'
|
|
stats['names'][sha].add(filename)
|
|
stats['allnames'].add(filename)
|
|
|
|
# If the file (or equivalence class of files) was recorded as deleted,
|
|
# clearly it isn't anymore
|
|
equiv = RepoAnalyze.equiv_class(stats, filename)
|
|
for f in equiv:
|
|
stats[delmode].pop(f, None)
|
|
|
|
# If we get a modify/add for a path that was renamed, we may need to break
|
|
# the equivalence class. However, if the modify/add was on a branch that
|
|
# doesn't have the rename in its history, we are still okay.
|
|
need_to_break_equivalence = False
|
|
if equiv[-1] != filename:
|
|
for rename_commit in stats['rename_history'][filename]:
|
|
if graph.is_ancestor(rename_commit, commit):
|
|
need_to_break_equivalence = True
|
|
|
|
if need_to_break_equivalence:
|
|
for f in equiv:
|
|
if f in stats['equivalence']:
|
|
del stats['equivalence'][f]
|
|
|
|
@staticmethod
|
|
def analyze_commit(stats, graph, commit, parents, date, file_changes):
|
|
graph.add_commit_and_parents(commit, parents)
|
|
for change in file_changes:
|
|
modes, shas, change_types, filenames = change
|
|
if len(parents) == 1 and change_types.startswith(b'R'):
|
|
change_types = b'R' # remove the rename score; we don't care
|
|
if modes[-1] == b'160000':
|
|
continue
|
|
elif modes[-1] == b'000000':
|
|
# Track when files/directories are deleted
|
|
for f in RepoAnalyze.equiv_class(stats, filenames[-1]):
|
|
if any(x == b'040000' for x in modes[0:-1]):
|
|
stats['tree_deletions'][f] = date
|
|
else:
|
|
stats['file_deletions'][f] = date
|
|
elif change_types.strip(b'AMT') == b'':
|
|
RepoAnalyze.handle_file(stats, graph, commit, modes, shas, filenames)
|
|
elif modes[-1] == b'040000' and change_types.strip(b'RAM') == b'':
|
|
RepoAnalyze.handle_file(stats, graph, commit, modes, shas, filenames)
|
|
elif change_types.strip(b'RAMT') == b'':
|
|
RepoAnalyze.handle_file(stats, graph, commit, modes, shas, filenames)
|
|
RepoAnalyze.handle_renames(stats, commit, change_types, filenames)
|
|
else:
|
|
raise SystemExit(_("Unhandled change type(s): %(change_type)s "
|
|
"(in commit %(commit)s)")
|
|
% ({'change_type': change_types, 'commit': commit})
|
|
) # pragma: no cover
|
|
|
|
@staticmethod
|
|
def gather_data(args):
|
|
unpacked_size, packed_size = GitUtils.get_blob_sizes()
|
|
stats = {'names': collections.defaultdict(set),
|
|
'allnames' : set(),
|
|
'file_deletions': {},
|
|
'tree_deletions': {},
|
|
'equivalence': {},
|
|
'rename_history': collections.defaultdict(set),
|
|
'unpacked_size': unpacked_size,
|
|
'packed_size': packed_size,
|
|
'num_commits': 0}
|
|
|
|
# Setup the rev-list/diff-tree process
|
|
processed_commits_msg = _("Processed %d commits")
|
|
commit_parse_progress = ProgressWriter()
|
|
num_commits = 0
|
|
cmd = ('git rev-list --topo-order --reverse {}'.format(' '.join(args.refs)) +
|
|
' | git diff-tree --stdin --always --root --format=%H%n%P%n%cd' +
|
|
' --date=short -M -t -c --raw --combined-all-paths')
|
|
dtp = subproc.Popen(cmd, shell=True, bufsize=-1, stdout=subprocess.PIPE)
|
|
f = dtp.stdout
|
|
line = f.readline()
|
|
if not line:
|
|
raise SystemExit(_("Nothing to analyze; repository is empty."))
|
|
cont = bool(line)
|
|
graph = AncestryGraph()
|
|
while cont:
|
|
commit = line.rstrip()
|
|
parents = f.readline().split()
|
|
date = f.readline().rstrip()
|
|
|
|
# We expect a blank line next; if we get a non-blank line then
|
|
# this commit modified no files and we need to move on to the next.
|
|
# If there is no line, we've reached end-of-input.
|
|
line = f.readline()
|
|
if not line:
|
|
cont = False
|
|
line = line.rstrip()
|
|
|
|
# If we haven't reached end of input, and we got a blank line meaning
|
|
# a commit that has modified files, then get the file changes associated
|
|
# with this commit.
|
|
file_changes = []
|
|
if cont and not line:
|
|
cont = False
|
|
for line in f:
|
|
if not line.startswith(b':'):
|
|
cont = True
|
|
break
|
|
n = 1+max(1, len(parents))
|
|
assert line.startswith(b':'*(n-1))
|
|
relevant = line[n-1:-1]
|
|
splits = relevant.split(None, n)
|
|
modes = splits[0:n]
|
|
splits = splits[n].split(None, n)
|
|
shas = splits[0:n]
|
|
splits = splits[n].split(b'\t')
|
|
change_types = splits[0]
|
|
filenames = [PathQuoting.dequote(x) for x in splits[1:]]
|
|
file_changes.append([modes, shas, change_types, filenames])
|
|
|
|
# If someone is trying to analyze a subset of the history, make sure
|
|
# to avoid dying on commits with parents that we haven't seen before
|
|
if args.refs:
|
|
graph.record_external_commits([p for p in parents
|
|
if not p in graph.value])
|
|
|
|
# Analyze this commit and update progress
|
|
RepoAnalyze.analyze_commit(stats, graph, commit, parents, date,
|
|
file_changes)
|
|
num_commits += 1
|
|
commit_parse_progress.show(processed_commits_msg % num_commits)
|
|
|
|
# Show the final commits processed message and record the number of commits
|
|
commit_parse_progress.finish()
|
|
stats['num_commits'] = num_commits
|
|
|
|
# Close the output, ensure rev-list|diff-tree pipeline completed successfully
|
|
dtp.stdout.close()
|
|
if dtp.wait():
|
|
raise SystemExit(_("Error: rev-list|diff-tree pipeline failed; see above.")) # pragma: no cover
|
|
|
|
return stats
|
|
|
|
@staticmethod
|
|
def write_report(reportdir, stats):
|
|
def datestr(datetimestr):
|
|
return datetimestr if datetimestr else _('<present>').encode()
|
|
|
|
def dirnames(path):
|
|
while True:
|
|
path = os.path.dirname(path)
|
|
yield path
|
|
if path == b'':
|
|
break
|
|
|
|
# Compute aggregate size information for paths, extensions, and dirs
|
|
total_size = {'packed': 0, 'unpacked': 0}
|
|
path_size = {'packed': collections.defaultdict(int),
|
|
'unpacked': collections.defaultdict(int)}
|
|
ext_size = {'packed': collections.defaultdict(int),
|
|
'unpacked': collections.defaultdict(int)}
|
|
dir_size = {'packed': collections.defaultdict(int),
|
|
'unpacked': collections.defaultdict(int)}
|
|
for sha in stats['names']:
|
|
size = {'packed': stats['packed_size'][sha],
|
|
'unpacked': stats['unpacked_size'][sha]}
|
|
for which in ('packed', 'unpacked'):
|
|
for name in stats['names'][sha]:
|
|
total_size[which] += size[which]
|
|
path_size[which][name] += size[which]
|
|
basename, ext = os.path.splitext(name)
|
|
ext_size[which][ext] += size[which]
|
|
for dirname in dirnames(name):
|
|
dir_size[which][dirname] += size[which]
|
|
|
|
# Determine if and when extensions and directories were deleted
|
|
ext_deleted_data = {}
|
|
for name in stats['allnames']:
|
|
when = stats['file_deletions'].get(name, None)
|
|
|
|
# Update the extension
|
|
basename, ext = os.path.splitext(name)
|
|
if when is None:
|
|
ext_deleted_data[ext] = None
|
|
elif ext in ext_deleted_data:
|
|
if ext_deleted_data[ext] is not None:
|
|
ext_deleted_data[ext] = max(ext_deleted_data[ext], when)
|
|
else:
|
|
ext_deleted_data[ext] = when
|
|
|
|
dir_deleted_data = {}
|
|
for name in dir_size['packed']:
|
|
dir_deleted_data[name] = stats['tree_deletions'].get(name, None)
|
|
|
|
with open(os.path.join(reportdir, b"README"), 'bw') as f:
|
|
# Give a basic overview of this file
|
|
f.write(b"== %s ==\n" % _("Overall Statistics").encode())
|
|
f.write((" %s: %d\n" % (_("Number of commits"),
|
|
stats['num_commits'])).encode())
|
|
f.write((" %s: %d\n" % (_("Number of filenames"),
|
|
len(path_size['packed']))).encode())
|
|
f.write((" %s: %d\n" % (_("Number of directories"),
|
|
len(dir_size['packed']))).encode())
|
|
f.write((" %s: %d\n" % (_("Number of file extensions"),
|
|
len(ext_size['packed']))).encode())
|
|
f.write(b"\n")
|
|
f.write((" %s: %d\n" % (_("Total unpacked size (bytes)"),
|
|
total_size['unpacked'])).encode())
|
|
f.write((" %s: %d\n" % (_("Total packed size (bytes)"),
|
|
total_size['packed'])).encode())
|
|
f.write(b"\n")
|
|
|
|
# Mention issues with the report
|
|
f.write(("== %s ==\n" % _("Caveats")).encode())
|
|
f.write(("=== %s ===\n" % _("Sizes")).encode())
|
|
f.write(textwrap.dedent(_("""
|
|
Packed size represents what size your repository would be if no
|
|
trees, commits, tags, or other metadata were included (though it may
|
|
fail to represent de-duplication; see below). It also represents the
|
|
current packing, which may be suboptimal if you haven't gc'ed for a
|
|
while.
|
|
|
|
Unpacked size represents what size your repository would be if no
|
|
trees, commits, tags, or other metadata were included AND if no
|
|
files were packed; i.e., without delta-ing or compression.
|
|
|
|
Both unpacked and packed sizes can be slightly misleading. Deleting
|
|
a blob from history not save as much space as the unpacked size,
|
|
because it is obviously normally stored in packed form. Also,
|
|
deleting a blob from history may not save as much space as its packed
|
|
size either, because another blob could be stored as a delta against
|
|
that blob, so when you remove one blob another blob's packed size may
|
|
grow.
|
|
|
|
Also, the sum of the packed sizes can add up to more than the
|
|
repository size; if the same contents appeared in the repository in
|
|
multiple places, git will automatically de-dupe and store only one
|
|
copy, while the way sizes are added in this analysis adds the size
|
|
for each file path that has those contents. Further, if a file is
|
|
ever reverted to a previous version's contents, the previous
|
|
version's size will be counted multiple times in this analysis, even
|
|
though git will only store it once.
|
|
""")[1:]).encode())
|
|
f.write(b"\n")
|
|
f.write(("=== %s ===\n" % _("Deletions")).encode())
|
|
f.write(textwrap.dedent(_("""
|
|
Whether a file is deleted is not a binary quality, since it can be
|
|
deleted on some branches but still exist in others. Also, it might
|
|
exist in an old tag, but have been deleted in versions newer than
|
|
that. More thorough tracking could be done, including looking at
|
|
merge commits where one side of history deleted and the other modified,
|
|
in order to give a more holistic picture of deletions. However, that
|
|
algorithm would not only be more complex to implement, it'd also be
|
|
quite difficult to present and interpret by users. Since --analyze
|
|
is just about getting a high-level rough picture of history, it instead
|
|
implements the simplistic rule that is good enough for 98% of cases:
|
|
A file is marked as deleted if the last commit in the fast-export
|
|
stream that mentions the file lists it as deleted.
|
|
This makes it dependent on topological ordering, but generally gives
|
|
the "right" answer.
|
|
""")[1:]).encode())
|
|
f.write(b"\n")
|
|
f.write(("=== %s ===\n" % _("Renames")).encode())
|
|
f.write(textwrap.dedent(_("""
|
|
Renames share the same non-binary nature that deletions do, plus
|
|
additional challenges:
|
|
* If the renamed file is renamed again, instead of just two names for
|
|
a path you can have three or more.
|
|
* Rename pairs of the form (oldname, newname) that we consider to be
|
|
different names of the "same file" might only be valid over certain
|
|
commit ranges. For example, if a new commit reintroduces a file
|
|
named oldname, then new versions of oldname aren't the "same file"
|
|
anymore. We could try to portray this to the user, but it's easier
|
|
for the user to just break the pairing and only report unbroken
|
|
rename pairings to the user.
|
|
* The ability for users to rename files differently in different
|
|
branches means that our chains of renames will not necessarily be
|
|
linear but may branch out.
|
|
""")[1:]).encode())
|
|
f.write(b"\n")
|
|
|
|
# Equivalence classes for names, so if folks only want to keep a
|
|
# certain set of paths, they know the old names they want to include
|
|
# too.
|
|
with open(os.path.join(reportdir, b"renames.txt"), 'bw') as f:
|
|
seen = set()
|
|
for pathname,equiv_group in sorted(stats['equivalence'].items(),
|
|
key=lambda x:(x[1], x[0])):
|
|
if equiv_group in seen:
|
|
continue
|
|
seen.add(equiv_group)
|
|
f.write(("{} ->\n ".format(decode(equiv_group[0])) +
|
|
"\n ".join(decode(x) for x in equiv_group[1:]) +
|
|
"\n").encode())
|
|
|
|
# List directories in reverse sorted order of unpacked size
|
|
with open(os.path.join(reportdir, b"directories-deleted-sizes.txt"), 'bw') as f:
|
|
msg = "=== %s ===\n" % _("Deleted directories by reverse size")
|
|
f.write(msg.encode())
|
|
msg = _("Format: unpacked size, packed size, date deleted, directory name\n")
|
|
f.write(msg.encode())
|
|
for dirname, size in sorted(dir_size['packed'].items(),
|
|
key=lambda x:(x[1],x[0]), reverse=True):
|
|
if (dir_deleted_data[dirname]):
|
|
f.write(b" %10d %10d %-10s %s\n" % (dir_size['unpacked'][dirname],
|
|
size,
|
|
datestr(dir_deleted_data[dirname]),
|
|
dirname or _('<toplevel>').encode()))
|
|
|
|
with open(os.path.join(reportdir, b"directories-all-sizes.txt"), 'bw') as f:
|
|
f.write(("=== %s ===\n" % _("All directories by reverse size")).encode())
|
|
msg = _("Format: unpacked size, packed size, date deleted, directory name\n")
|
|
f.write(msg.encode())
|
|
for dirname, size in sorted(dir_size['packed'].items(),
|
|
key=lambda x:(x[1],x[0]), reverse=True):
|
|
f.write(b" %10d %10d %-10s %s\n" % (dir_size['unpacked'][dirname],
|
|
size,
|
|
datestr(dir_deleted_data[dirname]),
|
|
dirname or _("<toplevel>").encode()))
|
|
|
|
# List extensions in reverse sorted order of unpacked size
|
|
with open(os.path.join(reportdir, b"extensions-deleted-sizes.txt"), 'bw') as f:
|
|
msg = "=== %s ===\n" % _("Deleted extensions by reverse size")
|
|
f.write(msg.encode())
|
|
msg = _("Format: unpacked size, packed size, date deleted, extension name\n")
|
|
f.write(msg.encode())
|
|
for extname, size in sorted(ext_size['packed'].items(),
|
|
key=lambda x:(x[1],x[0]), reverse=True):
|
|
if (ext_deleted_data[extname]):
|
|
f.write(b" %10d %10d %-10s %s\n" % (ext_size['unpacked'][extname],
|
|
size,
|
|
datestr(ext_deleted_data[extname]),
|
|
extname or _('<no extension>').encode()))
|
|
|
|
with open(os.path.join(reportdir, b"extensions-all-sizes.txt"), 'bw') as f:
|
|
f.write(("=== %s ===\n" % _("All extensions by reverse size")).encode())
|
|
msg = _("Format: unpacked size, packed size, date deleted, extension name\n")
|
|
f.write(msg.encode())
|
|
for extname, size in sorted(ext_size['packed'].items(),
|
|
key=lambda x:(x[1],x[0]), reverse=True):
|
|
f.write(b" %10d %10d %-10s %s\n" % (ext_size['unpacked'][extname],
|
|
size,
|
|
datestr(ext_deleted_data[extname]),
|
|
extname or _('<no extension>').encode()))
|
|
|
|
# List files in reverse sorted order of unpacked size
|
|
with open(os.path.join(reportdir, b"path-deleted-sizes.txt"), 'bw') as f:
|
|
msg = "=== %s ===\n" % _("Deleted paths by reverse accumulated size")
|
|
f.write(msg.encode())
|
|
msg = _("Format: unpacked size, packed size, date deleted, path name(s)\n")
|
|
f.write(msg.encode())
|
|
for pathname, size in sorted(path_size['packed'].items(),
|
|
key=lambda x:(x[1],x[0]), reverse=True):
|
|
when = stats['file_deletions'].get(pathname, None)
|
|
if when:
|
|
f.write(b" %10d %10d %-10s %s\n" % (path_size['unpacked'][pathname],
|
|
size,
|
|
datestr(when),
|
|
pathname))
|
|
|
|
with open(os.path.join(reportdir, b"path-all-sizes.txt"), 'bw') as f:
|
|
msg = "=== %s ===\n" % _("All paths by reverse accumulated size")
|
|
f.write(msg.encode())
|
|
msg = _("Format: unpacked size, packed size, date deleted, path name\n")
|
|
f.write(msg.encode())
|
|
for pathname, size in sorted(path_size['packed'].items(),
|
|
key=lambda x:(x[1],x[0]), reverse=True):
|
|
when = stats['file_deletions'].get(pathname, None)
|
|
f.write(b" %10d %10d %-10s %s\n" % (path_size['unpacked'][pathname],
|
|
size,
|
|
datestr(when),
|
|
pathname))
|
|
|
|
# List of filenames and sizes in descending order
|
|
with open(os.path.join(reportdir, b"blob-shas-and-paths.txt"), 'bw') as f:
|
|
f.write(("=== %s ===\n" % _("Files by sha and associated pathnames in reverse size")).encode())
|
|
f.write(_("Format: sha, unpacked size, packed size, filename(s) object stored as\n").encode())
|
|
for sha, size in sorted(stats['packed_size'].items(),
|
|
key=lambda x:(x[1],x[0]), reverse=True):
|
|
if sha not in stats['names']:
|
|
# Some objects in the repository might not be referenced, or not
|
|
# referenced by the branches/tags the user cares about; skip them.
|
|
continue
|
|
names_with_sha = stats['names'][sha]
|
|
if len(names_with_sha) == 1:
|
|
names_with_sha = names_with_sha.pop()
|
|
else:
|
|
names_with_sha = b'[' + b', '.join(sorted(names_with_sha)) + b']'
|
|
f.write(b" %s %10d %10d %s\n" % (sha,
|
|
stats['unpacked_size'][sha],
|
|
size,
|
|
names_with_sha))
|
|
|
|
@staticmethod
|
|
def run(args):
|
|
if args.report_dir:
|
|
reportdir = args.report_dir
|
|
else:
|
|
git_dir = GitUtils.determine_git_dir(b'.')
|
|
|
|
# Create the report directory as necessary
|
|
results_tmp_dir = os.path.join(git_dir, b'filter-repo')
|
|
if not os.path.isdir(results_tmp_dir):
|
|
os.mkdir(results_tmp_dir)
|
|
reportdir = os.path.join(results_tmp_dir, b"analysis")
|
|
|
|
if os.path.isdir(reportdir):
|
|
if args.force:
|
|
sys.stdout.write(_("Warning: Removing recursively: \"%s\"") % decode(reportdir))
|
|
shutil.rmtree(reportdir)
|
|
else:
|
|
sys.stdout.write(_("Error: dir already exists (use --force to delete): \"%s\"\n") % decode(reportdir))
|
|
sys.exit(1)
|
|
|
|
os.mkdir(reportdir)
|
|
|
|
# Gather the data we need
|
|
stats = RepoAnalyze.gather_data(args)
|
|
|
|
# Write the reports
|
|
sys.stdout.write(_("Writing reports to %s...") % decode(reportdir))
|
|
sys.stdout.flush()
|
|
RepoAnalyze.write_report(reportdir, stats)
|
|
sys.stdout.write(_("done.\n"))
|
|
|
|
class InputFileBackup:
|
|
def __init__(self, input_file, output_file):
|
|
self.input_file = input_file
|
|
self.output_file = output_file
|
|
|
|
def close(self):
|
|
self.input_file.close()
|
|
self.output_file.close()
|
|
|
|
def read(self, size):
|
|
output = self.input_file.read(size)
|
|
self.output_file.write(output)
|
|
return output
|
|
|
|
def readline(self):
|
|
line = self.input_file.readline()
|
|
self.output_file.write(line)
|
|
return line
|
|
|
|
class DualFileWriter:
|
|
def __init__(self, file1, file2):
|
|
self.file1 = file1
|
|
self.file2 = file2
|
|
|
|
def write(self, *args):
|
|
self.file1.write(*args)
|
|
self.file2.write(*args)
|
|
|
|
def flush(self):
|
|
self.file1.flush()
|
|
self.file2.flush()
|
|
|
|
def close(self):
|
|
self.file1.close()
|
|
self.file2.close()
|
|
|
|
class RepoFilter(object):
|
|
def __init__(self,
|
|
args,
|
|
filename_callback = None,
|
|
message_callback = None,
|
|
name_callback = None,
|
|
email_callback = None,
|
|
refname_callback = None,
|
|
blob_callback = None,
|
|
commit_callback = None,
|
|
tag_callback = None,
|
|
reset_callback = None,
|
|
done_callback = None):
|
|
|
|
self._args = args
|
|
|
|
# Repo we are exporting
|
|
self._repo_working_dir = None
|
|
|
|
# Store callbacks for acting on objects printed by FastExport
|
|
self._blob_callback = blob_callback
|
|
self._commit_callback = commit_callback
|
|
self._tag_callback = tag_callback
|
|
self._reset_callback = reset_callback
|
|
self._done_callback = done_callback
|
|
|
|
# Store callbacks for acting on slices of FastExport objects
|
|
self._filename_callback = filename_callback # filenames from commits
|
|
self._message_callback = message_callback # commit OR tag message
|
|
self._name_callback = name_callback # author, committer, tagger
|
|
self._email_callback = email_callback # author, committer, tagger
|
|
self._refname_callback = refname_callback # from commit/tag/reset
|
|
self._handle_arg_callbacks()
|
|
|
|
# Defaults for input
|
|
self._input = None
|
|
self._fep = None # Fast Export Process
|
|
self._fe_orig = None # Path to where original fast-export output stored
|
|
self._fe_filt = None # Path to where filtered fast-export output stored
|
|
self._parser = None # FastExportParser object we are working with
|
|
|
|
# Defaults for output
|
|
self._output = None
|
|
self._fip = None # Fast Import Process
|
|
self._import_pipes = None
|
|
self._managed_output = True
|
|
|
|
# A tuple of (depth, list-of-ancestors). Commits and ancestors are
|
|
# identified by their id (their 'mark' in fast-export or fast-import
|
|
# speak). The depth of a commit is one more than the max depth of any
|
|
# of its ancestors.
|
|
self._graph = AncestryGraph()
|
|
# Another one, for ancestry of commits in the original repo
|
|
self._orig_graph = AncestryGraph()
|
|
|
|
# Names of files that were tweaked in any commit; such paths could lead
|
|
# to subsequent commits being empty
|
|
self._files_tweaked = set()
|
|
|
|
# A set of commit hash pairs (oldhash, newhash) which used to be merge
|
|
# commits but due to filtering were turned into non-merge commits.
|
|
# The commits probably have suboptimal commit messages (e.g. "Merge branch
|
|
# next into master").
|
|
self._commits_no_longer_merges = []
|
|
|
|
# A dict of original_ids to new_ids; filtering commits means getting
|
|
# new commit hash (sha1sums), and we record the mapping both for
|
|
# diagnostic purposes and so we can rewrite commit messages. Note that
|
|
# the new_id can be None rather than a commit hash if the original
|
|
# commit became empty and was pruned or was otherwise dropped.
|
|
self._commit_renames = {}
|
|
|
|
# A set of original_ids for which we have not yet gotten the
|
|
# new_ids; we use OrderedDict because we need to know the order of
|
|
# insertion, but the values are always ignored (and set to None).
|
|
# If there was an OrderedSet class, I'd use it instead.
|
|
self._pending_renames = collections.OrderedDict()
|
|
|
|
# A dict of commit_hash[0:7] -> set(commit_hashes with that prefix).
|
|
#
|
|
# It's common for commit messages to refer to commits by abbreviated
|
|
# commit hashes, as short as 7 characters. To facilitate translating
|
|
# such short hashes, we have a mapping of prefixes to full old hashes.
|
|
self._commit_short_old_hashes = collections.defaultdict(set)
|
|
|
|
# A set of commit hash references appearing in commit messages which
|
|
# mapped to a valid commit that was removed entirely in the filtering
|
|
# process. The commit message will continue to reference the
|
|
# now-missing commit hash, since there was nothing to map it to.
|
|
self._commits_referenced_but_removed = set()
|
|
|
|
# Progress handling (number of commits parsed, etc.)
|
|
self._progress_writer = ProgressWriter()
|
|
self._num_commits = 0
|
|
|
|
# Size of blobs in the repo
|
|
self._unpacked_size = {}
|
|
|
|
# Other vars
|
|
self._sanity_checks_handled = False
|
|
self._finalize_handled = False
|
|
self._orig_refs = None
|
|
self._newnames = {}
|
|
|
|
# Cache a few message translations for performance reasons
|
|
self._parsed_message = _("Parsed %d commits")
|
|
|
|
# Compile some regexes and cache those
|
|
self._hash_re = re.compile(br'(\b[0-9a-f]{7,40}\b)')
|
|
|
|
def _handle_arg_callbacks(self):
|
|
def make_callback(argname, str):
|
|
exec('def callback({}, _do_not_use_this_var = None):\n'.format(argname)+
|
|
' '+'\n '.join(str.splitlines()), globals())
|
|
return callback #namespace['callback']
|
|
def handle(type):
|
|
callback_field = '_{}_callback'.format(type)
|
|
code_string = getattr(self._args, type+'_callback')
|
|
if code_string:
|
|
if os.path.exists(code_string):
|
|
with open(code_string, 'r', encoding='utf-8') as f:
|
|
code_string = f.read()
|
|
if getattr(self, callback_field):
|
|
raise SystemExit(_("Error: Cannot pass a %s_callback to RepoFilter "
|
|
"AND pass --%s-callback"
|
|
% (type, type)))
|
|
if 'return ' not in code_string and \
|
|
type not in ('blob', 'commit', 'tag', 'reset'):
|
|
raise SystemExit(_("Error: --%s-callback should have a return statement")
|
|
% type)
|
|
setattr(self, callback_field, make_callback(type, code_string))
|
|
handle('filename')
|
|
handle('message')
|
|
handle('name')
|
|
handle('email')
|
|
handle('refname')
|
|
handle('blob')
|
|
handle('commit')
|
|
handle('tag')
|
|
handle('reset')
|
|
|
|
def _run_sanity_checks(self):
|
|
self._sanity_checks_handled = True
|
|
if not self._managed_output:
|
|
if not self._args.replace_refs:
|
|
# If not _managed_output we don't want to make extra changes to the
|
|
# repo, so set default to no-op 'update-no-add'
|
|
self._args.replace_refs = 'update-no-add'
|
|
return
|
|
|
|
if self._args.debug:
|
|
print("[DEBUG] Passed arguments:\n{}".format(self._args))
|
|
|
|
# Determine basic repository information
|
|
target_working_dir = self._args.target or b'.'
|
|
self._orig_refs = GitUtils.get_refs(target_working_dir)
|
|
is_bare = GitUtils.is_repository_bare(target_working_dir)
|
|
|
|
# Determine if this is second or later run of filter-repo
|
|
tmp_dir = self.results_tmp_dir(create_if_missing=False)
|
|
already_ran = os.path.isfile(os.path.join(tmp_dir, b'already_ran'))
|
|
|
|
# Default for --replace-refs
|
|
if not self._args.replace_refs:
|
|
self._args.replace_refs = ('update-or-add' if already_ran
|
|
else 'update-and-add')
|
|
|
|
# Do sanity checks from the correct directory
|
|
if not self._args.force and not already_ran:
|
|
cwd = os.getcwd()
|
|
os.chdir(target_working_dir)
|
|
RepoFilter.sanity_check(self._orig_refs, is_bare)
|
|
os.chdir(cwd)
|
|
|
|
@staticmethod
|
|
def sanity_check(refs, is_bare):
|
|
def abort(reason):
|
|
try:
|
|
cmd = 'git config remote.origin.url'
|
|
output = subproc.check_output(cmd.split()).strip()
|
|
except subprocess.CalledProcessError as e:
|
|
output = None
|
|
msg = ""
|
|
if output and os.path.isdir(output):
|
|
msg = _("Note: when cloning local repositories, you need to pass\n"
|
|
" --no-local to git clone to avoid this issue.\n")
|
|
raise SystemExit(
|
|
_("Aborting: Refusing to destructively overwrite repo history since\n"
|
|
"this does not look like a fresh clone.\n"
|
|
" (%s)\n%s"
|
|
"Please operate on a fresh clone instead. If you want to proceed\n"
|
|
"anyway, use --force.") % (reason, msg))
|
|
|
|
# Make sure repo is fully packed, just like a fresh clone would be.
|
|
# Note that transfer.unpackLimit defaults to 100, meaning that a
|
|
# repository with no packs and less than 100 objects should be considered
|
|
# fully packed.
|
|
output = subproc.check_output('git count-objects -v'.split())
|
|
stats = dict(x.split(b': ') for x in output.splitlines())
|
|
num_packs = int(stats[b'packs'])
|
|
num_loose_objects = int(stats[b'count'])
|
|
if num_packs > 1 or \
|
|
(num_packs == 1 and num_loose_objects > 0) or \
|
|
num_loose_objects >= 100:
|
|
abort(_("expected freshly packed repo"))
|
|
|
|
# Make sure there is precisely one remote, named "origin"...or that this
|
|
# is a new bare repo with no packs and no remotes
|
|
output = subproc.check_output('git remote'.split()).strip()
|
|
if not (output == b"origin" or (num_packs == 0 and not output)):
|
|
abort(_("expected one remote, origin"))
|
|
|
|
# Avoid letting people running with weird setups and overwriting GIT_DIR
|
|
# elsewhere
|
|
git_dir = GitUtils.determine_git_dir(b'.')
|
|
if is_bare and git_dir != b'.':
|
|
abort(_("GIT_DIR must be ."))
|
|
elif not is_bare and git_dir != b'.git':
|
|
abort(_("GIT_DIR must be .git"))
|
|
|
|
# Make sure that all reflogs have precisely one entry
|
|
reflog_dir=os.path.join(git_dir, b'logs')
|
|
for root, dirs, files in os.walk(reflog_dir):
|
|
for filename in files:
|
|
pathname = os.path.join(root, filename)
|
|
with open(pathname, 'br') as f:
|
|
if len(f.read().splitlines()) > 1:
|
|
shortpath = pathname[len(reflog_dir)+1:]
|
|
abort(_("expected at most one entry in the reflog for %s") %
|
|
decode(shortpath))
|
|
|
|
# Make sure there are no stashed changes
|
|
if b'refs/stash' in refs:
|
|
abort(_("has stashed changes"))
|
|
|
|
# Do extra checks in non-bare repos
|
|
if not is_bare:
|
|
# Avoid uncommitted, unstaged, or untracked changes
|
|
if subproc.call('git diff --staged --quiet'.split()):
|
|
abort(_("you have uncommitted changes"))
|
|
if subproc.call('git diff --quiet'.split()):
|
|
abort(_("you have unstaged changes"))
|
|
if len(subproc.check_output('git ls-files -o'.split())) > 0:
|
|
abort(_("you have untracked changes"))
|
|
|
|
# Avoid unpushed changes
|
|
for refname, rev in refs.items():
|
|
if not refname.startswith(b'refs/heads/'):
|
|
continue
|
|
origin_ref = refname.replace(b'refs/heads/', b'refs/remotes/origin/')
|
|
if origin_ref not in refs:
|
|
abort(_('%s exists, but %s not found') % (decode(refname),
|
|
decode(origin_ref)))
|
|
if rev != refs[origin_ref]:
|
|
abort(_('%s does not match %s') % (decode(refname),
|
|
decode(origin_ref)))
|
|
|
|
# Make sure there is only one worktree
|
|
output = subproc.check_output('git worktree list'.split())
|
|
if len(output.splitlines()) > 1:
|
|
abort(_('you have multiple worktrees'))
|
|
|
|
@staticmethod
|
|
def cleanup(repo, repack, reset, run_quietly=False, show_debuginfo=False):
|
|
''' cleanup repo; if repack then expire reflogs and do a gc --prune=now.
|
|
if reset then do a reset --hard. Optionally also curb output if
|
|
run_quietly is True, or go the opposite direction and show extra
|
|
output if show_debuginfo is True. '''
|
|
assert not (run_quietly and show_debuginfo)
|
|
|
|
if (repack and not run_quietly and not show_debuginfo):
|
|
print(_("Repacking your repo and cleaning out old unneeded objects"))
|
|
quiet_flags = '--quiet' if run_quietly else ''
|
|
cleanup_cmds = []
|
|
if repack:
|
|
cleanup_cmds = ['git reflog expire --expire=now --all'.split(),
|
|
'git gc {} --prune=now'.format(quiet_flags).split()]
|
|
if reset:
|
|
cleanup_cmds.insert(0, 'git reset {} --hard'.format(quiet_flags).split())
|
|
location_info = ' (in {})'.format(decode(repo)) if repo != b'.' else ''
|
|
for cmd in cleanup_cmds:
|
|
if show_debuginfo:
|
|
print("[DEBUG] Running{}: {}".format(location_info, ' '.join(cmd)))
|
|
subproc.call(cmd, cwd=repo)
|
|
|
|
def _get_rename(self, old_hash):
|
|
# If we already know the rename, just return it
|
|
new_hash = self._commit_renames.get(old_hash, None)
|
|
if new_hash:
|
|
return new_hash
|
|
|
|
# If it's not in the remaining pending renames, we don't know it
|
|
if old_hash is not None and old_hash not in self._pending_renames:
|
|
return None
|
|
|
|
# Read through the pending renames until we find it or we've read them all,
|
|
# and return whatever we might find
|
|
self._flush_renames(old_hash)
|
|
return self._commit_renames.get(old_hash, None)
|
|
|
|
def _flush_renames(self, old_hash=None, limit=0):
|
|
# Parse through self._pending_renames until we have read enough. We have
|
|
# read enough if:
|
|
# self._pending_renames is empty
|
|
# old_hash != None and we found a rename for old_hash
|
|
# limit > 0 and len(self._pending_renames) started less than 2*limit
|
|
# limit > 0 and len(self._pending_renames) < limit
|
|
if limit and len(self._pending_renames) < 2 * limit:
|
|
return
|
|
fi_input, fi_output = self._import_pipes
|
|
while self._pending_renames:
|
|
orig_id, ignore = self._pending_renames.popitem(last=False)
|
|
new_id = fi_output.readline().rstrip()
|
|
self._commit_renames[orig_id] = new_id
|
|
if old_hash == orig_id:
|
|
return
|
|
if limit and len(self._pending_renames) < limit:
|
|
return
|
|
|
|
def _translate_commit_hash(self, matchobj_or_oldhash):
|
|
old_hash = matchobj_or_oldhash
|
|
if not isinstance(matchobj_or_oldhash, bytes):
|
|
old_hash = matchobj_or_oldhash.group(1)
|
|
orig_len = len(old_hash)
|
|
new_hash = self._get_rename(old_hash)
|
|
if new_hash is None:
|
|
if old_hash[0:7] not in self._commit_short_old_hashes:
|
|
self._commits_referenced_but_removed.add(old_hash)
|
|
return old_hash
|
|
possibilities = self._commit_short_old_hashes[old_hash[0:7]]
|
|
matches = [x for x in possibilities
|
|
if x[0:orig_len] == old_hash]
|
|
if len(matches) != 1:
|
|
self._commits_referenced_but_removed.add(old_hash)
|
|
return old_hash
|
|
old_hash = matches[0]
|
|
new_hash = self._get_rename(old_hash)
|
|
|
|
assert new_hash is not None
|
|
return new_hash[0:orig_len]
|
|
|
|
def _trim_extra_parents(self, orig_parents, parents):
|
|
'''Due to pruning of empty commits, some parents could be non-existent
|
|
(None) or otherwise redundant. Remove the non-existent parents, and
|
|
remove redundant parents so long as that doesn't transform a merge
|
|
commit into a non-merge commit.
|
|
|
|
Returns a tuple:
|
|
(parents, new_first_parent_if_would_become_non_merge)'''
|
|
|
|
always_prune = (self._args.prune_degenerate == 'always')
|
|
|
|
# Pruning of empty commits means multiple things:
|
|
# * An original parent of this commit may have been pruned causing the
|
|
# need to rewrite the reported parent to the nearest ancestor. We
|
|
# want to know when we're dealing with such a parent.
|
|
# * Further, there may be no "nearest ancestor" if the entire history
|
|
# of that parent was also pruned. (Detectable by the parent being
|
|
# 'None')
|
|
# Remove all parents rewritten to None, and keep track of which parents
|
|
# were rewritten to an ancestor.
|
|
tmp = zip(parents,
|
|
orig_parents,
|
|
[(x in _SKIPPED_COMMITS or always_prune) for x in orig_parents])
|
|
tmp2 = [x for x in tmp if x[0] is not None]
|
|
if not tmp2:
|
|
# All ancestors have been pruned; we have no parents.
|
|
return [], None
|
|
parents, orig_parents, is_rewritten = [list(x) for x in zip(*tmp2)]
|
|
|
|
# We can't have redundant parents if we don't have at least 2 parents
|
|
if len(parents) < 2:
|
|
return parents, None
|
|
|
|
# Don't remove redundant parents if user doesn't want us to
|
|
if self._args.prune_degenerate == 'never':
|
|
return parents, None
|
|
|
|
# Remove duplicate parents (if both sides of history have lots of commits
|
|
# which become empty due to pruning, the most recent ancestor on both
|
|
# sides may be the same commit), except only remove parents that have
|
|
# been rewritten due to previous empty pruning.
|
|
seen = set()
|
|
seen_add = seen.add
|
|
# Deleting duplicate rewritten parents means keeping parents if either
|
|
# they have not been seen or they are ones that have not been rewritten.
|
|
parents_copy = parents
|
|
uniq = [[p, orig_parents[i], is_rewritten[i]] for i, p in enumerate(parents)
|
|
if not (p in seen or seen_add(p)) or not is_rewritten[i]]
|
|
parents, orig_parents, is_rewritten = [list(x) for x in zip(*uniq)]
|
|
if len(parents) < 2:
|
|
return parents_copy, parents[0]
|
|
|
|
# Flatten unnecessary merges. (If one side of history is entirely
|
|
# empty commits that were pruned, we may end up attempting to
|
|
# merge a commit with its ancestor. Remove parents that are an
|
|
# ancestor of another parent.)
|
|
num_parents = len(parents)
|
|
to_remove = []
|
|
for cur in range(num_parents):
|
|
if not is_rewritten[cur]:
|
|
continue
|
|
for other in range(num_parents):
|
|
if cur == other:
|
|
continue
|
|
if not self._graph.is_ancestor(parents[cur], parents[other]):
|
|
continue
|
|
# parents[cur] is an ancestor of parents[other], so parents[cur]
|
|
# seems redundant. However, if it was intentionally redundant
|
|
# (e.g. a no-ff merge) in the original, then we want to keep it.
|
|
if not always_prune and \
|
|
self._orig_graph.is_ancestor(orig_parents[cur],
|
|
orig_parents[other]):
|
|
continue
|
|
# Some folks want their history to have all first parents be merge
|
|
# commits (except for any root commits), and always do a merge --no-ff.
|
|
# For such folks, don't remove the first parent even if it's an
|
|
# ancestor of other commits.
|
|
if self._args.no_ff and cur == 0:
|
|
continue
|
|
# Okay so the cur-th parent is an ancestor of the other-th parent,
|
|
# and it wasn't that way in the original repository; mark the
|
|
# cur-th parent as removable.
|
|
to_remove.append(cur)
|
|
break # cur removed, so skip rest of others -- i.e. check cur+=1
|
|
for x in reversed(to_remove):
|
|
parents.pop(x)
|
|
if len(parents) < 2:
|
|
return parents_copy, parents[0]
|
|
|
|
return parents, None
|
|
|
|
def _prunable(self, commit, new_1st_parent, had_file_changes, orig_parents):
|
|
parents = commit.parents
|
|
|
|
if self._args.prune_empty == 'never':
|
|
return False
|
|
always_prune = (self._args.prune_empty == 'always')
|
|
|
|
# For merge commits, unless there are prunable (redundant) parents, we
|
|
# do not want to prune
|
|
if len(parents) >= 2 and not new_1st_parent:
|
|
return False
|
|
|
|
if len(parents) < 2:
|
|
# Special logic for commits that started empty...
|
|
if not had_file_changes and not always_prune:
|
|
had_parents_pruned = (len(parents) < len(orig_parents) or
|
|
(len(orig_parents) == 1 and
|
|
orig_parents[0] in _SKIPPED_COMMITS))
|
|
# If the commit remains empty and had parents which were pruned,
|
|
# then prune this commit; otherwise, retain it
|
|
return (not commit.file_changes and had_parents_pruned)
|
|
|
|
# We can only get here if the commit didn't start empty, so if it's
|
|
# empty now, it obviously became empty
|
|
if not commit.file_changes:
|
|
return True
|
|
|
|
# If there are no parents of this commit and we didn't match the case
|
|
# above, then this commit cannot be pruned. Since we have no parent(s)
|
|
# to compare to, abort now to prevent future checks from failing.
|
|
if not parents:
|
|
return False
|
|
|
|
# Similarly, we cannot handle the hard cases if we don't have a pipe
|
|
# to communicate with fast-import
|
|
if not self._import_pipes:
|
|
return False
|
|
|
|
# If there have not been renames/remappings of IDs (due to insertion of
|
|
# new blobs), then we can sometimes know things aren't prunable with a
|
|
# simple check
|
|
if not _IDS.has_renames():
|
|
# non-merge commits can only be empty if blob/file-change editing caused
|
|
# all file changes in the commit to have the same file contents as
|
|
# the parent.
|
|
changed_files = set(change.filename for change in commit.file_changes)
|
|
if len(orig_parents) < 2 and changed_files - self._files_tweaked:
|
|
return False
|
|
|
|
# Finally, the hard case: due to either blob rewriting, or due to pruning
|
|
# of empty commits wiping out the first parent history back to the merge
|
|
# base, the list of file_changes we have may not actually differ from our
|
|
# (new) first parent's version of the files, i.e. this would actually be
|
|
# an empty commit. Check by comparing the contents of this commit to its
|
|
# (remaining) parent.
|
|
#
|
|
# NOTE on why this works, for the case of original first parent history
|
|
# having been pruned away due to being empty:
|
|
# The first parent history having been pruned away due to being
|
|
# empty implies the original first parent would have a tree (after
|
|
# filtering) that matched the merge base's tree. Since
|
|
# file_changes has the changes needed to go from what would have
|
|
# been the first parent to our new commit, and what would have been
|
|
# our first parent has a tree that matches the merge base, then if
|
|
# the new first parent has a tree matching the versions of files in
|
|
# file_changes, then this new commit is empty and thus prunable.
|
|
fi_input, fi_output = self._import_pipes
|
|
self._flush_renames() # Avoid fi_output having other stuff present
|
|
# Optimization note: we could have two loops over file_changes, the
|
|
# first doing all the self._output.write() calls, and the second doing
|
|
# the rest. But I'm worried about fast-import blocking on fi_output
|
|
# buffers filling up so I instead read from it as I go.
|
|
for change in commit.file_changes:
|
|
parent = new_1st_parent or commit.parents[0] # exists due to above checks
|
|
quoted_filename = PathQuoting.enquote(change.filename)
|
|
if isinstance(parent, int):
|
|
self._output.write(b"ls :%d %s\n" % (parent, quoted_filename))
|
|
else:
|
|
self._output.write(b"ls %s %s\n" % (parent, quoted_filename))
|
|
self._output.flush()
|
|
parent_version = fi_output.readline().split()
|
|
if change.type == b'D':
|
|
if parent_version != [b'missing', quoted_filename]:
|
|
return False
|
|
else:
|
|
blob_sha = change.blob_id
|
|
if isinstance(change.blob_id, int):
|
|
self._output.write(b"get-mark :%d\n" % change.blob_id)
|
|
self._output.flush()
|
|
blob_sha = fi_output.readline().rstrip()
|
|
if parent_version != [change.mode, b'blob', blob_sha, quoted_filename]:
|
|
return False
|
|
|
|
return True
|
|
|
|
def _record_remapping(self, commit, orig_parents):
|
|
new_id = None
|
|
# Record the mapping of old commit hash to new one
|
|
if commit.original_id and self._import_pipes:
|
|
fi_input, fi_output = self._import_pipes
|
|
self._output.write(b"get-mark :%d\n" % commit.id)
|
|
self._output.flush()
|
|
orig_id = commit.original_id
|
|
self._commit_short_old_hashes[orig_id[0:7]].add(orig_id)
|
|
# Note that we have queued up an id for later reading; flush a
|
|
# few of the older ones if we have too many queued up
|
|
self._pending_renames[orig_id] = None
|
|
self._flush_renames(None, limit=40)
|
|
# Also, record if this was a merge commit that turned into a non-merge
|
|
# commit.
|
|
if len(orig_parents) >= 2 and len(commit.parents) < 2:
|
|
self._commits_no_longer_merges.append((commit.original_id, new_id))
|
|
|
|
def callback_metadata(self, extra_items = dict()):
|
|
return {'commit_rename_func': self._translate_commit_hash,
|
|
'ancestry_graph': self._graph,
|
|
'original_ancestry_graph': self._orig_graph,
|
|
**extra_items}
|
|
|
|
def _tweak_blob(self, blob):
|
|
if self._args.max_blob_size and len(blob.data) > self._args.max_blob_size:
|
|
blob.skip()
|
|
|
|
if blob.original_id in self._args.strip_blobs_with_ids:
|
|
blob.skip()
|
|
|
|
if ( self._args.replace_text
|
|
# not (if blob contains zero byte in the first 8Kb, that is, if blob is binary data)
|
|
and not b"\0" in blob.data[0:8192]
|
|
):
|
|
for literal, replacement in self._args.replace_text['literals']:
|
|
blob.data = blob.data.replace(literal, replacement)
|
|
for regex, replacement in self._args.replace_text['regexes']:
|
|
blob.data = regex.sub(replacement, blob.data)
|
|
|
|
if self._blob_callback:
|
|
self._blob_callback(blob, self.callback_metadata())
|
|
|
|
def _filter_files(self, commit):
|
|
def filename_matches(path_expression, pathname):
|
|
''' Returns whether path_expression matches pathname or a leading
|
|
directory thereof, allowing path_expression to not have a trailing
|
|
slash even if it is meant to match a leading directory. '''
|
|
if path_expression == b'':
|
|
return True
|
|
n = len(path_expression)
|
|
if (pathname.startswith(path_expression) and
|
|
(path_expression[n-1:n] == b'/' or
|
|
len(pathname) == n or
|
|
pathname[n:n+1] == b'/')):
|
|
return True
|
|
return False
|
|
|
|
def newname(path_changes, pathname, use_base_name, filtering_is_inclusive):
|
|
''' Applies filtering and rename changes from path_changes to pathname,
|
|
returning any of None (file isn't wanted), original filename (file
|
|
is wanted with original name), or new filename. '''
|
|
wanted = False
|
|
full_pathname = pathname
|
|
if use_base_name:
|
|
pathname = os.path.basename(pathname)
|
|
for (mod_type, match_type, path_exp) in path_changes:
|
|
if mod_type == 'filter' and not wanted:
|
|
assert match_type in ('match', 'glob', 'regex')
|
|
if match_type == 'match' and filename_matches(path_exp, pathname):
|
|
wanted = True
|
|
if match_type == 'glob' and fnmatch.fnmatch(pathname, path_exp):
|
|
wanted = True
|
|
if match_type == 'regex' and path_exp.search(pathname):
|
|
wanted = True
|
|
elif mod_type == 'rename':
|
|
match, repl = path_exp
|
|
assert match_type in ('match','regex') # glob was translated to regex
|
|
if match_type == 'match' and filename_matches(match, full_pathname):
|
|
full_pathname = full_pathname.replace(match, repl, 1)
|
|
if match_type == 'regex':
|
|
full_pathname = match.sub(repl, full_pathname)
|
|
return full_pathname if (wanted == filtering_is_inclusive) else None
|
|
|
|
args = self._args
|
|
new_file_changes = {} # Assumes no renames or copies, otherwise collisions
|
|
for change in commit.file_changes:
|
|
# NEEDSWORK: _If_ we ever want to pass `--full-tree` to fast-export and
|
|
# parse that output, we'll need to modify this block; `--full-tree`
|
|
# issues a deleteall directive which has no filename, and thus this
|
|
# block would normally strip it. Of course, FileChange() and
|
|
# _parse_optional_filechange() would need updates too.
|
|
if change.type == b'DELETEALL':
|
|
new_file_changes[b''] = change
|
|
continue
|
|
if change.filename in self._newnames:
|
|
change.filename = self._newnames[change.filename]
|
|
else:
|
|
original_filename = change.filename
|
|
change.filename = newname(args.path_changes, change.filename,
|
|
args.use_base_name, args.inclusive)
|
|
if self._filename_callback:
|
|
change.filename = self._filename_callback(change.filename)
|
|
self._newnames[original_filename] = change.filename
|
|
if not change.filename:
|
|
continue # Filtering criteria excluded this file; move on to next one
|
|
if change.filename in new_file_changes:
|
|
# Getting here means that path renaming is in effect, and caused one
|
|
# path to collide with another. That's usually bad, but can be okay
|
|
# under two circumstances:
|
|
# 1) Sometimes people have a file named OLDFILE in old revisions of
|
|
# history, and they rename to NEWFILE, and would like to rewrite
|
|
# history so that all revisions refer to it as NEWFILE. As such,
|
|
# we can allow a collision when (at least) one of the two paths
|
|
# is a deletion. Note that if OLDFILE and NEWFILE are unrelated
|
|
# this also allows the rewrite to continue, which makes sense
|
|
# since OLDFILE is no longer in the way.
|
|
# 2) If OLDFILE and NEWFILE are exactly equal, then writing them
|
|
# both to the same location poses no problem; we only need one
|
|
# file. (This could come up if someone copied a file in some
|
|
# commit, then later either deleted the file or kept it exactly
|
|
# in sync with the original with any changes, and then decides
|
|
# they want to rewrite history to only have one of the two files)
|
|
colliding_change = new_file_changes[change.filename]
|
|
if change.type == b'D':
|
|
# We can just throw this one away and keep the other
|
|
continue
|
|
elif change.type == b'M' and (
|
|
change.mode == colliding_change.mode and
|
|
change.blob_id == colliding_change.blob_id):
|
|
# The two are identical, so we can throw this one away and keep other
|
|
continue
|
|
elif new_file_changes[change.filename].type != b'D':
|
|
raise SystemExit(_("File renaming caused colliding pathnames!\n") +
|
|
_(" Commit: {}\n").format(commit.original_id) +
|
|
_(" Filename: {}").format(change.filename))
|
|
# Strip files that are too large
|
|
if self._args.max_blob_size and \
|
|
self._unpacked_size.get(change.blob_id, 0) > self._args.max_blob_size:
|
|
continue
|
|
if self._args.strip_blobs_with_ids and \
|
|
change.blob_id in self._args.strip_blobs_with_ids:
|
|
continue
|
|
# Otherwise, record the change
|
|
new_file_changes[change.filename] = change
|
|
commit.file_changes = [v for k,v in sorted(new_file_changes.items())]
|
|
|
|
def _tweak_commit(self, commit, aux_info):
|
|
# Change the commit message according to callback
|
|
if not self._args.preserve_commit_hashes:
|
|
commit.message = self._hash_re.sub(self._translate_commit_hash,
|
|
commit.message)
|
|
if self._args.replace_message:
|
|
for literal, replacement in self._args.replace_message['literals']:
|
|
commit.message = commit.message.replace(literal, replacement)
|
|
for regex, replacement in self._args.replace_message['regexes']:
|
|
commit.message = regex.sub(replacement, commit.message)
|
|
if self._message_callback:
|
|
commit.message = self._message_callback(commit.message)
|
|
|
|
# Change the author & committer according to mailmap rules
|
|
args = self._args
|
|
if args.mailmap:
|
|
commit.author_name, commit.author_email = \
|
|
args.mailmap.translate(commit.author_name, commit.author_email)
|
|
commit.committer_name, commit.committer_email = \
|
|
args.mailmap.translate(commit.committer_name, commit.committer_email)
|
|
# Change author & committer according to callbacks
|
|
if self._name_callback:
|
|
commit.author_name = self._name_callback(commit.author_name)
|
|
commit.committer_name = self._name_callback(commit.committer_name)
|
|
if self._email_callback:
|
|
commit.author_email = self._email_callback(commit.author_email)
|
|
commit.committer_email = self._email_callback(commit.committer_email)
|
|
|
|
# Sometimes the 'branch' given is a tag; if so, rename it as requested so
|
|
# we don't get any old tagnames
|
|
if self._args.tag_rename:
|
|
commit.branch = RepoFilter._do_tag_rename(args.tag_rename, commit.branch)
|
|
if self._refname_callback:
|
|
commit.branch = self._refname_callback(commit.branch)
|
|
|
|
# Filter or rename the list of file changes
|
|
orig_file_changes = set(commit.file_changes)
|
|
self._filter_files(commit)
|
|
|
|
# Record ancestry graph
|
|
parents, orig_parents = commit.parents, aux_info['orig_parents']
|
|
if self._args.state_branch:
|
|
external_parents = parents
|
|
else:
|
|
external_parents = [p for p in parents if not isinstance(p, int)]
|
|
self._graph.record_external_commits(external_parents)
|
|
self._orig_graph.record_external_commits(external_parents)
|
|
self._graph.add_commit_and_parents(commit.id, parents)
|
|
self._orig_graph.add_commit_and_parents(commit.old_id, orig_parents)
|
|
|
|
# Prune parents (due to pruning of empty commits) if relevant
|
|
old_1st_parent = parents[0] if parents else None
|
|
parents, new_1st_parent = self._trim_extra_parents(orig_parents, parents)
|
|
commit.parents = parents
|
|
|
|
# If parents were pruned, then we need our file changes to be relative
|
|
# to the new first parent
|
|
if parents and old_1st_parent != parents[0]:
|
|
commit.file_changes = GitUtils.get_file_changes(self._repo_working_dir,
|
|
ID_TO_HASH[parents[0]],
|
|
commit.original_id)
|
|
orig_file_changes = set(commit.file_changes)
|
|
self._filter_files(commit)
|
|
|
|
# Find out which files were modified by the callbacks. Such paths could
|
|
# lead to subsequent commits being empty (e.g. if removing a line containing
|
|
# a password from every version of a file that had the password, and some
|
|
# later commit did nothing more than remove that line)
|
|
final_file_changes = set(commit.file_changes)
|
|
if self._args.replace_text or self._blob_callback:
|
|
differences = orig_file_changes.union(final_file_changes)
|
|
else:
|
|
differences = orig_file_changes.symmetric_difference(final_file_changes)
|
|
self._files_tweaked.update(x.filename for x in differences)
|
|
|
|
# Call the user-defined callback, if any
|
|
if self._commit_callback:
|
|
self._commit_callback(commit, self.callback_metadata(aux_info))
|
|
|
|
# Now print the resulting commit, or if prunable skip it
|
|
if not commit.dumped:
|
|
if not self._prunable(commit, new_1st_parent,
|
|
aux_info['had_file_changes'], orig_parents):
|
|
self._insert_into_stream(commit)
|
|
self._record_remapping(commit, orig_parents)
|
|
else:
|
|
rewrite_to = new_1st_parent or commit.first_parent()
|
|
commit.skip(new_id = rewrite_to)
|
|
if self._args.state_branch:
|
|
alias = Alias(commit.old_id or commit.id, rewrite_to or deleted_hash)
|
|
self._insert_into_stream(alias)
|
|
reset = Reset(commit.branch, rewrite_to or deleted_hash)
|
|
self._insert_into_stream(reset)
|
|
self._commit_renames[commit.original_id] = None
|
|
|
|
# Show progress
|
|
self._num_commits += 1
|
|
if not self._args.quiet:
|
|
self._progress_writer.show(self._parsed_message % self._num_commits)
|
|
|
|
@staticmethod
|
|
def _do_tag_rename(rename_pair, tagname):
|
|
old, new = rename_pair.split(b':', 1)
|
|
old, new = b'refs/tags/'+old, b'refs/tags/'+new
|
|
if tagname.startswith(old):
|
|
return tagname.replace(old, new, 1)
|
|
return tagname
|
|
|
|
def _tweak_tag(self, tag):
|
|
# Tweak the tag message according to callbacks
|
|
if self._args.replace_message:
|
|
for literal, replacement in self._args.replace_message['literals']:
|
|
tag.message = tag.message.replace(literal, replacement)
|
|
for regex, replacement in self._args.replace_message['regexes']:
|
|
tag.message = regex.sub(replacement, tag.message)
|
|
if self._message_callback:
|
|
tag.message = self._message_callback(tag.message)
|
|
|
|
# Tweak the tag name according to tag-name-related callbacks
|
|
tag_prefix = b'refs/tags/'
|
|
fullref = tag_prefix+tag.ref
|
|
if self._args.tag_rename:
|
|
fullref = RepoFilter._do_tag_rename(self._args.tag_rename, fullref)
|
|
if self._refname_callback:
|
|
fullref = self._refname_callback(fullref)
|
|
if not fullref.startswith(tag_prefix):
|
|
msg = "Error: fast-import requires tags to be in refs/tags/ namespace."
|
|
msg += "\n {} renamed to {}".format(tag_prefix+tag.ref, fullref)
|
|
raise SystemExit(msg)
|
|
tag.ref = fullref[len(tag_prefix):]
|
|
|
|
# Tweak the tagger according to callbacks
|
|
if self._args.mailmap:
|
|
tag.tagger_name, tag.tagger_email = \
|
|
self._args.mailmap.translate(tag.tagger_name, tag.tagger_email)
|
|
if self._name_callback:
|
|
tag.tagger_name = self._name_callback(tag.tagger_name)
|
|
if self._email_callback:
|
|
tag.tagger_email = self._email_callback(tag.tagger_email)
|
|
|
|
# Call general purpose tag callback
|
|
if self._tag_callback:
|
|
self._tag_callback(tag, self.callback_metadata())
|
|
|
|
def _tweak_reset(self, reset):
|
|
if self._args.tag_rename:
|
|
reset.ref = RepoFilter._do_tag_rename(self._args.tag_rename, reset.ref)
|
|
if self._refname_callback:
|
|
reset.ref = self._refname_callback(reset.ref)
|
|
if self._reset_callback:
|
|
self._reset_callback(reset, self.callback_metadata())
|
|
|
|
def results_tmp_dir(self, create_if_missing=True):
|
|
target_working_dir = self._args.target or b'.'
|
|
git_dir = GitUtils.determine_git_dir(target_working_dir)
|
|
d = os.path.join(git_dir, b'filter-repo')
|
|
if create_if_missing and not os.path.isdir(d):
|
|
os.mkdir(d)
|
|
return d
|
|
|
|
def _load_marks_file(self, marks_basename):
|
|
full_branch = 'refs/heads/{}'.format(self._args.state_branch)
|
|
marks_file = os.path.join(self.results_tmp_dir(), marks_basename)
|
|
working_dir = self._args.target or b'.'
|
|
cmd = ['git', '-C', working_dir, 'show-ref', full_branch]
|
|
contents = b''
|
|
if subproc.call(cmd, stdout=subprocess.DEVNULL) == 0:
|
|
cmd = ['git', '-C', working_dir, 'show',
|
|
'%s:%s' % (full_branch, decode(marks_basename))]
|
|
try:
|
|
contents = subproc.check_output(cmd)
|
|
except subprocess.CalledProcessError as e: # pragma: no cover
|
|
raise SystemExit(_("Failed loading %s from %s") %
|
|
(decode(marks_basename), full_branch))
|
|
if contents:
|
|
biggest_id = max(int(x.split()[0][1:]) for x in contents.splitlines())
|
|
_IDS._next_id = max(_IDS._next_id, biggest_id+1)
|
|
with open(marks_file, 'bw') as f:
|
|
f.write(contents)
|
|
return marks_file
|
|
|
|
def _save_marks_files(self):
|
|
basenames = [b'source-marks', b'target-marks']
|
|
working_dir = self._args.target or b'.'
|
|
|
|
# Check whether the branch exists
|
|
parent = []
|
|
full_branch = 'refs/heads/{}'.format(self._args.state_branch)
|
|
cmd = ['git', '-C', working_dir, 'show-ref', full_branch]
|
|
if subproc.call(cmd, stdout=subprocess.DEVNULL) == 0:
|
|
parent = ['-p', full_branch]
|
|
|
|
# Run 'git hash-object $MARKS_FILE' for each marks file, save result
|
|
blob_hashes = {}
|
|
for marks_basename in basenames:
|
|
marks_file = os.path.join(self.results_tmp_dir(), marks_basename)
|
|
if not os.path.isfile(marks_file): # pragma: no cover
|
|
raise SystemExit(_("Failed to find %s to save to %s")
|
|
% (marks_file, self._args.state_branch))
|
|
cmd = ['git', '-C', working_dir, 'hash-object', '-w', marks_file]
|
|
blob_hashes[marks_basename] = subproc.check_output(cmd).strip()
|
|
|
|
# Run 'git mktree' to create a tree out of it
|
|
p = subproc.Popen(['git', '-C', working_dir, 'mktree'],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
|
|
for b in basenames:
|
|
p.stdin.write(b'100644 blob %s\t%s\n' % (blob_hashes[b], b))
|
|
p.stdin.close()
|
|
p.wait()
|
|
tree = p.stdout.read().strip()
|
|
|
|
# Create the new commit
|
|
cmd = (['git', '-C', working_dir, 'commit-tree', '-m', 'New mark files',
|
|
tree] + parent)
|
|
commit = subproc.check_output(cmd).strip()
|
|
subproc.call(['git', '-C', working_dir, 'update-ref', full_branch, commit])
|
|
|
|
def importer_only(self):
|
|
self._run_sanity_checks()
|
|
self._setup_output()
|
|
|
|
def set_output(self, outputRepoFilter):
|
|
assert outputRepoFilter._output
|
|
|
|
# set_output implies this RepoFilter is doing exporting, though may not
|
|
# be the only one.
|
|
self._setup_input(use_done_feature = False)
|
|
|
|
# Set our output management up to pipe to outputRepoFilter's locations
|
|
self._managed_output = False
|
|
self._output = outputRepoFilter._output
|
|
self._import_pipes = outputRepoFilter._import_pipes
|
|
|
|
# Handle sanity checks, though currently none needed for export-only cases
|
|
self._run_sanity_checks()
|
|
|
|
def _setup_input(self, use_done_feature):
|
|
if self._args.stdin:
|
|
self._input = sys.stdin.detach()
|
|
sys.stdin = None # Make sure no one tries to accidentally use it
|
|
self._fe_orig = None
|
|
else:
|
|
skip_blobs = (self._blob_callback is None and
|
|
self._args.replace_text is None and
|
|
self._args.source == self._args.target)
|
|
extra_flags = []
|
|
if skip_blobs:
|
|
extra_flags.append('--no-data')
|
|
if self._args.max_blob_size:
|
|
self._unpacked_size, packed_size = GitUtils.get_blob_sizes()
|
|
if use_done_feature:
|
|
extra_flags.append('--use-done-feature')
|
|
if write_marks:
|
|
extra_flags.append(b'--mark-tags')
|
|
if self._args.state_branch:
|
|
assert(write_marks)
|
|
source_marks_file = self._load_marks_file(b'source-marks')
|
|
extra_flags.extend([b'--export-marks='+source_marks_file,
|
|
b'--import-marks='+source_marks_file])
|
|
if self._args.preserve_commit_encoding is not None: # pragma: no cover
|
|
reencode = 'no' if self._args.preserve_commit_encoding else 'yes'
|
|
extra_flags.append('--reencode='+reencode)
|
|
location = ['-C', self._args.source] if self._args.source else []
|
|
fep_cmd = ['git'] + location + ['fast-export', '--show-original-ids',
|
|
'--signed-tags=strip', '--tag-of-filtered-object=rewrite',
|
|
'--fake-missing-tagger', '--reference-excluded-parents'
|
|
] + extra_flags + self._args.refs
|
|
self._fep = subproc.Popen(fep_cmd, bufsize=-1, stdout=subprocess.PIPE)
|
|
self._input = self._fep.stdout
|
|
if self._args.dry_run or self._args.debug:
|
|
self._fe_orig = os.path.join(self.results_tmp_dir(),
|
|
b'fast-export.original')
|
|
output = open(self._fe_orig, 'bw')
|
|
self._input = InputFileBackup(self._input, output)
|
|
if self._args.debug:
|
|
tmp = [decode(x) if isinstance(x, bytes) else x for x in fep_cmd]
|
|
print("[DEBUG] Running: {}".format(' '.join(tmp)))
|
|
print(" (saving a copy of the output at {})"
|
|
.format(decode(self._fe_orig)))
|
|
|
|
def _setup_output(self):
|
|
if not self._args.dry_run:
|
|
location = ['-C', self._args.target] if self._args.target else []
|
|
fip_cmd = ['git'] + location + ['-c', 'core.ignorecase=false',
|
|
'fast-import', '--force', '--quiet']
|
|
if date_format_permissive:
|
|
fip_cmd.append('--date-format=raw-permissive')
|
|
if self._args.state_branch:
|
|
target_marks_file = self._load_marks_file(b'target-marks')
|
|
fip_cmd.extend([b'--export-marks='+target_marks_file,
|
|
b'--import-marks='+target_marks_file])
|
|
self._fip = subproc.Popen(fip_cmd, bufsize=-1,
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
|
|
self._import_pipes = (self._fip.stdin, self._fip.stdout)
|
|
if self._args.dry_run or self._args.debug:
|
|
self._fe_filt = os.path.join(self.results_tmp_dir(),
|
|
b'fast-export.filtered')
|
|
self._output = open(self._fe_filt, 'bw')
|
|
else:
|
|
self._output = self._fip.stdin
|
|
if self._args.debug and not self._args.dry_run:
|
|
self._output = DualFileWriter(self._fip.stdin, self._output)
|
|
tmp = [decode(x) if isinstance(x, bytes) else x for x in fip_cmd]
|
|
print("[DEBUG] Running: {}".format(' '.join(tmp)))
|
|
print(" (using the following file as input: {})"
|
|
.format(decode(self._fe_filt)))
|
|
|
|
def _migrate_origin_to_heads(self):
|
|
refs_to_migrate = set(x for x in self._orig_refs
|
|
if x.startswith(b'refs/remotes/origin/'))
|
|
if not refs_to_migrate:
|
|
return
|
|
if self._args.debug:
|
|
print("[DEBUG] Migrating refs/remotes/origin/* -> refs/heads/*")
|
|
target_working_dir = self._args.target or b'.'
|
|
p = subproc.Popen('git update-ref --no-deref --stdin'.split(),
|
|
stdin=subprocess.PIPE, cwd=target_working_dir)
|
|
for ref in refs_to_migrate:
|
|
if ref == b'refs/remotes/origin/HEAD':
|
|
p.stdin.write(b'delete %s %s\n' % (ref, self._orig_refs[ref]))
|
|
del self._orig_refs[ref]
|
|
continue
|
|
newref = ref.replace(b'refs/remotes/origin/', b'refs/heads/')
|
|
if newref not in self._orig_refs:
|
|
p.stdin.write(b'create %s %s\n' % (newref, self._orig_refs[ref]))
|
|
p.stdin.write(b'delete %s %s\n' % (ref, self._orig_refs[ref]))
|
|
self._orig_refs[newref] = self._orig_refs[ref]
|
|
del self._orig_refs[ref]
|
|
p.stdin.close()
|
|
if p.wait():
|
|
raise SystemExit(_("git update-ref failed; see above")) # pragma: no cover
|
|
|
|
# Now remove
|
|
if self._args.debug:
|
|
print("[DEBUG] Removing 'origin' remote (rewritten history will no ")
|
|
print(" longer be related; consider re-pushing it elsewhere.")
|
|
subproc.call('git remote rm origin'.split(), cwd=target_working_dir)
|
|
|
|
def _final_commands(self):
|
|
self._finalize_handled = True
|
|
self._done_callback and self._done_callback()
|
|
|
|
if not self._args.quiet:
|
|
self._progress_writer.finish()
|
|
|
|
def _ref_update(self, target_working_dir):
|
|
# Start the update-ref process
|
|
p = subproc.Popen('git update-ref --no-deref --stdin'.split(),
|
|
stdin=subprocess.PIPE,
|
|
cwd=target_working_dir)
|
|
|
|
# Remove replace_refs from _orig_refs
|
|
replace_refs = {k:v for k, v in self._orig_refs.items()
|
|
if k.startswith(b'refs/replace/')}
|
|
reverse_replace_refs = collections.defaultdict(list)
|
|
for k,v in replace_refs.items():
|
|
reverse_replace_refs[v].append(k)
|
|
all(map(self._orig_refs.pop, replace_refs))
|
|
|
|
# Remove unused refs
|
|
exported_refs, imported_refs = self.get_exported_and_imported_refs()
|
|
refs_to_nuke = exported_refs - imported_refs
|
|
if self._args.partial:
|
|
refs_to_nuke = set()
|
|
if refs_to_nuke and self._args.debug:
|
|
print("[DEBUG] Deleting the following refs:\n "+
|
|
decode(b"\n ".join(refs_to_nuke)))
|
|
p.stdin.write(b''.join([b"delete %s\n" % x
|
|
for x in refs_to_nuke]))
|
|
|
|
# Delete or update and add replace_refs; note that fast-export automatically
|
|
# handles 'update-no-add', we only need to take action for the other four
|
|
# choices for replace_refs.
|
|
self._flush_renames()
|
|
actual_renames = {k:v for k,v in self._commit_renames.items() if k != v}
|
|
if self._args.replace_refs in ['delete-no-add', 'delete-and-add']:
|
|
# Delete old replace refs, if unwanted
|
|
replace_refs_to_nuke = set(replace_refs)
|
|
if self._args.replace_refs == 'delete-and-add':
|
|
# git-update-ref won't allow us to update a ref twice, so be careful
|
|
# to avoid deleting refs we'll later update
|
|
replace_refs_to_nuke = replace_refs_to_nuke.difference(
|
|
[b'refs/replace/'+x for x in actual_renames])
|
|
p.stdin.write(b''.join([b"delete %s\n" % x
|
|
for x in replace_refs_to_nuke]))
|
|
if self._args.replace_refs in ['delete-and-add', 'update-or-add',
|
|
'update-and-add']:
|
|
# Add new replace refs
|
|
update_only = (self._args.replace_refs == 'update-or-add')
|
|
p.stdin.write(b''.join([b"update refs/replace/%s %s\n" % (old, new)
|
|
for old,new in actual_renames.items()
|
|
if new and not (update_only and
|
|
old in reverse_replace_refs)]))
|
|
|
|
# Complete the update-ref process
|
|
p.stdin.close()
|
|
if p.wait():
|
|
raise SystemExit(_("git update-ref failed; see above")) # pragma: no cover
|
|
|
|
def _record_metadata(self, metadata_dir, orig_refs):
|
|
self._flush_renames()
|
|
with open(os.path.join(metadata_dir, b'commit-map'), 'bw') as f:
|
|
f.write(("%-40s %s\n" % (_("old"), _("new"))).encode())
|
|
for (old,new) in self._commit_renames.items():
|
|
msg = b'%s %s\n' % (old, new if new != None else deleted_hash)
|
|
f.write(msg)
|
|
|
|
exported_refs, imported_refs = self.get_exported_and_imported_refs()
|
|
|
|
batch_check_process = None
|
|
batch_check_output_re = re.compile(b'^([0-9a-f]{40}) ([a-z]+) ([0-9]+)$')
|
|
with open(os.path.join(metadata_dir, b'ref-map'), 'bw') as f:
|
|
f.write(("%-40s %-40s %s\n" % (_("old"), _("new"), _("ref"))).encode())
|
|
for refname, old_hash in orig_refs.items():
|
|
if refname not in exported_refs:
|
|
continue
|
|
if refname not in imported_refs:
|
|
new_hash = deleted_hash
|
|
elif old_hash in self._commit_renames:
|
|
new_hash = self._commit_renames[old_hash]
|
|
new_hash = new_hash if new_hash != None else deleted_hash
|
|
else: # Must be either an annotated tag, or a ref whose tip was pruned
|
|
if not batch_check_process:
|
|
cmd = 'git cat-file --batch-check'.split()
|
|
target_working_dir = self._args.target or b'.'
|
|
batch_check_process = subproc.Popen(cmd,
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
cwd=target_working_dir)
|
|
batch_check_process.stdin.write(refname+b"\n")
|
|
batch_check_process.stdin.flush()
|
|
line = batch_check_process.stdout.readline()
|
|
m = batch_check_output_re.match(line)
|
|
if m and m.group(2) in (b'tag', b'commit'):
|
|
new_hash = m.group(1)
|
|
elif line.endswith(b' missing\n'):
|
|
new_hash = deleted_hash
|
|
else:
|
|
raise SystemExit(_("Failed to find new id for %(refname)s "
|
|
"(old id was %(old_hash)s)")
|
|
% ({'refname': refname, 'old_hash': old_hash})
|
|
) # pragma: no cover
|
|
f.write(b'%s %s %s\n' % (old_hash, new_hash, refname))
|
|
if self._args.source or self._args.target:
|
|
new_refs = GitUtils.get_refs(self._args.target or b'.')
|
|
for ref, new_hash in new_refs.items():
|
|
if ref not in orig_refs and not ref.startswith(b'refs/replace/'):
|
|
old_hash = b'0'*len(new_hash)
|
|
f.write(b'%s %s %s\n' % (old_hash, new_hash, ref))
|
|
if batch_check_process:
|
|
batch_check_process.stdin.close()
|
|
batch_check_process.wait()
|
|
|
|
with open(os.path.join(metadata_dir, b'suboptimal-issues'), 'bw') as f:
|
|
issues_found = False
|
|
if self._commits_no_longer_merges:
|
|
issues_found = True
|
|
|
|
f.write(textwrap.dedent(_('''
|
|
The following commits used to be merge commits but due to filtering
|
|
are now regular commits; they likely have suboptimal commit messages
|
|
(e.g. "Merge branch next into master"). Original commit hash on the
|
|
left, commit hash after filtering/rewriting on the right:
|
|
''')[1:]).encode())
|
|
for oldhash, newhash in self._commits_no_longer_merges:
|
|
f.write(' {} {}\n'.format(oldhash, newhash).encode())
|
|
f.write(b'\n')
|
|
|
|
if self._commits_referenced_but_removed:
|
|
issues_found = True
|
|
f.write(textwrap.dedent(_('''
|
|
The following commits were filtered out, but referenced in another
|
|
commit message. The reference to the now-nonexistent commit hash
|
|
(or a substring thereof) was left as-is in any commit messages:
|
|
''')[1:]).encode())
|
|
for bad_commit_reference in self._commits_referenced_but_removed:
|
|
f.write(' {}\n'.format(bad_commit_reference).encode())
|
|
f.write(b'\n')
|
|
|
|
if not issues_found:
|
|
f.write(_("No filtering problems encountered.\n").encode())
|
|
|
|
with open(os.path.join(metadata_dir, b'already_ran'), 'bw') as f:
|
|
f.write(_("This file exists to allow you to filter again without --force.\n").encode())
|
|
|
|
def finish(self):
|
|
''' Alternative to run() when there is no input of our own to parse,
|
|
meaning that run only really needs to close the handle to fast-import
|
|
and let it finish, thus making a call to "run" feel like a misnomer. '''
|
|
assert not self._input
|
|
assert self._managed_output
|
|
self.run()
|
|
|
|
def insert(self, obj, direct_insertion = False):
|
|
if not direct_insertion:
|
|
if type(obj) == Blob:
|
|
self._tweak_blob(obj)
|
|
elif type(obj) == Commit:
|
|
aux_info = {'orig_parents': obj.parents,
|
|
'had_file_changes': bool(obj.file_changes)}
|
|
self._tweak_commit(obj, aux_info)
|
|
elif type(obj) == Reset:
|
|
self._tweak_reset(obj)
|
|
elif type(obj) == Tag:
|
|
self._tweak_tag(obj)
|
|
self._insert_into_stream(obj)
|
|
|
|
def _insert_into_stream(self, obj):
|
|
if not obj.dumped:
|
|
if self._parser:
|
|
self._parser.insert(obj)
|
|
else:
|
|
obj.dump(self._output)
|
|
|
|
def get_exported_and_imported_refs(self):
|
|
return self._parser.get_exported_and_imported_refs()
|
|
|
|
def run(self):
|
|
start = time.time()
|
|
if not self._input and not self._output:
|
|
self._run_sanity_checks()
|
|
if not self._args.dry_run and not self._args.partial:
|
|
self._migrate_origin_to_heads()
|
|
self._setup_input(use_done_feature = True)
|
|
self._setup_output()
|
|
assert self._sanity_checks_handled
|
|
|
|
if self._input:
|
|
# Create and run the filter
|
|
self._repo_working_dir = self._args.source or b'.'
|
|
self._parser = FastExportParser(blob_callback = self._tweak_blob,
|
|
commit_callback = self._tweak_commit,
|
|
tag_callback = self._tweak_tag,
|
|
reset_callback = self._tweak_reset,
|
|
done_callback = self._final_commands)
|
|
self._parser.run(self._input, self._output)
|
|
if not self._finalize_handled:
|
|
self._final_commands()
|
|
|
|
# Make sure fast-export completed successfully
|
|
if not self._args.stdin and self._fep.wait():
|
|
raise SystemExit(_("Error: fast-export failed; see above.")) # pragma: no cover
|
|
self._input.close()
|
|
|
|
# If we're not the manager of self._output, we should avoid post-run cleanup
|
|
if not self._managed_output:
|
|
return
|
|
|
|
# Close the output and ensure fast-import successfully completes
|
|
self._output.close()
|
|
if not self._args.dry_run and self._fip.wait():
|
|
raise SystemExit(_("Error: fast-import failed; see above.")) # pragma: no cover
|
|
|
|
# With fast-export and fast-import complete, update state if requested
|
|
if self._args.state_branch:
|
|
self._save_marks_files()
|
|
|
|
# Notify user how long it took, before doing a gc and such
|
|
msg = "New history written in {:.2f} seconds..."
|
|
if self._args.repack:
|
|
msg = "New history written in {:.2f} seconds; now repacking/cleaning..."
|
|
print(msg.format(time.time()-start))
|
|
|
|
# Exit early, if requested
|
|
if self._args.dry_run:
|
|
print(_("NOTE: Not running fast-import or cleaning up; --dry-run passed."))
|
|
if self._fe_orig:
|
|
print(_(" Requested filtering can be seen by comparing:"))
|
|
print(" " + decode(self._fe_orig))
|
|
else:
|
|
print(_(" Requested filtering can be seen at:"))
|
|
print(" " + decode(self._fe_filt))
|
|
return
|
|
|
|
target_working_dir = self._args.target or b'.'
|
|
if self._input:
|
|
self._ref_update(target_working_dir)
|
|
|
|
# Write out data about run
|
|
self._record_metadata(self.results_tmp_dir(), self._orig_refs)
|
|
|
|
# Final cleanup:
|
|
# If we need a repack, then nuke the reflogs and repack.
|
|
# If we need a reset, do a reset --hard
|
|
reset = not GitUtils.is_repository_bare(target_working_dir)
|
|
RepoFilter.cleanup(target_working_dir, self._args.repack, reset,
|
|
run_quietly=self._args.quiet,
|
|
show_debuginfo=self._args.debug)
|
|
|
|
# Let user know how long it took
|
|
print(_("Completely finished after {:.2f} seconds.")
|
|
.format(time.time()-start))
|
|
|
|
def main():
|
|
setup_gettext()
|
|
args = FilteringOptions.parse_args(sys.argv[1:])
|
|
if args.analyze:
|
|
RepoAnalyze.run(args)
|
|
else:
|
|
filter = RepoFilter(args)
|
|
filter.run()
|
|
|
|
if __name__ == '__main__':
|
|
main() |