From bd2d50c339a33190169242b5399d33071f3c1226 Mon Sep 17 00:00:00 2001 From: ArneBachmann Date: Thu, 1 Feb 2018 17:28:49 +0100 Subject: [PATCH] Fixes #153. Fixes #149. Fixes #147. --- install.bat | 4 +- install.sh | 4 +- setup.py | 13 ++---- sos/sos.coco | 46 ++++++++++--------- sos/tests.coco | 97 ++++++++++++++++++++++------------------ sos/utility.coco | 114 +++++++++++++++++++++++++++-------------------- 6 files changed, 151 insertions(+), 127 deletions(-) diff --git a/install.bat b/install.bat index 56bb218..8c8d0ff 100644 --- a/install.bat +++ b/install.bat @@ -1,6 +1,6 @@ echo NOMYPY=%NOMYPY% if "%NOMYPY%" == "" ( - pip install --upgrade appdirs chardet configr wcwidth coverage python-coveralls coconut[mypy] + pip install --upgrade appdirs chardet configr termwidth coverage python-coveralls coconut[mypy] ) else ( - pip install --upgrade appdirs chardet configr wcwidth coverage python-coveralls coconut + pip install --upgrade appdirs chardet configr termwidth coverage python-coveralls coconut ) \ No newline at end of file diff --git a/install.sh b/install.sh index 3cf0e91..d03b9fc 100644 --- a/install.sh +++ b/install.sh @@ -2,7 +2,7 @@ echo NOMYPY=$NOMYPY if [ "x$NOMYPY" == "x" ] then - pip install --upgrade appdirs chardet configr wcwidth coverage python-coveralls coconut[mypy] + pip install --upgrade appdirs chardet configr termwidth coverage python-coveralls coconut[mypy] else - pip install --upgrade appdirs chardet configr wcwidth coverage python-coveralls coconut + pip install --upgrade appdirs chardet configr termwidth coverage python-coveralls coconut fi \ No newline at end of file diff --git a/setup.py b/setup.py index 107e652..0eeacdc 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ print("Transpiling Coconut files to Python...") cmd = "-develop" if 0 == subprocess.Popen("coconut-develop --help", shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE, bufsize = 10000000).wait() and os.getenv("NODEV", "false").strip().lower() != "true" else "" - assert 0 == os.system("coconut%s %s -l -t 3 sos %s" % (cmd, "-p" if not "--mypy" in sys.argv else "", "--mypy" if "--mypy" in sys.argv else "")) # TODO remove --target once Python 2 problems have been fixed + assert 0 == os.system("coconut%s %s -l -t 3 sos %s" % (cmd, "-p" if not "--mypy" in sys.argv else "", "--mypy" if "--mypy" in sys.argv else "")) try: sys.argv.remove('--mypy') except: pass @@ -21,7 +21,7 @@ print("Preparing documentation for PyPI by converting from Markdown to reStructuredText via pandoc") try: so, se = subprocess.Popen("git describe --always", shell = sys.platform != 'win32', bufsize = 1, stdout = subprocess.PIPE).communicate() # use tag or hash - extra = (so.strip() if sys.version_info.major < 3 else so.strip().decode(sys.stdout.encoding)).replace("\n", "-") + extra = so.strip().decode(sys.stdout.encoding).replace("\n", "-") if "\x0d" in extra: extra = extra.split("\x0d")[1] print("Found Git hash %s" % extra) except: extra = "svn" @@ -94,7 +94,7 @@ Programming Language :: Python :: 3.6 Programming Language :: Python :: 3 :: Only """.split('\n') if c.strip()], # https://pypi.python.org/pypi?:action=list_classifiers -# Programming Language :: Coconut + # Programming Language :: Coconut keywords = 'VCS SCM version control system Subversion Git gitless Fossil Bazaar Mercurial CVS SVN gl fsl bzr hg', author = 'Arne Bachmann', author_email = 'ArneBachmann@users.noreply.github.com', @@ -109,12 +109,7 @@ zip_safe = False, entry_points = { 'console_scripts': [ - 'sos=sos.sos:main', # Subversion offline solution - # 'vcos=sos.sos:main', # version control offline solution - # 'mvcs=sos.sos:main' # meta version control system + 'sos=sos.sos:main' # Subversion offline solution ] - }, - extras_require = { - 'backport': ["enum34"] # , "option"] # for Python 2 without backported enum } ) diff --git a/sos/sos.coco b/sos/sos.coco index c767a93..09163f9 100644 --- a/sos/sos.coco +++ b/sos/sos.coco @@ -1,12 +1,11 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Copyright Arne Bachmann +# This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. # Standard modules import codecs, collections, fnmatch, json, logging, mimetypes, os, shutil, sys, time try: from typing import Any, Dict, FrozenSet, IO, Iterator, List, Set, Tuple, Type, Union # only required for mypy -except: pass # typing not available (e.g. Python 2) +except: pass # typing not available (prior Python 3.5) sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) try: import sos.version as version @@ -70,6 +69,7 @@ Usage: {cmd} [] [, ...] When operating in of --add | --rm | --ask Only add new files / only remove vanished files / Ask what to do. Default: add and remove --add-lines | --rm-lines | --ask-lines Only add inserted lines / only remove deleted lines / Ask what to do. Default: add and remove --add-chars | --rm-chars | --ask-chars Only add new characters / only remove vanished characters / Ask what to do. Default: add and remove + --eol Use EOL style from the integrated file instead. Default: EOL style of current file Working with files: changes [][/] List changed paths vs. last or specified revision @@ -536,8 +536,8 @@ def diff(argument:str = "", options:str[] = [], onlys:FrozenSet[str]? = None, ex content:bytes? if pinfo.size == 0: content = b"" # empty file contents else: content = m.restoreFile(None, branch, revision, pinfo); assert content is not None # versioned file - abspath = os.path.join(m.root, path.replace(SLASH, os.sep)) # current file - blocks:List[MergeBlock] = merge(file = content, intoname = abspath, diffOnly = True) # only determine change blocks + abspath:str = os.path.normpath(os.path.join(m.root, path.replace(SLASH, os.sep))) # current file + blocks:List[MergeBlock] = merge(filename = abspath, into = content, diffOnly = True) # only determine change blocks print("DIF %s%s" % (path, " " if len(blocks) == 1 and blocks[0].tipe == MergeBlockType.KEEP else "")) for block in blocks: if block.tipe in [MergeBlockType.INSERT, MergeBlockType.REMOVE]: @@ -548,7 +548,7 @@ def diff(argument:str = "", options:str[] = [], onlys:FrozenSet[str]? = None, ex elif block.tipe == MergeBlockType.REMOVE: for no, line in enumerate(block.lines): print("--- %04d |%s|" % (no + block.line, line)) - elif block.tipe == MergeBlockType.REPLACE: # TODO for MODIFY also show intra-line change ranges + elif block.tipe == MergeBlockType.REPLACE: # TODO for MODIFY also show intra-line change ranges (TODO remove if that code was also removed) for no, line in enumerate(block.replaces.lines): print("- | %04d |%s|" % (no + block.replaces.line, line)) for no, line in enumerate(block.lines): @@ -586,12 +586,12 @@ def status(options:str[] = [], onlys:FrozenSet[str]? = None, excps:FrozenSet[str m.loadBranches() # knows current branch current:int = m.branch strict:bool = '--strict' in options or m.strict - trackingPatterns:FrozenSet[str] = m.getTrackingPatterns() - changes:ChangeSet = m.findChanges(checkContent = strict, considerOnly = onlys if not m.track and not m.picky else conditionalIntersection(onlys, trackingPatterns), dontConsider = excps, progress = '--progress' in options) info(MARKER + " Offline repository status") - info("Content checking %sactivated" % ("" if m.strict else "de")) - info("Data compression %sactivated" % ("" if m.compress else "de")) - info("Repository mode %s" % ("track" if m.track else ("picky" if m.picky else "simple"))) + info("Content checking: %sactivated" % ("" if m.strict else "de")) + info("Data compression: %sactivated" % ("" if m.compress else "de")) + info("Repository mode: %s" % ("track" if m.track else ("picky" if m.picky else "simple"))) + trackingPatterns:FrozenSet[str] = m.getTrackingPatterns() + changes:ChangeSet = m.findChanges(checkContent = strict, considerOnly = onlys if not m.track and not m.picky else conditionalIntersection(onlys, trackingPatterns), dontConsider = excps, progress = True) print("File tree %s" % ("has changes vs. last revision of current branch" if modified(changes) else "is unchanged")) sl:int = max([len(b.name ?? "") for b in m.branches.values()]) for branch in sorted(m.branches.values(), key = (b) -> b.number): @@ -665,9 +665,10 @@ def update(argument:str, options:str[] = [], onlys:FrozenSet[str]? = None, excps In tracking mode, this also updates the set of tracked patterns. User options for merge operation: --add/--rm/--ask --add-lines/--rm-lines/--ask-lines (inside each file), --add-chars/--rm-chars/--ask-chars ''' - mrg:MergeOperation = getAnyOfMap({"--add": MergeOperation.INSERT, "--rm": MergeOperation.REMOVE, "--ask": MergeOperation.ASK}, options, MergeOperation.BOTH) # default operation is replicate remote state + mrg:MergeOperation = getAnyOfMap({"--add": MergeOperation.INSERT, "--rm": MergeOperation.REMOVE, "--ask": MergeOperation.ASK}, options, MergeOperation.BOTH) # default operation is replicate remote state mrgline:MergeOperation = getAnyOfMap({'--add-lines': MergeOperation.INSERT, '--rm-lines': MergeOperation.REMOVE, "--ask-lines": MergeOperation.ASK}, options, mrg) # default operation for modified files is same as for files mrgchar:MergeOperation = getAnyOfMap({'--add-chars': MergeOperation.INSERT, '--rm-chars': MergeOperation.REMOVE, "--ask-chars": MergeOperation.ASK}, options, mrgline) # default operation for modified files is same as for lines + eol:bool = '--eol' in options # use remote eol style m:Metadata = Metadata(os.getcwd()) # TODO same is called inside stop on changes - could return both current and designated branch instead m.loadBranches(); changes:ChangeSet currentBranch:int? = m.branch @@ -692,26 +693,29 @@ def update(argument:str, options:str[] = [], onlys:FrozenSet[str]? = None, excps if mrg.value & MergeOperation.REMOVE.value: os.unlink(m.root + os.sep + path.replace(SLASH, os.sep)) print("DEL " + path if mrg.value & MergeOperation.REMOVE.value else "(D) " + path) # not contained in other branch, but maybe kept for path, pinfo in changes.modifications.items(): - into:str = os.path.join(m.root, path.replace(SLASH, os.sep)) # TODO normalize \.\ - binary = not m.isTextType(path) - op:str = "i" # mine + into:str = os.path.normpath(os.path.join(m.root, path.replace(SLASH, os.sep))) + binary:bool = not m.isTextType(path) + op:str = "m" # merge as default for text files, always asks for binary (TODO unless --theirs or --mine) if mrg == MergeOperation.ASK or binary: # TODO this may ask user even if no interaction was asked for print(("MOD " if not binary else "BIN ") + path) - op = user_input(" Resolve: *M[I]ne (skip), [T]heirs" + (": " if binary else ", [M]erge: ")).strip().lower() # TODO set encoding on stdin + while True: + print(into) # TODO print mtime, size differences? + op = input(" Resolve: *M[I]ne (skip), [T]heirs" + (": " if binary else ", [M]erge: ")).strip().lower() # TODO set encoding on stdin + if op in ("it" if binary else "itm"): break if op == "t": m.readOrCopyVersionedFile(branch, revision, pinfo.nameHash, toFile = into) # blockwise copy of contents print("THR " + path) - elif op == "m" and not binary: + elif op == "m": current:bytes with open(into, "rb") as fd: current = fd.read() # TODO slurps file file:bytes? = m.readOrCopyVersionedFile(branch, revision, pinfo.nameHash) if pinfo.size > 0 else b'' # parse lines if current == file: debug("No difference to versioned file") elif file is not None: # if None, error message was already logged - contents:bytes = merge(file = file, intoname = into, mergeOperation = mrgline, charMergeOperation = mrgchar) + contents:bytes = merge(file = file, into = current, mergeOperation = mrgline, charMergeOperation = mrgchar, eol = eol) if contents != current: with open(path, "wb") as fd: fd.write(contents) # TODO write to temp file first, in case writing fails else: debug("No change") # TODO but update timestamp? - else: + else: # mine or wrong input print("MNE " + path) # nothing to do! same as skip info("Integrated changes from '%s/r%02d' into file tree" % (m.branches[branch].name ?? "b%02d" % branch, revision)) m.branches[currentBranch] = dataCopy(BranchInfo, m.branches[currentBranch], inSync = False, tracked = list(trackingUnion)) @@ -780,8 +784,8 @@ def ls(argument:str? = None, options:str[] = []) -> None: if ig: for wl in m.c.ignoresWhitelist: if fnmatch.fnmatch(file, wl): ignore = None; break # found a white list entry for ignored file, undo ignoring it + matches:List[str] = [] if not ignore: - matches:List[str] = [] for pattern in (p for p in trackingPatterns if os.path.dirname(p).replace(os.sep, SLASH) == relPath): # only patterns matching current folder if fnmatch.fnmatch(file, os.path.basename(pattern)): matches.append(os.path.basename(pattern)) matches.sort(key = (element) -> len(element)) diff --git a/sos/tests.coco b/sos/tests.coco index 0382148..0dadf2c 100644 --- a/sos/tests.coco +++ b/sos/tests.coco @@ -1,10 +1,9 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Copyright Arne Bachmann +# This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. import builtins, enum, json, logging, os, shutil, sys, time, traceback, unittest, uuid -StringIO = (__import__("StringIO" if sys.version_info.major < 3 else "io")).StringIO # enables import via ternary expression -try: from unittest import mock # Py3 +from io import StringIO +try: from unittest import mock # Python 3 except: import mock # installed via pip try: from typing import Any, List, Union # only required for mypy except: pass @@ -14,11 +13,11 @@ try: import configr # optional dependency os.environ["TEST"] = testFolder # needed to mock configr library calls in sos except: configr = None # declare as undefined -import sos +import sos # import of package, not file sos.defaults["defaultbranch"] = "trunk" # because sos.main() is never called def sync() -> None: - if (sys.version_info.major, sys.version_info.minor) >= (3, 3): os.sync() + if sys.version_info[:2] >= (3, 3): os.sync() def determineFilesystemTimeResolution() -> float = @@ -72,7 +71,7 @@ def wrapChannels(func: -> Any) = buf.getvalue() def mockInput(datas:str[], func) -> Any: - with mock.patch("builtins.input" if sys.version_info.major >= 3 else "utility._coconut_raw_input", side_effect = datas): return func() + with mock.patch("builtins.input", side_effect = datas): return func() def setRepoFlag(name:str, value:bool) -> None: with open(sos.metaFolder + os.sep + sos.metaFile, "r") as fd: flags, branches = json.loads(fd.read()) @@ -452,8 +451,8 @@ class Tests(unittest.TestCase): _.createFile(10, theirs) mine = b"a\nc\nd\ne\ng\nf\nx\nh\ny\ny\nj" # missing "b", inserted g, modified g->x, replace x/x -> y/y, removed k _.createFile(11, mine) - _.assertEqual(b"a\nb\nc\nd\ne\nf\ng\nh\nx\nx\nj\nk", sos.merge(filename = "." + os.sep + "file10", intoname = "." + os.sep + "file11", mergeOperation = sos.MergeOperation.BOTH, lineConflictResolution = sos.ConflictResolution.THEIRS)) # completely recreated other file - _.assertEqual(b"a\nb\nc\nd\ne\ng\nf\ng\nx\nh\nx\nx\ny\ny\nj\nk", sos.merge(filename = "." + os.sep + "file10", intoname = "." + os.sep + "file11", mergeOperation = sos.MergeOperation.INSERT, lineConflictResolution = sos.ConflictResolution.MINE)) + _.assertEqual(b"a\nb\nc\nd\ne\nf\ng\nh\nx\nx\nj\nk", sos.merge(filename = "." + os.sep + "file10", intoname = "." + os.sep + "file11", mergeOperation = sos.MergeOperation.BOTH)) # completely recreated other file + _.assertEqual(b'a\nb\nc\nd\ne\ng\nf\ng\nh\ny\ny\nx\nx\nj\nk', sos.merge(filename = "." + os.sep + "file10", intoname = "." + os.sep + "file11", mergeOperation = sos.MergeOperation.INSERT)) def testUpdate2(_): _.createFile("test.txt", "x" * 10) @@ -464,17 +463,16 @@ class Tests(unittest.TestCase): sos.commit("mod") # create b0/r1 sos.switch("trunk", ["--force"]) # should replace contents, force in case some other files were modified (e.g. during working on the code) TODO investigate more with open("test.txt", "rb") as fd: _.assertEqual(b"x" * 10, fd.read()) - sos.update("mod", ["--theirs"]) # integrate changes TODO same with ask -> theirs + sos.update("mod") # integrate changes TODO same with ask -> theirs with open("test.txt", "rb") as fd: _.assertEqual(b"x" * 5 + b"y" * 5, fd.read()) _.createFile("test.txt", "x" * 10) - sos.update("mod", ["--ask-lines"]) # won't ask for lines, as it is recognized as a full line replacement + mockInput(["t"], () -> sos.update("mod", ["--ask-lines"])) + with open("test.txt", "rb") as fd: _.assertEqual(b"x" * 5 + b"y" * 5, fd.read()) + _.createFile("test.txt", "x" * 5 + "z" + "y" * 4) + sos.update("mod") # auto-insert/removes (no intra-line conflict) + _.createFile("test.txt", "x" * 5 + "z" + "y" * 4) + mockInput(["t"], () -> sos.update("mod", ["--ask"])) # same as above with interaction -> use theirs (overwrite current file state) with open("test.txt", "rb") as fd: _.assertEqual(b"x" * 5 + b"y" * 5, fd.read()) - # TODO add code with an intra-line replacement when implemented in getIntraLineMarkers, test method also separately -# _.createFile("test.txt", "x" * 5 + "z" + "y" * 4) -# import pdb; pdb.set_trace() -# sos.update("mod", ["--ask-lines"]) # won't ask for lines, as it is recognized as a full line replacement -# mockInput(["t"], () -> sos.update("mod", ["--ask"])) # same as above with interaction -> use theirs (overwrite current file state) -# with open("test.txt", "rb") as fd: _.assertEqual(b"x" * 5 + b"y" * 5, fd.read()) def testIsTextType(_): m = sos.Metadata(".") @@ -500,29 +498,36 @@ class Tests(unittest.TestCase): _.assertIsNone(sos.eoldet(b"sdf")) def testMerge(_): - ''' Check merge results depending on conflict solution options. ''' + ''' Check merge results depending on user options. ''' + a:bytes = b"a\nb\ncc\nd" + b:bytes = b"a\nb\nee\nd" # replaces cc by ee + _.assertEqual(b"a\nb\ncc\nd", sos.merge(a, b, mergeOperation = sos.MergeOperation.INSERT)) # one-line block replacement using lineMerge + _.assertEqual(b"a\nb\neecc\nd", sos.merge(a, b, mergeOperation = sos.MergeOperation.INSERT, charMergeOperation = sos.MergeOperation.INSERT)) # means insert changes from a into b, but don't replace + _.assertEqual(b"a\nb\n\nd", sos.merge(a, b, mergeOperation = sos.MergeOperation.INSERT, charMergeOperation = sos.MergeOperation.REMOVE)) # means insert changes from a into b, but don't replace + _.assertEqual(b"a\nb\ncc\nd", sos.merge(a, b, mergeOperation = sos.MergeOperation.REMOVE)) # one-line block replacement using lineMerge + _.assertEqual(b"a\nb\n\nd", sos.merge(a, b, mergeOperation = sos.MergeOperation.REMOVE, charMergeOperation = sos.MergeOperation.REMOVE)) + _.assertEqual(a, sos.merge(a, b, mergeOperation = sos.MergeOperation.BOTH)) # keeps any changes in b a = b"a\nb\ncc\nd" - b = b"a\nb\nee\nd" - _.assertEqual(b"a\nb\ncc\nee\nd", sos.merge(a, b, mergeOperation = sos.MergeOperation.INSERT)) # means insert changes from a into b, but don't replace + b = b"a\nb\nee\nf\nd" # replaces cc by block of two lines ee, f + _.assertEqual(b"a\nb\nee\nf\ncc\nd", sos.merge(a, b, mergeOperation = sos.MergeOperation.INSERT)) # multi-line block replacement _.assertEqual(b"a\nb\nd", sos.merge(a, b, mergeOperation = sos.MergeOperation.REMOVE)) - _.assertEqual(b, sos.merge(a, b, mergeOperation = sos.MergeOperation.BOTH)) - # Now test intra-line merging without conflicts - _.assertEqual(b"a\nbc d\ne", sos.merge(b"a\nbc d\ne", b"a\nbcd\ne", mergeOperation = sos.MergeOperation.INSERT, lineConflictResolution = sos.ConflictResolution.MINE)) # because it's a deletion [' a', '- bc d', '? -\n', '+ bcd', ' e'] - _.assertEqual(b"a\nbcd\ne", sos.merge(b"a\nbc d\ne", b"a\nbcd\ne", mergeOperation = sos.MergeOperation.REMOVE, lineConflictResolution = sos.ConflictResolution.THEIRS)) # [' a', '- bc d', '? -\n', '+ bcd', ' e'] - _.assertEqual(b"a\nbc d\ne", sos.merge(b"a\nbcd\ne", b"a\nbc d\ne", mergeOperation = sos.MergeOperation.BOTH)) # nothing to insert, don't change - _.assertEqual(b"a\nbcd\nbc d\ne", sos.merge(b"a\nbcd\ne", b"a\nbc d\ne", mergeOperation = sos.MergeOperation.INSERT)) # don't remove replaced TODO invalid?, should recognize no deletion present! - import pdb; pdb.set_trace() - _.assertEqual(b"a\nbc d\ne", sos.merge(b"a\nbcd\ne", b"a\nbc d\ne", mergeOperation = sos.MergeOperation.BOTH, charMergeOperation = sos.MergeOperation.REMOVE)) # nothing to remove TODO show be a modified but is treated like replace - _.assertEqual(b"a\nbcd\ne", sos.merge(b"a\nbcd\ne", b"a\nbc d\ne", mergeOperation = sos.MergeOperation.INSERT)) # insert space - # Test with change + insert (conflict) - _.assertEqual(b"a\nb fdd d\ne", sos.merge(b"a\nb cd d\ne", b"a\nb fdd d\ne", lineConflictResolution = sos.ConflictResolution.MINE)) - _.assertEqual(b"a\nb cd d\ne", sos.merge(b"a\nb cd d\ne", b"a\nb fdd d\ne", lineConflictResolution = sos.ConflictResolution.THEIRS)) # [' a', '- b cd d', '? ^\n', '+ b fdd d', '? ^^\n', ' e'] - _.assertEqual(b"a\nb fdd d\ne", mockInput(["i"], () -> sos.merge(b"a\nb cd d\ne", b"a\nb fdd d\ne", lineConflictResolution = sos.ConflictResolution.ASK))) # [' a', '- b cd d', '? ^\n', '+ b fdd d', '? ^^\n', ' e'] - _.assertEqual(b"a\nb cd d\ne", mockInput(["t"], () -> sos.merge(b"a\nb cd d\ne", b"a\nb fdd d\ne", lineConflictResolution = sos.ConflictResolution.ASK))) # [' a', '- b cd d', '? ^\n', '+ b fdd d', '? ^^\n', ' e'] - _.assertEqual(b"abbc", sos.merge(b"abbc", b"addc", mergeOperation = sos.MergeOperation.BOTH, lineConflictResolution = sos.ConflictResolution.THEIRS)) - _.assertEqual(b"a\nbb\nc", sos.merge(b"a\nbb\nc", b"a\ndd\nc", mergeOperation = sos.MergeOperation.BOTH, lineConflictResolution = sos.ConflictResolution.THEIRS)) - _.assertIn("Differing EOL-styles", wrapChannels(() -> sos.merge(b"a\nb", b"a\r\nb"))) # expect warning - _.assertIn(b"a\r\nb", sos.merge(b"a\nb", b"a\r\nb")) # in doubt, use "mine" CR-LF + _.assertEqual(a, sos.merge(a, b, mergeOperation = sos.MergeOperation.BOTH)) # keeps any changes in b + # Test with change + insert + _.assertEqual(b"a\nb fdcd d\ne", sos.merge(b"a\nb cd d\ne", b"a\nb fdd d\ne", charMergeOperation = sos.MergeOperation.INSERT)) + _.assertEqual(b"a\nb d d\ne", sos.merge(b"a\nb cd d\ne", b"a\nb fdd d\ne", charMergeOperation = sos.MergeOperation.REMOVE)) + # Test interactive merge + a = b"a\nb\nb\ne" # block-wise replacement + b = b"a\nc\ne" + _.assertEqual(b, mockInput(["i"], () -> sos.merge(a, b, mergeOperation = sos.MergeOperation.ASK))) + _.assertEqual(a, mockInput(["t"], () -> sos.merge(a, b, mergeOperation = sos.MergeOperation.ASK))) + a = b"a\nb\ne" # intra-line merge + _.assertEqual(b, mockInput(["i"], () -> sos.merge(a, b, charMergeOperation = sos.MergeOperation.ASK))) + _.assertEqual(a, mockInput(["t"], () -> sos.merge(a, b, charMergeOperation = sos.MergeOperation.ASK))) + + def testMergeEol(_): + _.assertIn("Differing EOL-styles", wrapChannels(() -> sos.merge(b"a\nb", b"a\r\nb"))) # expects a warning + _.assertIn(b"a\r\nb", sos.merge(b"a\nb", b"a\r\nb")) # when in doubt, use "mine" CR-LF + _.assertIn(b"a\nb", sos.merge(b"a\nb", b"a\r\nb", eol = True)) def testPickyMode(_): ''' Confirm that picky mode reset tracked patterns after commits. ''' @@ -640,8 +645,9 @@ class Tests(unittest.TestCase): _.assertIn("TRK a (a)", sos.safeSplit(wrapChannels(() -> sos.ls("sub")).replace("\r", ""), "\n")) def testLineMerge(_): - _.assertEqual("a bd", sos.lineMerge("xabc", "a bd")) - _.assertEqual("xa bcd", sos.lineMerge("xabc", "a bd", mergeOperation = sos.MergeOperation.INSERT)) + _.assertEqual("xabc", sos.lineMerge("xabc", "a bd")) + _.assertEqual("xabxxc", sos.lineMerge("xabxxc", "a bd")) + _.assertEqual("xa bdc", sos.lineMerge("xabc", "a bd", mergeOperation = sos.MergeOperation.INSERT)) _.assertEqual("ab", sos.lineMerge("xabc", "a bd", mergeOperation = sos.MergeOperation.REMOVE)) def testCompression(_): # TODO test output ratio/advantage, also depending on compress flag set or not @@ -729,7 +735,7 @@ class Tests(unittest.TestCase): try: sos.config("set", ["ignoresWhitelist", "ign1;ign2"]) # define a list of ignore patterns except SystemExit as E: _.assertEqual(0, E.code) out = wrapChannels(() -> sos.config("show")).replace("\r", "") - _.assertIn(" ignores: ['ign1', 'ign2']", out) + _.assertIn(" ignores [user] ['ign1', 'ign2']", out) out = sos.safeSplit(wrapChannels(() -> sos.ls()).replace("\r", ""), "\n") _.assertInAny('... file1', out) _.assertInAny('... ign1', out) @@ -817,7 +823,7 @@ class Tests(unittest.TestCase): _.assertAllNotIn(["MOD ./file1", "DIF ./file1", "MOD ./file2"], wrapChannels(() -> sos.diff(onlys = f{"./file2"}))) def testDiff(_): - try: sos.config("set", ["texttype", "file1"]) + try: sos.config("set", ["texttype", "file1"]) # manually mark this file as "textual" except SystemExit as E: _.assertEqual(0, E.code) sos.offline(options = ["--strict"]) _.createFile(1) @@ -827,7 +833,10 @@ class Tests(unittest.TestCase): _.createFile(2, "12343") sos.commit() _.createFile(1, "foobar") - _.assertAllIn(["MOD ./file2", "DIF ./file1", "- | 0000 |xxxxxxxxxx|", "+ | 0000 |foobar|"], wrapChannels(() -> sos.diff("/-2"))) # vs. second last + _.createFile(3) + out = wrapChannels(() -> sos.diff("/-2")) # compare with r1 (second counting from last which is r2) + _.assertIn("ADD ./file3", out) + _.assertAllIn(["MOD ./file2", "DIF ./file1", "- | 0001 |xxxxxxxxxx|", "+ | 0000 |foobar|"], out) _.assertAllNotIn(["MOD ./file1", "DIF ./file1"], wrapChannels(() -> sos.diff("/-2", onlys = f{"./file2"}))) def testReorderRenameActions(_): diff --git a/sos/utility.coco b/sos/utility.coco index 6b41160..09f7ebb 100644 --- a/sos/utility.coco +++ b/sos/utility.coco @@ -1,6 +1,5 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Copyright Arne Bachmann +# This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. # Utiliy functions import bz2, codecs, difflib, hashlib, logging, os, re, sys, time; START_TIME = time.time() @@ -9,7 +8,7 @@ except: raise Exception("Please install SOS via 'pip install -U sos-vcs[backport try: from typing import Any, Callable, Dict, FrozenSet, IO, List, Sequence, Set, Tuple, Type, TypeVar, Union # only required for mypy Number = Union[int,float] -except: pass # typing not available (e.g. Python 2) +except: pass # typing not available (prior Python 3.5) try: import wcwidth except: pass # optional dependency @@ -113,10 +112,8 @@ def eoldet(file:bytes) -> bytes? = if cr > lf: return b"\r" # older 8-bit machines None # no new line contained, cannot determine -def user_input(msg:str) -> str = input(msg) # referenceso __builtin__.raw_input on Python 2 - try: Splittable = TypeVar("Splittable", str, bytes) -except: pass # Python 2 +except: pass def safeSplit(s:Splittable, d:Splittable? = None) -> Splittable[]: return s.split(d ?? ("\n" if isinstance(s, str) else b"\n")) if len(s) > 0 else [] def ajoin(sep:str, seq:str[], nl = "") -> str = (sep + (nl + sep).join(seq)) if seq else "" @@ -196,14 +193,14 @@ def diffPathSets(last:Dict[str,PathInfo], diff:Dict[str,PathInfo]) -> ChangeSet changes try: DataType = TypeVar("DataType", ChangeSet, MergeBlock, BranchInfo) -except: pass # Python 2 +except: pass def dataCopy(_tipe:Type[DataType], _old:DataType, *_args, **_kwargs) -> DataType: r = _old._asdict(); r.update(**_kwargs); return makedata(_tipe, *(list(_args) + [r[field] for field in _old._fields])) def user_block_input(output:List[str]) -> None: - sep:str = user_input("Enter end-of-text marker (default: : "); line:str = sep + sep:str = input("Enter end-of-text marker (default: : "); line:str = sep while True: - line = user_input("> ") + line = input("> ") if line == sep: break output.append(line) @@ -212,11 +209,15 @@ def merge( filename:str? = None, intoname:str? = None, mergeOperation:MergeOperation = MergeOperation.BOTH, charMergeOperation:MergeOperation = MergeOperation.BOTH, - diffOnly:bool = False + diffOnly:bool = False, + eol:bool = False ) -> Union[bytes,List[MergeBlock]] = - ''' Merges other binary text contents 'file' (or reads file 'filename') - into current text contents 'into' (or reads file 'intoname'), returning merged result. + ''' Merges other binary text contents 'file' (or reads file 'filename') into current text contents 'into' (or reads file 'intoname'), returning merged result. + For update, the other version is assumed to be the "new/added" one, while for diff, the current changes are the ones "added". + However, change direction markers are insert ("+") for elements only in into, and remove ("-") for elements only in other file (just like the diff marks +/-) diffOnly returns detected change blocks only, no text merging + eol flag will use the other file's EOL marks + in case of replace block and INSERT strategy, the change will be added **behind** the original ''' encoding:str; othr:str[]; othreol:bytes?; curr:str[]; curreol:bytes? try: # load files line-wise and normalize line endings (keep the one of the current file) TODO document @@ -235,19 +236,13 @@ def merge( if len(tmp) > 0: blocks.append(MergeBlock(MergeBlockType.KEEP, [line for line in tmp], line = no - len(tmp))) # avoid adding empty keep block elif last == "-": # may be a pure deletion or part of a replacement (with next block being "+") blocks.append(MergeBlock(MergeBlockType.REMOVE, [line for line in tmp], line = no - len(tmp))) + if len(blocks) >= 2 and blocks[-2].tipe == MergeBlockType.INSERT: + blocks[-2] = MergeBlock(MergeBlockType.REPLACE, blocks[-1].lines, line = no - len(tmp) - 1, replaces = blocks[-2]) # remember replaced stuff with reference to other merge block TODO why -1 necessary? + blocks.pop() elif last == "+": # may be insertion or replacement (with previous - block) blocks.append(MergeBlock(MergeBlockType.INSERT, [line for line in tmp], line = no - len(tmp))) # first, assume simple insertion, then check for replacement - if len(blocks) >= 2: # and len(blocks[-1].lines) == len(blocks[-2].lines): # requires previous block and same number of lines TODO allow multiple intra-line merge for same-length blocks - # if more than one lines: could be swap, replacement, or modification (for some lines) - handle as replacement with conflict resolution option - # if one line: can be handled via lineMerge and character modifications -# if len(blocks[-1].lines) >= 2 or (blocks[-1].changes is None and blocks[-2].changes is None): # > 1 lines and no intra-line comment -# if len(blocks[-1].lines) == 1: -# intraMerge = lineMerge(blocks[-1].lines[0], blocks[-2].lines[0], charMergeOperation = charMergeOperation, charConflictResolution = charConflictResolution, diffOnly = diffOnly) -# else: -# intraMerge = blocks[-1].lines - blocks[-2] = MergeBlock(MergeBlockType.REPLACE, blocks[-1].lines, line = no - len(tmp) - 1, replaces = blocks[-2]) # remember replaced stuff with reference to other merge block TODO why -1 necessary? -# else: # either just one line in block or has intra-line changes: may have intra-line modifications (character level of merge) TODO use merge recursively? -# blocks[-2] = MergeBlock(MergeBlockType.MODIFY, blocks[-1].lines, line = no - len(tmp), replaces = blocks[-2], changes = blocks[-1].changes) + if len(blocks) >= 2 and blocks[-2].tipe == MergeBlockType.REMOVE: # and len(blocks[-1].lines) == len(blocks[-2].lines): # requires previous block and same number of lines TODO allow multiple intra-line merge for same-length blocks + blocks[-2] = MergeBlock(MergeBlockType.REPLACE, blocks[-2].lines, line = no - len(tmp) - 1, replaces = blocks[-1]) # remember replaced stuff with reference to other merge block TODO why -1 necessary? blocks.pop() # remove TOS due to merging two blocks into replace or modify elif last == "?": # marker for intra-line change comment -> add to block info ilm:Range = getIntraLineMarkers(tmp[0]) # TODO still true? "? " line includes a trailing \n for some reason @@ -268,37 +263,56 @@ def merge( output.extend(block.lines) elif block.tipe == MergeBlockType.REPLACE: # complete block replacement if len(block.lines) == len(block.replaces.lines) == 1: # one-liner - output.append(lineMerge(block.replaces.lines[0], block.lines[0], mergeOperation = charMergeOperation)) + output.append(lineMerge(block.lines[0], block.replaces.lines[0], mergeOperation = charMergeOperation)) elif mergeOperation == MergeOperation.ASK: # more than one line: needs user input - print(ajoin("- ", block.replaces.lines)) # TODO check +/- in update mode, could be swapped - print(ajoin("+ ", block.lines)) + print(ajoin("- ", block.replaces.lines, nl = "\n")) # TODO check +/- in update mode, could be swapped + print(ajoin("+ ", block.lines, nl = "\n")) while True: - op:str = user_input(" Line replacement: *M[I]ne (+), [T]heirs (-), [B]oth, [U]ser input: ").strip().lower() - if op in "tb": output.extend(block.replaces.lines); break - if op in "ib": output.extend(block.lines); break + op:str = input(" Line replacement: *M[I]ne (+), [T]heirs (-), [B]oth, [U]ser input: ").strip().lower() + if op in "tb": output.extend(block.lines); break + if op in "ib": output.extend(block.replaces.lines); break if op == "m": user_block_input(output); break else: # more than one line and not ask if mergeOperation == MergeOperation.REMOVE: pass elif mergeOperation == MergeOperation.BOTH: output.extend(block.lines) elif mergeOperation == MergeOperation.INSERT: output.extend(list(block.replaces.lines) + list(block.lines)) # TODO optionally allow insertion BEFORE or AFTER original (order of these both lines) # debug("Merge output: " + "; ".join(output)) - nl:bytes = curreol ?? othreol ?? b"\n" + nl:bytes = othreol if eol else (curreol ?? othreol ?? b"\n") nl.join([line.encode(encoding) for line in output]) # returning bytes # TODO handle check for more/less lines in found -/+ blocks to find common section and splitting prefix/suffix out def lineMerge(othr:str, into:str, mergeOperation:MergeOperation = MergeOperation.BOTH, diffOnly:bool = False) -> Union[str,List[MergeBlock]]: - ''' Finds character differences. ''' + ''' Merges string 'othr' into current string 'into'. + change direction mark is insert for elements only in into, and remove for elements only in file (according to diff marks +/-) + ''' out:List[str] = list(difflib.Differ().compare(othr, into)) blocks:List[MergeBlock] = [] for i, line in enumerate(out): - if line[0] == "+": - if i > 0 and blocks[i - 1].tipe == MergeBlockType.REMOVE: - blocks.append(MergeBlock(MergeBlockType.REPLACE, [line[2]], i, replaces = blocks[i - 1])) - else: - blocks.append(MergeBlock(MergeBlockType.INSERT, [line[2]], i)) - elif line[0] == "-": blocks.append(MergeBlock(MergeBlockType.REMOVE, [line[2]], i)) - elif line[0] == " ": blocks.append(MergeBlock(MergeBlockType.KEEP, [line[2]], i)) + if line[0] == "+": + if i + 1 < len(out) and out[i + 1][0] == "+": # block will continue + if i > 0 and blocks[-1].tipe == MergeBlockType.INSERT: # middle of + block + blocks[-1].lines.append(line[2]) # add one more character to the accumulating list + else: # first + in block + blocks.append(MergeBlock(MergeBlockType.INSERT, [line[2]], i)) + else: # last line of + block + if i > 0 and blocks[-1].tipe == MergeBlockType.INSERT: # end of a block + blocks[-1].lines.append(line[2]) + else: # single line + blocks.append(MergeBlock(MergeBlockType.INSERT, [line[2]], i)) + if i >= 1 and blocks[-2].tipe == MergeBlockType.REMOVE: # previous - and now last in + block creates a replacement block + blocks[-2] = MergeBlock(MergeBlockType.REPLACE, blocks[-2].lines, i, replaces = blocks[-1]); blocks.pop() + elif line[0] == "-": + if i > 0 and blocks[-1].tipe == MergeBlockType.REMOVE: # part of - block + blocks[-1].lines.append(line[2]) + else: # first in block + blocks.append(MergeBlock(MergeBlockType.REMOVE, [line[2]], i)) + elif line[0] == " ": + if i > 0 and blocks[-1].tipe == MergeBlockType.KEEP: # part of block + blocks[-1].lines.append(line[2]) + else: # first in block + blocks.append(MergeBlock(MergeBlockType.KEEP, [line[2]], i)) else: raise Exception("Cannot parse diff line %r" % line) + blocks[:] = [dataCopy(MergeBlock, block, lines = ["".join(block.lines)], replaces = dataCopy(MergeBlock, block.replaces, lines = ["".join(block.replaces.lines)]) if block.replaces else None) for block in blocks] if diffOnly: return blocks out[:] = [] for i, block in enumerate(blocks): @@ -310,20 +324,22 @@ def lineMerge(othr:str, into:str, mergeOperation:MergeOperation = MergeOperation print("+ " + (" " * i) + block.lines[0]) print(ajoin("+ ", into)) while True: - op:str = user_input(" Character replacement: *M[I]ne (+), [T]heirs (-), [B]oth, [U]ser input: ").strip().lower() - if op in "tb": output.extend(block.replaces.lines); break - if op in "ib": output.extend(block.lines); break - if op == "m": user_block_input(output); break - elif block.tipe == MergeBlockType.INSERT: - if mergeOperation.value & MergeType.INSERT: out.extend(block.replaces.lines) - if not (mergeOperation.value & MergeType.REMOVE): out.extend(block.lines) - elif block.tipe == MergeBlockType.REMOVE: - if mergeOperation.value & MergeType.INSERT: out.extend(block.lines) + op:str = input(" Character replacement: *M[I]ne (+), [T]heirs (-), [B]oth, [U]ser input: ").strip().lower() + if op in "tb": out.extend(block.lines); break + if op in "ib": out.extend(block.replaces.lines); break + if op == "m": user_block_input(out); break + else: # non-interactive + if mergeOperation == MergeOperation.REMOVE: pass + elif mergeOperation == MergeOperation.BOTH: out.extend(block.lines) + elif mergeOperation == MergeOperation.INSERT: out.extend(list(block.replaces.lines) + list(block.lines)) + elif block.tipe == MergeBlockType.INSERT and not (mergeOperation.value & MergeOperation.REMOVE.value): out.extend(block.lines) + elif block.tipe == MergeBlockType.REMOVE and mergeOperation.value & MergeOperation.INSERT.value: out.extend(block.lines) + # TODO ask for insert or remove as well return "".join(out) def getIntraLineMarkers(line:str) -> Range = ''' Return (type, [affected indices]) of "? "-line diff markers ("? " prefix must be removed). difflib never returns mixed markers per line. ''' - if "^" in line: return Range(MergeBlockType.MODIFY, [i for i, c in enumerate(line) if c == "^"]) + if "^" in line: return Range(MergeBlockType.REPLACE, [i for i, c in enumerate(line) if c == "^"]) # TODO wrong, needs removal anyway if "+" in line: return Range(MergeBlockType.INSERT, [i for i, c in enumerate(line) if c == "+"]) if "-" in line: return Range(MergeBlockType.REMOVE, [i for i, c in enumerate(line) if c == "-"]) Range(MergeBlockType.KEEP, [])