From 576c32c720b65751c58d0978096330b39f6269b9 Mon Sep 17 00:00:00 2001 From: ArneBachmann Date: Mon, 25 Dec 2017 23:21:45 +0100 Subject: [PATCH] Fixes #8 --- README.md | 23 ++++++++++++ setup.py | 6 ++-- sos/sos.coco | 42 +++++++++++++++++++--- sos/tests.coco | 24 +++++++++++++ sos/utility.coco | 92 +++++++++++++++++++++++++++++++++++++++++++----- 5 files changed, 170 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 73061d5..1c590a1 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,15 @@ SOS supports three different file handling models that you may use to your likin - **Tracking mode**: Only files that match certain file name tracking patterns are respected during `commit`, `update` and `branch` (just like in SVN, gitless, and Fossil), requiring users to specifically add or remove files per branch. Drawback: Need to declare files to track for every offline repository - **Picky mode**: Each operation needs the explicit declaration of file name patterns for versioning (like Git does). Drawback: Need to stage files for every single commit +### Unique features of SOS ### +- Initializes repositories by default with the *simple mode*, which makes effortless versioning a piece of cake +- In the optional tracking mode, files are tracked based on glob patterns instead of pure filenames or paths (in a manner comparable to how SVN ignores files) +- In-place command line replacement for traditional VCS that transparently pipes commands to them +- Straightforward and simplified semantics for common VCS operations (`branch`, `commit`, integrate changes) + +### Limitations ### +- Designed for use by single user, network synchronization is a non-goal. Don't attempt to use SOS in a shared location, concurrent access to the repository may corrupt your data, as there is currently no locking in place (could be augmented, but it's a non-goal too) +- Has a small user base as of now, therefore no reliable reports of compatibility and operational capability except for the automatic unit tests run on Travis CI and AppVeyor ### Compatibility ### - SOS runs on any Python 3 distribution, except PyPy (TODO). Support for Python 2 is only partial, as the test suite doesn't run through entirely yet, although SOS's programming language Coconut is generally able to transpile to valid Python 2 source code @@ -40,6 +49,19 @@ SOS supports three different file handling models that you may use to your likin - File name encoding and console encoding: Full roundtrip support (on Windows) started only with Python 3.6.4 and has not been tested nor confirmed yet for SOS +## Latest changes ## +- Version 1.1 published on 2017-12-26: + - [Bug 90](https://github.com/ArneBachmann/sos/issues/90) Removed directories weren't picked up + - [Bug 93](https://github.com/ArneBachmann/sos/issues/93) Picky mode lists any file as added + - [Enhancement 63](https://github.com/ArneBachmann/sos/issues/63) Show more change details in `log` and `status` + - [Enhancement 86](https://github.com/ArneBachmann/sos/issues/86) Renamed command for branch removal to `destroy` + - [Feature 61](https://github.com/ArneBachmann/sos/issues/61) Added option to only consider or exclude certain file patterns for relevant operations using `--only` and `--except` + - [Feature 8](https://github.com/ArneBachmann/sos/issues/8) Added functionality to rename tracking patterns and move files accordingly + - [Feature 80](https://github.com/ArneBachmann/sos/issues/80) Added functionality to use tags + - [QA 79](https://github.com/ArneBachmann/sos/issues/79) Added AppVeyor automated testing + - [QA 94](https://github.com/ArneBachmann/sos/issues/94) More test coverage + + ## Comparison with Traditional VCS ## When completing SOS 1.0 I incidentally discovered an interesting article by ... that discusses central weaknesses in the design of VCSs, with a focus on Git. Many of these arguments I have intuitively felt to be true as well and were the reason for the development of SOS: mainly the reduction of barriers between the developer's typical workflow and the VCS, which is most often used as a structured tool for "type and save in increments", while advanced features of Git are just very difficult to remember and get done right. @@ -54,6 +76,7 @@ Here is a comparison between SOS and VCS's commands: - The first revision (created during execution of `sos offline` or `sos branch`) always has the number `0` - Each `sos commit` increments the revision number by one; revisions are referenced by this numeric index only - `delete` destroys and removes a branch. It's a command, not an option flag as in `git branch -d ` +- `move` renames a file tracking pattern and all matching files accordingly; only useful in tracking or picky mode. It supports reordering of literal substrings, but no reordering of glob markers, and no adjacent glob markers. Use `--soft` to avoid files actually being renamed in the file tree - `switch` works like `checkout` in Git for a revision of another branch (or of the current), or `update` to latest or a specific revision in SVN. Please note that switching to a different revision will in no way fix or remember that revision. The file tree will always be compared to the branch's latest commit for change detection - `update` works a bit like `pull` in Git or `update` in SVN and replays the given branch's and/or revision's changes into the file tree. There are plenty of options to configure what changes are actually integrated. This command will not change the current branch like `switch` does diff --git a/setup.py b/setup.py index b0aaa63..7c31067 100644 --- a/setup.py +++ b/setup.py @@ -5,12 +5,10 @@ readmeFile = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'README.md') if 'build' in sys.argv: - print("Transpiling Coconut for packaging...") + print("Transpiling Coconut files to Python...") cmd = "-develop" if 0 == subprocess.Popen("coconut-develop --help", shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE, bufsize = 10000000).wait() and os.getenv("NODEV", "false").strip().lower() != "true" else "" - assert 0 == os.system("coconut%s -p -l -t 3 sos%sutility.coco" % (cmd, os.sep)) - assert 0 == os.system("coconut%s -p -l -t 3 sos%ssos.coco" % (cmd, os.sep)) # TODO remove target once Python 2 problems have been fixed - assert 0 == os.system("coconut%s -p -l -t 3 sos%stests.coco" % (cmd, os.sep)) + assert 0 == os.system("coconut%s -p -l -t 3 sos" % cmd) # TODO remove target once Python 2 problems have been fixed if os.path.exists(".git"): print("Preparing documentation for PyPI by converting from Markdown to reStructuredText via pandoc") diff --git a/sos/sos.coco b/sos/sos.coco index 9527962..95ee216 100644 --- a/sos/sos.coco +++ b/sos/sos.coco @@ -61,7 +61,8 @@ Usage: {cmd} [] [, ...] When operating in of changes [][/] List changed paths vs. last or specified revision diff [][/] List changes vs. last or specified revision add [] Add a tracking pattern to current branch (path/filename or glob pattern) - mv [] [] Rename, move or move and rename tracked files according to glob patterns + mv [] [] Rename, move, or move and rename tracked files according to glob patterns + --soft Don't move or rename files, only the tracking pattern rm [] Remove a tracking pattern. Only useful after "offline --track" or "offline --picky" ls List file tree and mark changes and tracking status @@ -91,8 +92,8 @@ Usage: {cmd} [] [, ...] When operating in of --only Restrict operation to specified pattern(s). Available for "changes", "commit", "diff", "switch", and "update" --except Avoid operation for specified pattern(s). Available for "changes", "commit", "diff", "switch", and "update" --{cmd} When executing {CMD} not being offline, pass arguments to {CMD} instead (e.g. {cmd} --{cmd} config set key value.) - --log Enable internals logger - --verbose Enable debugging output""".format(appname = APPNAME, cmd = "sos", CMD = "SOS")) + --log Enable logging details + --verbose Enable verbose output""".format(appname = APPNAME, cmd = "sos", CMD = "SOS")) # Main data class #@runtime_validation @@ -780,7 +781,40 @@ def config(argument:str, options:List[str] = []) -> None: def move(relPath:str, pattern:str, newRelPath:str, newPattern:str, options:List[str] = []) -> None: ''' Path differs: Move files, create folder if not existing. Pattern differs: Attempt to rename file, unless exists in target or not unique. ''' - pass + force:bool = '--force' in options + soft:bool = '--soft' in options + if not os.path.exists(relPath): Exit("Source folder doesn't exist") + matching:str[] = fnmatch.filter(os.listdir(relPath.replace(SLASH, os.sep)), os.path.basename(pattern)) # find matching files in source + if not matching and not force: Exit("No files match the specified file or glob pattern. Use --force to proceed anyway") + m:Metadata = Metadata(os.getcwd()) + m.loadBranches() # knows current branch + if not m.track and not m.picky: Exit("Repository is in simple mode. Simply use basic file operations to modify files, then execute 'sos commit' to version the changes") + if pattern not in m.branches[m.branch].tracked: + for tracked in (t for t in m.branches[m.branch].tracked if os.path.dirname(t) == relPath): # for all patterns of the same source folder + alternative:str[] = fnmatch.filter(matching, os.path.basename(tracked)) # find if it matches any of the files in the source folder, too + if alternative: info(" '%s' matches %d files" % (tracked, len(alternative))) + Exit("File or glob pattern '%s' is not tracked on current branch. 'sos move' only works on tracked patterns" % pattern) + basePattern:str = os.path.basename(pattern) # pure glob without folder + newBasePattern:str = os.path.basename(newPattern) + if basePattern.count("*") != newBasePattern.count("*")\ + or (basePattern.count("?") - basePattern.count("[?]")) != (newBasePattern.count("?") - newBasePattern.count("[?]"))\ + or (basePattern.count("[") - basePattern.count("\\[")) != (newBasePattern.count("[") - newBasePattern.count("\\["))\ + or (basePattern.count("]") - basePattern.count("\\]")) != (newBasePattern.count("]") - newBasePattern.count("\\]")): + Exit("Glob markers don't match, cannot move/rename tracked matching files") + oldTokens:GlobBlock[]; newToken:GlobBlock[] + oldTokens, newTokens = tokenizeGlobPatterns(os.path.basename(pattern), os.path.basename(newPattern)) + matches:Tuple[str,str][] = convertGlobFiles(matching, oldTokens, newTokens) # computes list of source - target filename pairs + matches = reorderRenameActions(matches, exitOnConflict = not soft) # attempts to find conflict-free renaming order, or exits + if os.path.exists(newRelPath): + exists:str[] = [filename[1] for filename in matches if os.path.exists(os.path.join(newRelPath, filename[1]).replace(SLASH, os.sep))] + if exists and not (force or soft): Exit("%s files would write over existing files in %s cases. Use --force to execute it anyway" % ("Moving" if relPath != newRelPath else "Renaming", "all" if len(exists) == len(matches) else "some")) + else: os.makedirs(os.path.absdir(newRelPath.replace(SLASH, os.sep))) + if not soft: # perform actual renaming + for (source, target) in matches: + try: shutil.move(os.path.abspath(os.path.join(relPath, source).replace(SLASH, os.sep)), os.path.abspath(os.path.join(newRelPath, target).replace(SLASH, os.sep))) + except Exception as E: error("Cannot move/rename file '%s' to '%s'" % (source, os.path.join(newRelPath, target))) # one error can lead to another in case of delicate renaming order + m.branches[m.branch].tracked[m.branches[m.branch].tracked.index(pattern)] = newPattern + m.saveBranches() def parse(root:str, cwd:str): ''' Main operation. Main has already chdir into VCS root folder, cwd is original working directory for add, rm. ''' diff --git a/sos/tests.coco b/sos/tests.coco index 23c7b0b..ae18499 100644 --- a/sos/tests.coco +++ b/sos/tests.coco @@ -195,6 +195,30 @@ class Tests(unittest.TestCase): try: sos.commit("nothing"); _.fail() # should not commit anything, as the file in base folder doesn't match the tracked pattern except: pass + def testTokenizeGlobPattern(_): + _.assertEqual([], sos.tokenizeGlobPattern("")) + _.assertEqual([sos.GlobBlock(False, "*", 0)], sos.tokenizeGlobPattern("*")) + _.assertEqual([sos.GlobBlock(False, "*", 0), sos.GlobBlock(False, "???", 1)], sos.tokenizeGlobPattern("*???")) + _.assertEqual([sos.GlobBlock(True, "x", 0), sos.GlobBlock(False, "*", 1), sos.GlobBlock(True, "x", 2)], sos.tokenizeGlobPattern("x*x")) + _.assertEqual([sos.GlobBlock(True, "x", 0), sos.GlobBlock(False, "*", 1), sos.GlobBlock(False, "??", 2), sos.GlobBlock(False, "*", 4), sos.GlobBlock(True, "x", 5)], sos.tokenizeGlobPattern("x*??*x")) + _.assertEqual([sos.GlobBlock(False, "?", 0), sos.GlobBlock(True, "abc", 1), sos.GlobBlock(False, "*", 4)], sos.tokenizeGlobPattern("?abc*")) + + def testTokenizeGlobPatterns(_): + try: sos.tokenizeGlobPatterns("x*x", "x*"); _.fail() # because number of literal strings differs + except: pass + try: sos.tokenizeGlobPatterns("x*", "x?"); _.fail() # because glob patterns differ + except: pass + try: sos.tokenizeGlobPatterns("x*", "?x"); _.fail() # glob patterns differ, regardless of position + except: pass + sos.tokenizeGlobPatterns("x*", "*x") # succeeds, because glob patterns match (differ only in position) + sos.tokenizeGlobPatterns("*xb?c", "*x?bc") # succeeds, because glob patterns match (differ only in position) + try: sos.tokenizeGlobPatterns("a???b*", "ab???*"); _.fail() # succeeds, because glob patterns match (differ only in position) + except: pass + + def testConvertGlobFiles(_): + _.assertEqual(["xxayb", "aacb"], [r[1] for r in sos.convertGlobFiles(["axxby", "aabc"], *sos.tokenizeGlobPatterns("a*b?", "*a?b"))]) + _.assertEqual(["1qq2ww3", "1abcbx2xbabc3"], [r[1] for r in sos.convertGlobFiles(["qqxbww", "abcbxxbxbabc"], *sos.tokenizeGlobPatterns("*xb*", "1*2*3"))]) + def testComputeSequentialPathSet(_): os.makedirs(branchFolder(0, 0)) os.makedirs(branchFolder(0, 1)) diff --git a/sos/utility.coco b/sos/utility.coco index c623a67..c9de99a 100644 --- a/sos/utility.coco +++ b/sos/utility.coco @@ -1,7 +1,7 @@ # Utiliy functions -import bz2, codecs, difflib, hashlib, logging, os, sys, time +import bz2, codecs, difflib, hashlib, logging, os, re, sys, time try: - from typing import Any, Callable, Dict, IO, List, Tuple, Type, TypeVar, Union # only required for mypy + from typing import Any, Callable, Dict, FrozenSet, IO, List, Sequence, Tuple, Type, TypeVar, Union # only required for mypy Number = Union[int,float] except: pass # typing not available (e.g. Python 2) try: import wcwidth @@ -60,6 +60,8 @@ data PathInfo(nameHash:str, size:int?, mtime:int, hash:str?) # size == None mea data ChangeSet(additions:Dict[str,PathInfo], deletions:Dict[str,PathInfo], modifications:Dict[str,PathInfo]) # avoid default assignment of {} as it leads to runtime errors (contains data on init for unknown reason) data Range(tipe:int, indexes:int[]) # MergeBlockType[1,2,4], line number, length data MergeBlock(tipe:int, lines:str[], line:int, replaces:MergeBlock? = None, changes:Range? = None) +data GlobBlock(isLiteral:bool, content:str, index:int) # for file pattern rename/move matching +data GlobBlock2(isLiteral:bool, content:str, matches:str) # matching file pattern and input filename for translation # Enums @@ -91,6 +93,8 @@ def wcswidth(string:str) -> int = except: return len(string) l +def conditionalIntersection(a:FrozenSet[str]?, b:FrozenSet[str]) -> FrozenSet[str] = a & b if a else b + def openIt(file:str, mode:str, compress:bool = False) -> IO = bz2.BZ2File(file, mode) if compress else open(file, mode + "b") # Abstraction for opening both compressed and plain files def eoldet(file:bytes) -> bytes? = @@ -122,6 +126,8 @@ def hashStr(datas:str) -> str = hashlib.sha256(datas.encode(UTF8)).hexdigest() def modified(changes:ChangeSet, onlyBinary:bool = False) -> bool = len(changes.additions) > 0 or len(changes.deletions) > 0 or len(changes.modifications) > 0 +def listindex(lizt:Sequence[Any], what:Any, index:int = 0) -> int = lizt[index:].index(what) + index + def getTermWidth() -> int = try: import termwidth except: return 80 @@ -312,25 +318,93 @@ def findSosVcsBase() -> Tuple[str?,str?,str?] = if choice: return (sos, path, choice) (None, vcs[0], vcs[1]) +def tokenizeGlobPattern(pattern:str) -> List[GlobBlock] = + index:int = 0 + out:List[GlobBlock] = [] # literal = True, first index + while index < len(pattern): + if pattern[index:index + 3] in ("[?]", "[*]", "[[]", "[]]"): out.append(GlobBlock(False, pattern[index:index + 3], index)); continue + if pattern[index] in "*?": + count:int = 1 + while index + count < len(pattern) and pattern[index] == "?" and pattern[index + count] == "?": count += 1 + out.append(GlobBlock(False, pattern[index:index + count], index)); index += count; continue + if pattern[index:index + 2] == "[!": out.append(GlobBlock(False, pattern[index:pattern.index("]", index + 2) + 1], index)); index += len(out[-1][1]); continue + count = 1 + while index + count < len(pattern) and pattern[index + count] not in "*?[": count += 1 + out.append(GlobBlock(True, pattern[index:index + count], index)); index += count + # TODO [abc] + out + +def tokenizeGlobPatterns(oldPattern:str, newPattern:str) -> Tuple[GlobBlock[], GlobBlock[]] = + ot:List[GlobBlock] = tokenizeGlobPattern(oldPattern) + nt:List[GlobBlock] = tokenizeGlobPattern(newPattern) +# if len(ot) != len(nt): Exit("Source and target patterns can't be translated due to differing number of parsed glob markers and literal strings") + if len([o for o in ot if not o.isLiteral]) != len([n for n in nt if not n.isLiteral]): Exit("Source and target file patterns contain differing number of glob markers and can't be translated") + if any(O.content != N.content for O, N in zip([o for o in ot if not o.isLiteral], [n for n in nt if not n.isLiteral])): Exit("Source and target file patterns differ in semantics") + (ot, nt) + +def convertGlobFiles(filenames:str[], oldPattern:GlobBlock[], newPattern:GlobBlock[]) -> Tuple[str,str][] = + ''' Converts given filename according to specified file patterns. No support for adjacent glob markers currently. ''' # TODO use list of filenames instead + pairs:Tuple[str,str][] = [] + for filename in filenames: + literals:List[GlobBlock] = [l for l in oldPattern if l.isLiteral] # source literals + nextliteral:int = 0 + parsedOld:List[GlobBlock2] = [] + index:int = 0 + for part in oldPattern: # match everything in the old filename + if part.isLiteral: parsedOld.append(GlobBlock2(True, part.content, part.content)); index += len(part.content); nextliteral += 1 + elif part.content.startswith("?"): parsedOld.append(GlobBlock2(False, part.content, filename[index:index + len(part.content)])); index += len(part.content) + elif part.content.startswith("["): parsedOld.append(GlobBlock2(False, part.content, filename[index])); index += 1 + elif part.content == "*": + if nextliteral >= len(literals): parsedOld.append(GlobBlock2(False, part.content, filename[index:])); break + nxt:int = filename.index(literals[nextliteral].content, index) # also matches empty string + parsedOld.append(GlobBlock2(False, part.content, filename[index:nxt])); index = nxt + else: Exit("Invalid tracking pattern specified for move/rename") + globs:List[GlobBlock2] = [g for g in parsedOld if not g.isLiteral] + literals = [l for l in newPattern if l.isLiteral] # target literals + nextliteral = 0; nextglob:int = 0 + outname:str = "" # TODO join a list instead? + for part in newPattern: # generate new filename + if part.isLiteral: outname += literals[nextliteral].content; nextliteral += 1 + else: outname += globs[nextglob].matches; nextglob += 1 + pairs.append((filename, outname)) + pairs + +def reorderRenameActions(actions:Tuple[str,str][], exitOnConflict:bool = True) -> Tuple[str,str][] = + ''' Attempt to put all rename actions into an order that avoids target = source names. ''' + sources:str[]; targets;str[] + sources, targets = zip(*actions) + last:int = len(actions) + while last > 1: + for i in range(1, last): + try: + index:int = sources[:i].index(targets[i]) + sources.insert(index, sources.pop(i)) # bubble up the action right before conflict + targets.insert(index, targets.pop(i)) + except: continue + last -= 1 # we know that the last entry in the list has the least conflicts, so we can disregard it in the next iteration + if exitOnConflict: + for i in range(1, len(actions)): + if targets[i] in sources[:i]: Exit("There is no order of renaming actions that avoids copying over not-yet renamed files: '%s' is contained in matching source filenames" % (targets[i])) + zip(sources, targets) + def relativize(root:str, path:str) -> Tuple[str,str] = - ''' Gets relative path for specified file. ''' + ''' Determine OS-independent relative folder path, and relative pattern path. ''' relpath = os.path.relpath(os.path.dirname(os.path.abspath(path)), root).replace(os.sep, SLASH) relpath, os.path.join(relpath, os.path.basename(path)).replace(os.sep, SLASH) -def parseOnlyOptions(root:str, options:str[]) -> Tuple[Frozenset[str]?, Frozenset[str]?] = +def parseOnlyOptions(root:str, options:str[]) -> Tuple[FrozenSet[str]?, FrozenSet[str]?] = + ''' Returns set of --only arguments, and set or --except arguments. ''' cwd:str = os.getcwd() - onlys:str[] = []; excps:str[] = []; index:int = 0 + onlys:List[str] = []; excps:List[str] = []; index:int = 0 while True: try: - index = 1 + options.index("--only", index) + index = 1 + listindex(options, "--only", index) onlys.append(options[index]) except: break index = 0 while True: try: - index = 1 + options.index("--except", index) + index = 1 + listindex(options, "--except", index) excps.append(options[index]) except: break (frozenset(relativize(root, o) for o in onlys) if onlys else None, frozenset(relativize(root, e) for e in excps) if excps else None) - -def conditionalIntersection(a:FrozenSet[str]?, b:FrozenSet[str]) -> FrozenSet[str] = a & b if a else b