Fixes #94

README.md

@@ -1,4 +1,4 @@
-# Subversion Offline Solution (SOS 1.0.8) #
+# Subversion Offline Solution (SOS 1.0.9) #
 
 [![Travis badge](https://travis-ci.org/ArneBachmann/sos.svg?branch=master)](https://travis-ci.org/ArneBachmann/sos)
 [![Build status](https://ci.appveyor.com/api/projects/status/fe915rtx02buqe4r?svg=true)](https://ci.appveyor.com/project/ArneBachmann/sos)
@@ -36,15 +36,15 @@ SOS supports three different file handling models that you may use to your likin
 ### Unique features of SOS ###
 - Initializes repositories by default with the *simple mode*, which makes effortless versioning a piece of cake
 - In the optional tracking mode, files are tracked based on glob patterns instead of pure filenames or paths (in a manner comparable to how SVN ignores files)
-- In-place command line replacement for traditional VCS that transparently pipes commands to them
+- Command line replacement for traditional VCS that transparently pipes commands to them
 - Straightforward and simplified semantics for common VCS operations (`branch`, `commit`, integrate changes)
 
 ### Limitations ###
 - Designed for use by single user, network synchronization is a non-goal. Don't attempt to use SOS in a shared location, concurrent access to the repository may corrupt your data, as there is currently no locking in place (could be augmented, but it's a non-goal too)
 - Has a small user base as of now, therefore no reliable reports of compatibility and operational capability except for the automatic unit tests run on Travis CI and AppVeyor
 
 ### Compatibility ###
-- SOS runs on any Python 3 distribution, except PyPy (TODO). Support for Python 2 is only partial, as the test suite doesn't run through entirely yet, although SOS's programming language Coconut is generally able to transpile to valid Python 2 source code
+- SOS runs on any Python 3 distribution, including some versions of PyPy. Python 2 is not fully supported yet due to library issues, although SOS's programming language *Coconut* is generally able to transpile to valid Python 2 source code
 - SOS is compatible with above mentioned traditional VCSs: SVN, Git, gitless, Bazaar, Mercurial and Fossil
 - File name encoding and console encoding: Full roundtrip support (on Windows) started only with Python 3.6.4 and has not been tested nor confirmed yet for SOS
 
@@ -55,8 +55,8 @@ SOS supports three different file handling models that you may use to your likin
     - [Bug 93](https://github.com/ArneBachmann/sos/issues/93) Picky mode lists any file as added
     - [Enhancement 63](https://github.com/ArneBachmann/sos/issues/63) Show more change details in `log` and `status`
     - [Enhancement 86](https://github.com/ArneBachmann/sos/issues/86) Renamed command for branch removal to `destroy`
-    - [Feature 61](https://github.com/ArneBachmann/sos/issues/61) Added option to only consider or exclude certain file patterns for relevant operations using `--only` and `--except`
     - [Feature 8](https://github.com/ArneBachmann/sos/issues/8) Added functionality to rename tracking patterns and move files accordingly
+    - [Feature 61](https://github.com/ArneBachmann/sos/issues/61) Added option to only consider or exclude certain file patterns for relevant operations using `--only` and `--except`. Note: These have to be already tracked file patterns, currently, see [#99](https://github.com/ArneBachmann/sos/issues/99) and [#100](https://github.com/ArneBachmann/sos/issues/100)
     - [Feature 80](https://github.com/ArneBachmann/sos/issues/80) Added functionality to use tags
     - [QA 79](https://github.com/ArneBachmann/sos/issues/79) Added AppVeyor automated testing
     - [QA 94](https://github.com/ArneBachmann/sos/issues/94) More test coverage

setup.py

@@ -1,7 +1,7 @@
 import os, shutil, subprocess, sys, time, unittest
 from setuptools import setup, find_packages
 
-RELEASE = "1.0.8"
+RELEASE = "1.0.9"
 
 print("sys.argv is %r" % sys.argv)
 readmeFile = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'README.md')

sos/sos.coco

@@ -89,8 +89,8 @@ Usage: {cmd} <command> [<argument>] [<option1>, ...]        When operating in of
     --strict                     Always perform full content comparison, don't rely only on file size and timestamp
                                    for offline command: memorize strict mode setting in repository
                                    for changes, diff, commit, switch, update, delete: perform operation in strict mode, regardless of repository setting
-    --only <filename or glob>    Restrict operation to specified pattern(s). Available for "changes", "commit", "diff", "switch", and "update"
-    --except <filename or glob>  Avoid operation for specified pattern(s). Available for "changes", "commit", "diff", "switch", and "update"
+    --only <tracked pattern>     Restrict operation to specified pattern(s). Available for "changes", "commit", "diff", "switch", and "update"
+    --except <tracked pattern >  Avoid operation for specified pattern(s). Available for "changes", "commit", "diff", "switch", and "update"
     --{cmd}                      When executing {CMD} not being offline, pass arguments to {CMD} instead (e.g. {cmd} --{cmd} config set key value.)
     --log                        Enable logging details
     --verbose                    Enable verbose output""".format(appname = APPNAME, cmd = "sos", CMD = "SOS"))
@@ -265,14 +265,14 @@ class Metadata:
     knownPaths:Dict[str,List[str]] = collections.defaultdict(list)
     for path, pinfo in _.paths.items():  # TODO check dontConsider below
       if pinfo.size is not None\
-          and (considerOnly is None or any(path[:path.rindex(SLASH)] == pattern[:pattern.rindex(SLASH) and fnmatch.fnmatch(path[path.rindex(SLASH) + 1:], pattern)] for pattern in considerOnly))\
-          and (dontConsider is None or not any(path[:path.rindex(SLASH)] == pattern[:pattern.rindex(SLASH) and fnmatch.fnmatch(path[path.rindex(SLASH) + 1:], pattern)] for pattern in dontConsider)):
+          and (considerOnly is None or     any(path[:path.rindex(SLASH)] == pattern[:pattern.rindex(SLASH)] and fnmatch.fnmatch(path[path.rindex(SLASH) + 1:], pattern[pattern.rindex(SLASH) + 1:]) for pattern in considerOnly))\
+          and (dontConsider is None or not any(path[:path.rindex(SLASH)] == pattern[:pattern.rindex(SLASH)] and fnmatch.fnmatch(path[path.rindex(SLASH) + 1:], pattern[pattern.rindex(SLASH) + 1:]) for pattern in dontConsider)):
         knownPaths[os.path.dirname(path)].append(os.path.basename(path))  # TODO reimplement using fnmatch.filter for all files per path for speed
     for path, dirnames, filenames in os.walk(_.root):
       dirnames[:]  = [f for f in dirnames  if len([n for n in _.c.ignoreDirs if fnmatch.fnmatch(f, n)]) == 0 or len([p for p in _.c.ignoreDirsWhitelist if fnmatch.fnmatch(f, p)]) > 0]
       filenames[:] = [f for f in filenames if len([n for n in _.c.ignores    if fnmatch.fnmatch(f, n)]) == 0 or len([p for p in _.c.ignoresWhitelist    if fnmatch.fnmatch(f, p)]) > 0]
       dirnames.sort(); filenames.sort()
-      relPath = os.path.relpath(path, _.root).replace(os.sep, SLASH)
+      relPath = os.path.relpath(path, _.root).replace(os.sep, SLASH)  # TODO removes ./ ??
       walk = filenames if considerOnly is None else list(reduce((last, pattern) -> last | set(fnmatch.filter(filenames, os.path.basename(pattern))), (p for p in considerOnly if os.path.dirname(p).replace(os.sep, SLASH) == relPath), s{}))
       if dontConsider:
         walk[:] = [fn for fn in walk if not any(fnmatch.fnmatch(fn, os.path.basename(p)) for p in dontConsider if os.path.dirname(p).replace(os.sep, SLASH) == relPath)]  # TODO dirname is correct noramalized? same above?
@@ -284,7 +284,7 @@ class Metadata:
         if progress and newtime - timer > .1:
           outstring = "\rPreparing %s  %s" % (PROGRESS_MARKER[counter.inc() % 4], filename)
           sys.stdout.write(outstring + " " * max(0, termWidth - len(outstring))); sys.stdout.flush(); timer = newtime  # TODO could write to new line instead of carriage return, also needs terminal width
-        if filename not in _.paths:  # detected file not present (or untracked) in other branch
+        if filename not in _.paths:  # detected file not present (or untracked) in (other) branch
           nameHash = hashStr(filename)
           hashed, written = hashFile(filepath, _.compress, saveTo = os.path.join(_.root, metaFolder, "b%d" % branch, "r%d" % revision, nameHash) if write else None) if size > 0 else (None, 0)
           changes.additions[filename] = PathInfo(nameHash, size, mtime, hashed)
@@ -294,7 +294,7 @@ class Metadata:
         if last.size is None:  # was removed before but is now added back - does not apply for tracking mode (which never marks files for removal in the history)
           hashed, written = hashFile(filepath, _.compress, saveTo = os.path.join(_.root, metaFolder, "b%d" % branch, "r%d" % revision, last.nameHash) if write else None) if size > 0 else (None, 0)
           changes.additions[filename] = PathInfo(last.nameHash, size, mtime, hashed); continue
-        elif size != last.size or mtime != last.mtime or (checkContent and hashFile(filepath, _.compress) != last.hash):  # detected a modification
+        elif size != last.size or mtime != last.mtime or (checkContent and hashFile(filepath, _.compress)[0] != last.hash):  # detected a modification
           hashed, written = hashFile(filepath, _.compress, saveTo = os.path.join(_.root, metaFolder, "b%d" % branch, "r%d" % revision, last.nameHash) if write else None) if (last.size if inverse else size) > 0 else (last.hash if inverse else None, 0)
           changes.modifications[filename] = PathInfo(last.nameHash, last.size if inverse else size, last.mtime if inverse else mtime, hashed)
         else: continue
@@ -476,7 +476,7 @@ def changes(argument:str = None, options:str[] = [], onlys:FrozenSet[str]? = Non
   m.computeSequentialPathSet(branch, revision)  # load all commits up to specified revision
   changes:ChangeSet = m.findChanges(checkContent = strict, considerOnly = onlys if not m.track and not m.picky else conditionalIntersection(onlys, m.getTrackingPatterns() | m.getTrackingPatterns(branch)), dontConsider = excps)
   m.listChanges(changes)
-  changes
+  changes  # for unit tests only
 
 def diff(argument:str, options:str[] = [], onlys:FrozenSet[str]? = None, excps:FrozenSet[str]? = None) -> None:
   ''' Show text file differences of file tree vs. (last or specified) revision on current or specified branch. '''
@@ -536,7 +536,7 @@ def commit(argument:str? = None, options:str[] = [], onlys:FrozenSet[str]? = Non
   m.commits[revision] = CommitInfo(revision, longint(time.time() * 1000), argument)  # comment can be None
   m.saveBranch(m.branch)
   m.loadBranches()  # TODO is it necessary to load again?
-  if m.picky:  # HINT was changed from only picky to include track as well
+  if m.picky:
     m.branches[m.branch] = dataCopy(BranchInfo, m.branches[m.branch], tracked = [], inSync = False)  # remove tracked patterns
   else:  # track or simple mode
     m.branches[m.branch] = dataCopy(BranchInfo, m.branches[m.branch], inSync = False)  # set branch dirty
@@ -576,7 +576,7 @@ def exitOnChanges(argument:str? = None, options:str[] = [], check:bool = True, c
   # Determine current changes
   trackingPatterns:FrozenSet[str] = m.getTrackingPatterns()
   m.computeSequentialPathSet(m.branch, max(m.commits))  # load all commits up to specified revision
-  changes:ChangeSet = m.findChanges(m.branch if commit else None, max(m.commits) + 1 if commit else None, checkContent = strict, considerOnly = onlys if not m.track and not m.picky else conditionalIntersection(onlys, trackingPatterns), dontConsider = excps)
+  changes:ChangeSet = m.findChanges(m.branch if commit else None, max(m.commits) + 1 if commit else None, checkContent = strict, considerOnly = onlys if not (m.track or m.picky) else conditionalIntersection(onlys, trackingPatterns), dontConsider = excps)
   if check and modified(changes) and not force:
     m.listChanges(changes)
     if not commit: Exit("File tree contains changes. Use --force to proceed")

sos/tests.coco

@@ -97,6 +97,7 @@ class Tests(unittest.TestCase):
   def assertNotInAny(_, what:str, where:str[]) -> None: _.assertFalse(any(what in w for w in where))
 
   def createFile(_, number:Union[int,str], contents:str = "x" * 10, prefix:str? = None) -> None:
+    if prefix and not os.path.exists(prefix): os.makedirs(prefix)
     with open(("." if prefix is None else prefix) + os.sep + (("file%d" % number) if isinstance(number, int) else number), "wb") as fd: fd.write(contents if isinstance(contents, bytes) else contents.encode("cp1252"))
 
   def existsFile(_, number:Union[int, str], expectedContents:bytes = None) -> bool:
@@ -211,6 +212,22 @@ class Tests(unittest.TestCase):
     _.assertEqual(["xxayb", "aacb"], [r[1] for r in sos.convertGlobFiles(["axxby", "aabc"], *sos.tokenizeGlobPatterns("a*b?", "*a?b"))])
     _.assertEqual(["1qq2ww3", "1abcbx2xbabc3"], [r[1] for r in sos.convertGlobFiles(["qqxbww", "abcbxxbxbabc"], *sos.tokenizeGlobPatterns("*xb*", "1*2*3"))])
 
+  def testFolderRemove(_):
+    m = sos.Metadata(os.getcwd())
+    _.createFile(1)
+    _.createFile("a", prefix = "sub")
+    sos.offline()
+    _.createFile(2)
+    os.unlink("sub" + os.sep + "a")
+    os.rmdir("sub")
+    changes = sos.changes()
+    _.assertEqual(1, len(changes.additions))
+    _.assertEqual(0, len(changes.modifications))
+    _.assertEqual(1, len(changes.deletions))
+    _.createFile("a", prefix = "sub")
+    changes = sos.changes()
+    _.assertEqual(0, len(changes.deletions))
+
   def testComputeSequentialPathSet(_):
     os.makedirs(branchFolder(0, 0))
     os.makedirs(branchFolder(0, 1))
@@ -480,6 +497,8 @@ class Tests(unittest.TestCase):
   def testPickyMode(_):
     ''' Confirm that picky mode reset tracked patterns after commits. '''
     sos.offline("trunk", ["--picky"])
+    changes = sos.changes()
+    _.assertEqual(0, len(changes.additions))  # do not list any existing file as an addition
     sos.add(".", "./file?", ["--force"])
     _.createFile(1, "aa")
     sos.commit("First")  # add one file
@@ -707,6 +726,22 @@ class Tests(unittest.TestCase):
     _.assertEqual(f{"B"}, sos.conditionalIntersection(f{"A", "B", "C"}, f{"B", "D"}))
     _.assertEqual(f{"B", "D"}, sos.conditionalIntersection(f{}, f{"B", "D"}))
     _.assertEqual(f{"B", "D"}, sos.conditionalIntersection(None, f{"B", "D"}))
+    sos.offline(os.getcwd(), ["--track", "--strict"])
+    _.createFile(1)
+    _.createFile(2)
+    sos.add(".", "./file1")
+    sos.add(".", "./file2")
+    sos.commit(onlys = f{"./file1"})
+    _.assertEqual(2, len(os.listdir(branchFolder(0, 1))))  # only meta and file1
+    import pdb; pdb.set_trace()
+    sos.commit()  # adds also file2
+    _.assertEqual(2, len(os.listdir(branchFolder(0, 2))))  # only meta and file1
+    _.createFile(1, "cc")  # modify both files
+    _.createFile(2, "dd")
+    changes = sos.changes(excps = ["./file1"])
+    _.assertEqual(1, len(changes.modifications))  # only file2
+    _.assertTrue("./file2" in changes.modifications)
+    _.assertAllIn(["MOD ./file2", "DIF ./file2"], wrapChannels(() -> sos.diff(onlys = f{"./file2"})))
 
   def testDiff(_):
     sos.offline(options = ["--strict"])