From 2751f6ff86e3f90c9e14b8b8acc8ef02c86987e1 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Fri, 4 Oct 2019 23:32:10 +0200 Subject: [PATCH 1/2] Quick fix for CVE Request 766166 --- rediswrapper/models.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/rediswrapper/models.py b/rediswrapper/models.py index b5304b9..8b0ec06 100644 --- a/rediswrapper/models.py +++ b/rediswrapper/models.py @@ -191,18 +191,11 @@ def _from_iterable(self, other): def from_value(value): """Convert a value to be stored in redis""" - if isinstance(value, basestring): - # Keep most readability, do not pickle string values - return value - try: - return pickle.dumps(value) - except Exception: - return value + return pickle.dumps(value) + def to_value(pickled): """Convert a storage value from redis to human readable""" - try: - return pickle.loads(pickled) - except: - return pickled.decode('utf8') + return pickle.loads(pickled) + From f24212589b568c9f9ae11d4bf504f42303b341d5 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sat, 5 Oct 2019 16:02:18 +0800 Subject: [PATCH 2/2] Fix test cases --- .gitignore | 1 + rediswrapper/models.py | 38 ++++---- setup.py | 78 ++++++++-------- test_rediswrapper.py | 206 ++++++++++++++++++++--------------------- 4 files changed, 162 insertions(+), 161 deletions(-) diff --git a/.gitignore b/.gitignore index f2ef958..24058cd 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,4 @@ target/ .ipynb_checkpoints venv/ .python-version +.vscode/ diff --git a/rediswrapper/models.py b/rediswrapper/models.py index 8b0ec06..01f460c 100644 --- a/rediswrapper/models.py +++ b/rediswrapper/models.py @@ -4,6 +4,7 @@ from collections import MutableSet from collections import MutableMapping from collections import MutableSequence + try: import cPickle as pickle except ImportError: @@ -23,6 +24,7 @@ def __init__(self, key, client): class HashType(RedisType, MutableMapping): """Dict-like wrapper for list data""" + def __getitem__(self, key): if not self._r.hexists(self.key, key): raise KeyError(key) @@ -39,7 +41,7 @@ def __delitem__(self, key): def __iter__(self): for key in self._r.hkeys(self.key): if isinstance(key, bytes): - key = key.decode('utf8') + key = key.decode("utf8") yield key def __len__(self): @@ -52,7 +54,7 @@ def __getattr__(self, name): raise AttributeError(name) def __repr__(self): - return '%s value(%s)' % (self.__class__.__name__, dict(self)) + return "%s value(%s)" % (self.__class__.__name__, dict(self)) def _set(self, other): for key in self: @@ -62,6 +64,7 @@ def _set(self, other): class ListType(RedisType, MutableSequence): """List-like wrapper for list data""" + def __setitem__(self, index, value): if isinstance(index, slice): gen = iter(value) @@ -74,7 +77,7 @@ def __setitem__(self, index, value): if indices[2] != 1: raise ValueError("The value length doesn't match") else: - del self[i:indices[1]] + del self[i : indices[1]] break else: # The value length is larger than slice @@ -88,11 +91,10 @@ def __setitem__(self, index, value): elif isinstance(index, int): index = index + len(self) if index < 0 else index if index >= len(self): - raise IndexError('Index out of range') + raise IndexError("Index out of range") self._r.lset(self.key, index, from_value(value)) else: - raise TypeError('list indices must be integers, not %r' - % type(index)) + raise TypeError("list indices must be integers, not %r" % type(index)) def __getitem__(self, index): if isinstance(index, slice): @@ -101,9 +103,9 @@ def __getitem__(self, index): elif isinstance(index, int): index = index + len(self) if index < 0 else index if index >= len(self): - raise IndexError('Index out of range') + raise IndexError("Index out of range") return to_value(self._r.lindex(self.key, index)) - raise TypeError('list indices must be integers, not %r' % type(index)) + raise TypeError("list indices must be integers, not %r" % type(index)) def __delitem__(self, index): if isinstance(index, slice): @@ -114,18 +116,17 @@ def __delitem__(self, index): elif isinstance(index, int): index = index + len(self) if index < 0 else index if index >= len(self): - raise IndexError('Index out of range') + raise IndexError("Index out of range") if index == 0: return self._r.lpop(self.key) elif index == len(self) - 1: return self._r.rpop(self.key) - temp = self._r.lrange(self.key, index+1, len(self)) - self._r.ltrim(self.key, 0, index-1) + temp = self._r.lrange(self.key, index + 1, len(self)) + self._r.ltrim(self.key, 0, index - 1) for item in temp: self._r.rpush(self.key, item) else: - raise TypeError('list indices must be integers, not %r' - % type(index)) + raise TypeError("list indices must be integers, not %r" % type(index)) def __len__(self): return self._r.llen(self.key) @@ -136,7 +137,7 @@ def __eq__(self, other): return list(self) == list(other) def __repr__(self): - return '%s value(%s)' % (self.__class__.__name__, list(self)) + return "%s value(%s)" % (self.__class__.__name__, list(self)) def insert(self, index, value): index = index + len(self) if index < 0 else index @@ -144,7 +145,7 @@ def insert(self, index, value): if index == 0: return self._r.lpush(self.key, from_value(value)) temp = self._r.lrange(self.key, index, len(self)) - self._r.ltrim(self.key, 0, index-1) + self._r.ltrim(self.key, 0, index - 1) self.append(value) for item in temp: self._r.rpush(self.key, item) @@ -157,6 +158,7 @@ def _set(self, other): class SetType(RedisType, MutableSet): """Set-like wrapper for list data""" + def __contains__(self, value): return self._r.sismember(self.key, from_value(value)) @@ -174,7 +176,7 @@ def discard(self, value): self._r.srem(self.key, from_value(value)) def __repr__(self): - return '%s value(%s)' % (self.__class__.__name__, list(self)) + return "%s value(%s)" % (self.__class__.__name__, list(self)) def _set(self, other): for v in self: @@ -186,16 +188,14 @@ def _from_iterable(self, other): return set(other) -type_map = {'list': ListType, 'hash': HashType, 'set': SetType} +type_map = {"list": ListType, "hash": HashType, "set": SetType} def from_value(value): """Convert a value to be stored in redis""" return pickle.dumps(value) - def to_value(pickled): """Convert a storage value from redis to human readable""" return pickle.loads(pickled) - diff --git a/setup.py b/setup.py index a8553ad..668b9b5 100644 --- a/setup.py +++ b/setup.py @@ -4,28 +4,29 @@ from setuptools import setup -if sys.argv[-1] == 'publish': +if sys.argv[-1] == "publish": os.system("python setup.py sdist upload") sys.exit() -if sys.argv[-1] == 'test': +if sys.argv[-1] == "test": try: - __import__('py') + __import__("pytest") except ImportError: - print('pytest required.') + print("pytest required.") sys.exit(1) - errors = os.system('py.test') + errors = os.system("pytest") sys.exit(bool(errors)) def get_version(): - content = open('rediswrapper/__init__.py').read() + content = open("rediswrapper/__init__.py").read() return re.findall(r'__version__\s*=\s*[\'"](.*)[\'"]', content)[0] -readme = 'README' -for name in os.listdir('.'): - if name.startswith('README'): + +readme = "README" +for name in os.listdir("."): + if name.startswith("README"): readme = name break try: @@ -34,32 +35,33 @@ def get_version(): except: long_description = "RedisWrapper, a pythonic wrapper for redis client" -setup(name='rediswrapper', - version=get_version(), - description='Pythonic wrapper for Redis Client.', - url='https://github.com/frostming/rediswrapper', - author='Frost Ming', - author_email='mianghong@gmail.com', - license='MIT', - packages=['rediswrapper'], - test_suite='test_rediswrapper', - zip_safe=False, - long_description=long_description, - keywords='redis client mock', - test_requires=['pytest', 'fakeredis'], - install_requires=['redis'], - classifiers=[ - "Intended Audience :: Developers", - "Operating System :: OS Independent", - "Topic :: Software Development", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Development Status :: 3 - Alpha", - "License :: OSI Approved :: MIT License" - ], - ) +setup( + name="rediswrapper", + version=get_version(), + description="Pythonic wrapper for Redis Client.", + url="https://github.com/frostming/rediswrapper", + author="Frost Ming", + author_email="mianghong@gmail.com", + license="MIT", + packages=["rediswrapper"], + test_suite="test_rediswrapper", + zip_safe=False, + long_description=long_description, + keywords="redis client mock", + test_requires=["pytest", "fakeredis"], + install_requires=["redis"], + classifiers=[ + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Topic :: Software Development", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: MIT License", + ], +) diff --git a/test_rediswrapper.py b/test_rediswrapper.py index 1954fc3..2309019 100644 --- a/test_rediswrapper.py +++ b/test_rediswrapper.py @@ -9,14 +9,16 @@ from collections import Set, MutableSequence now = datetime.datetime.now() -golden = {'a': 'hello', - 'b': True, - 'c': 1, - 'd': None, - 'e': now, - 'f': list(range(5)), - 'g': dict(zip('abcd', range(4))), - 'h': set(['a', 'b', 'c'])} +golden = { + "a": "hello", + "b": True, + "c": 1, + "d": None, + "e": now, + "f": list(range(5)), + "g": dict(zip("abcd", range(4))), + "h": set(["a", "b", "c"]), +} def cmp_no_order(seq1, seq2): @@ -36,7 +38,7 @@ class RedisDictTestCase(unittest.TestCase): def setUp(self): global redis - redis = RedisDict(prefix='test', client=FakeStrictRedis) + redis = RedisDict(prefix="test", client=FakeStrictRedis) redis.update(golden) def tearDown(self): @@ -44,93 +46,93 @@ def tearDown(self): def test_prefix(self): """Test the key is prefixed correctly""" - for c in 'abcdefgh': - key = 'test.' + c + for c in "abcdefgh": + key = "test." + c assert redis._r.exists(key) def test_contains(self): - for c in 'abcdefgh': + for c in "abcdefgh": assert c in redis def test_get_item(self): - assert redis['d'] is None - assert redis.get('a') == 'hello' + assert redis["d"] is None + assert redis.get("a") == "hello" with pytest.raises(KeyError): - redis['z'] + redis["z"] def test_iter_item(self): assert dict(redis) == golden def test_keys(self): - assert cmp_no_order(redis.keys(), list('abcdefgh')) + assert cmp_no_order(redis.keys(), list("abcdefgh")) def test_set_item(self): - redis['a'] = 'hello michael' - assert redis['a'] == 'hello michael' - assert redis.setdefault('i', 1.2) == 1.2 - assert 'i' in redis + redis["a"] = "hello michael" + assert redis["a"] == "hello michael" + assert redis.setdefault("i", 1.2) == 1.2 + assert "i" in redis def test_set_different_type(self): - redis['b'] = [1, 2, 3] - assert isinstance(redis['b'], MutableSequence) - redis['f'] = 'hello' - assert redis['f'] == 'hello' - redis['g'] = set([1, 2, 3]) - assert isinstance(redis['g'], Set) + redis["b"] = [1, 2, 3] + assert isinstance(redis["b"], MutableSequence) + redis["f"] = "hello" + assert redis["f"] == "hello" + redis["g"] = set([1, 2, 3]) + assert isinstance(redis["g"], Set) def test_del_item(self): - del redis['a'] - assert 'a' not in redis + del redis["a"] + assert "a" not in redis with pytest.raises(KeyError): - del redis['z'] + del redis["z"] def test_get_attribute(self): assert redis.f == list(range(5)) - redis['get'] = 1 + redis["get"] = 1 assert callable(redis.get) with pytest.raises(AttributeError): redis.other def test_list_get_item(self): - data = redis['f'] + data = redis["f"] assert data[1] == 1 with pytest.raises(IndexError): data[10] with pytest.raises(TypeError): - data['1'] + data["1"] def test_list_get_slice(self): - data = redis['f'] + data = redis["f"] assert data[2:] == [2, 3, 4] assert data[:-2] == [0, 1, 2] assert data[1:4:2] == [1, 3] def test_list_contains(self): - data = redis['f'] + data = redis["f"] assert 1 in data - assert 'a' not in data + assert "a" not in data def test_list_set_item(self): - data = redis['f'] + data = redis["f"] data[1] = 4 - assert redis['f'] == [0, 4, 2, 3, 4] + assert redis["f"] == [0, 4, 2, 3, 4] data[-1] = 1 - assert redis['f'] == [0, 4, 2, 3, 1] + assert redis["f"] == [0, 4, 2, 3, 1] with pytest.raises(IndexError): data[10] = 10 with pytest.raises(TypeError): - data['1'] = 1 + data["1"] = 1 def test_list_set_slice(self): - data = redis['f'] - data[1:3] = ['a', 'b'] - assert redis['f'] == [0, 'a', 'b', 3, 4] - data[1:1] = ['c', 'd'] - assert redis['f'] == [0, 'c', 'd', 'a', 'b', 3, 4] + data = redis["f"] + data[1:3] = ["a", "b"] + assert redis["f"] == [0, "a", "b", 3, 4] + data[1:1] = ["c", "d"] + assert redis["f"] == [0, "c", "d", "a", "b", 3, 4] data[:] = [1, 2, 3] - assert redis['f'] == [1, 2, 3] - data[::2] = ['a', 'b'] - assert redis['f'] == ['a', 2, 'b'] + assert redis["f"] == [1, 2, 3] + data[::2] = ["a", "b"] + assert redis["f"] == ["a", 2, "b"] with pytest.raises(ValueError): # target length < source length and step != 1 data[::2] = [1] @@ -142,106 +144,102 @@ def test_list_set_slice(self): data[::-1] = [1, 3] def test_list_del_item(self): - data = redis['f'] + data = redis["f"] del data[1] - assert len(redis['f']) == 4 + assert len(redis["f"]) == 4 del data[-1] - assert redis['f'] == [0, 2, 3] + assert redis["f"] == [0, 2, 3] with pytest.raises(IndexError): del data[10] with pytest.raises(TypeError): - del data['1'] + del data["1"] def test_list_del_slice(self): - data = redis['f'] + data = redis["f"] del data[1:1] - assert len(redis['f']) == 5 + assert len(redis["f"]) == 5 del data[1:5:2] - assert redis['f'] == [0, 2, 4] + assert redis["f"] == [0, 2, 4] del data[:] - assert len(redis['f']) == 0 + assert "f" not in redis def test_list_insert(self): - data = redis['f'] - data.insert(0, 'a') - assert data == ['a', 0, 1, 2, 3, 4] - data.insert(2, 'b') - assert data == ['a', 0, 'b', 1, 2, 3, 4] - data.insert(10, 'c') - assert data == ['a', 0, 'b', 1, 2, 3, 4, 'c'] + data = redis["f"] + data.insert(0, "a") + assert data == ["a", 0, 1, 2, 3, 4] + data.insert(2, "b") + assert data == ["a", 0, "b", 1, 2, 3, 4] + data.insert(10, "c") + assert data == ["a", 0, "b", 1, 2, 3, 4, "c"] def test_list_append_pop(self): - data = redis['f'] - data.append('a') - assert data.pop() == 'a' + data = redis["f"] + data.append("a") + assert data.pop() == "a" assert data.pop(0) == 0 assert len(data) == 4 def test_hash_contains(self): - assert 'a' in redis['g'] - assert 2 not in redis['g'] + assert "a" in redis["g"] + assert 2 not in redis["g"] def test_hash_get_item(self): - assert redis['g']['b'] == 1 - assert redis['g'].get('c') == 2 + assert redis["g"]["b"] == 1 + assert redis["g"].get("c") == 2 with pytest.raises(KeyError): - redis['g'][5] + redis["g"][5] def test_hash_set_item(self): - redis['g'][1] = True - assert redis['g'][1] is True - redis['g']['c'] = 'c' - assert redis['g']['c'] == 'c' - redis['g']['other'] = [1, 2] - assert redis['g']['other'] == [1, 2] + redis["g"][1] = True + assert redis["g"][1] is True + redis["g"]["c"] = "c" + assert redis["g"]["c"] == "c" + redis["g"]["other"] = [1, 2] + assert redis["g"]["other"] == [1, 2] def test_hash_del_item(self): - del redis['g']['d'] - assert 'd' not in redis['g'] + del redis["g"]["d"] + assert "d" not in redis["g"] with pytest.raises(KeyError): - del redis['g'][5] + del redis["g"][5] def test_hash_representation(self): - assert redis['g'] == {'a': 0, - 'b': 1, - 'c': 2, - 'd': 3} + assert redis["g"] == {"a": 0, "b": 1, "c": 2, "d": 3} def test_hash_keys(self): - assert cmp_no_order(redis['g'].keys(), list('abcd')) + assert cmp_no_order(redis["g"].keys(), list("abcd")) def test_hash_get_attr(self): - assert callable(redis['g'].get) - assert redis['g'].c == 2 + assert callable(redis["g"].get) + assert redis["g"].c == 2 with pytest.raises(AttributeError): - redis['g'].other + redis["g"].other def test_set_contains(self): - assert 'a' in redis['h'] - assert 5 not in redis['h'] + assert "a" in redis["h"] + assert 5 not in redis["h"] def test_set_representation(self): - assert set(redis['h']) == set(['a', 'b', 'c']) + assert set(redis["h"]) == set(["a", "b", "c"]) def test_set_add(self): - redis['h'].add('a') - assert len(redis['h']) == 3 - redis['h'].add('d') - assert 'd' in redis['h'] + redis["h"].add("a") + assert len(redis["h"]) == 3 + redis["h"].add("d") + assert "d" in redis["h"] def test_set_discard(self): - redis['h'].discard('d') - assert len(redis['h']) == 3 - redis['h'].discard('b') - assert len(redis['h']) == 2 + redis["h"].discard("d") + assert len(redis["h"]) == 3 + redis["h"].discard("b") + assert len(redis["h"]) == 2 def test_set_op(self): - assert redis['h'] == set(['a', 'b', 'c']) - assert redis['h'] < set(['a', 'b', 'c', 'd', 'e']) - assert redis['h'] & set(['a']) == set(['a']) - assert redis['h'] | set(['a', 'e']) \ - == set(['a', 'b', 'c', 'e']) + assert redis["h"] == set(["a", "b", "c"]) + assert redis["h"] < set(["a", "b", "c", "d", "e"]) + assert redis["h"] & set(["a"]) == set(["a"]) + assert redis["h"] | set(["a", "e"]) == set(["a", "b", "c", "e"]) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main()