From bb433ea3dffad8d2171e2ea89db92f9e5fe94bb5 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Thu, 29 Sep 2016 09:35:28 -0700 Subject: [PATCH 1/5] Making setup.py look like the other setup files. --- setup.py | 66 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/setup.py b/setup.py index c8f65c3..05c4baa 100644 --- a/setup.py +++ b/setup.py @@ -14,48 +14,58 @@ import os -from setuptools import setup from setuptools import find_packages +from setuptools import setup -here = os.path.abspath(os.path.dirname(__file__)) +PACKAGE_ROOT = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(here, 'README.rst')) as f: - README = f.read() +with open(os.path.join(PACKAGE_ROOT, 'README.rst')) as file_obj: + README = file_obj.read() + +# NOTE: This is duplicated throughout and we should try to +# consolidate. +SETUP_BASE = { + 'author': 'Google Cloud Platform', + 'author_email': 'jjg+google-cloud-python@google.com', + 'scripts': [], + 'url': 'https://github.com/GoogleCloudPlatform/google-cloud-python', + 'license': 'Apache 2.0', + 'platforms': 'Posix; MacOS X; Windows', + 'include_package_data': True, + 'zip_safe': False, + 'classifiers': [ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Topic :: Internet', + ], +} REQUIREMENTS = [ - 'gcloud', - 'grpcio >= 1.0.0, < 2.0dev', + 'google-cloud-bigtable >= 0.20.0', ] +SETUP_BASE.pop('url') setup( name='google-cloud-happybase', version='0.19.0', - description='API Client library for Google Cloud Happybase layer', - author='Google Cloud Platform', - author_email='jjg+gcloud-python@google.com', + description='Client library for Google Cloud Bigtable: HappyBase layer', long_description=README, - scripts=[], - url='https://github.com/GoogleCloudPlatform/gcloud-python', - license='Apache 2.0', - platforms='Posix; MacOS X; Windows', + url='https://github.com/GoogleCloudPlatform/google-cloud-python-happybase', + namespace_packages=[ + 'google', + 'google.cloud', + ], packages=find_packages('src'), package_dir={'': 'src'}, - namespace_packages=['google', 'google.cloud'], - include_package_data=True, - zip_safe=False, install_requires=REQUIREMENTS, - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Topic :: Internet', - ] + **SETUP_BASE ) From bc9e924b3fb7145ab2c858997e4c3526f37c0d55 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Thu, 29 Sep 2016 09:51:23 -0700 Subject: [PATCH 2/5] Upgrading from gcloud to google-cloud-bigtable. In the process, also renaming all instances of gcloud. --- .gitignore | 6 +- README.rst | 10 +- scripts/run_pylint.py | 8 +- src/google/cloud/happybase/__init__.py | 6 +- src/google/cloud/happybase/batch.py | 8 +- src/google/cloud/happybase/connection.py | 34 +++---- src/google/cloud/happybase/table.py | 69 ++++++------- system_tests/happybase.py | 10 +- system_tests/local_test_setup.sample | 5 +- system_tests/run_emulator.py | 8 +- system_tests/run_system_test.py | 2 +- system_tests/system_test_utils.py | 4 +- tox.ini | 4 +- unit_tests/test_batch.py | 10 +- unit_tests/test_connection.py | 38 ++++---- unit_tests/test_pool.py | 6 +- unit_tests/test_table.py | 118 ++++++++++++----------- 17 files changed, 176 insertions(+), 170 deletions(-) diff --git a/.gitignore b/.gitignore index fb3f348..8e4001c 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ pip-log.txt # Unit test / coverage reports .coverage .tox +.cache # Translations *.mo @@ -38,9 +39,12 @@ pip-log.txt .project .pydevproject +# JetBrains +.idea + # Built documentation docs/_build -docs/_build_rtd +docs/_build_doc2dash # Virtual environment env/ diff --git a/README.rst b/README.rst index 8216222..c35e295 100644 --- a/README.rst +++ b/README.rst @@ -32,12 +32,12 @@ Quick Start Authentication -------------- -With ``gcloud-python`` we try to make authentication as painless as possible. -Check out the `Authentication section`_ in our documentation to learn more. -You may also find the `authentication document`_ shared by all the ``gcloud-*`` -libraries to be helpful. +With ``google-cloud-python`` we try to make authentication as painless as +possible. Check out the `Authentication section`_ in our documentation to +learn more. You may also find the `authentication document`_ shared by all +the ``google-cloud-*`` libraries to be helpful. -.. _Authentication section: http://gcloud-python.readthedocs.org/en/latest/gcloud-auth.html +.. _Authentication section: http://google-cloud-python.readthedocs.io/en/latest/google-cloud-auth.html .. _authentication document: https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/authentication Contributing diff --git a/scripts/run_pylint.py b/scripts/run_pylint.py index 81cc578..98740a8 100644 --- a/scripts/run_pylint.py +++ b/scripts/run_pylint.py @@ -141,8 +141,8 @@ def get_files_for_linting(allow_limited=True): this value is not dependable. To allow faster local ``tox`` runs, the environment variables - ``GCLOUD_REMOTE_FOR_LINT`` and ``GCLOUD_BRANCH_FOR_LINT`` can be set to - specify a remote branch to diff against. + ``GOOGLE_CLOUD_REMOTE_FOR_LINT`` and ``GOOGLE_CLOUD_BRANCH_FOR_LINT`` can + be set to specify a remote branch to diff against. :type allow_limited: bool :param allow_limited: Boolean indicating if a reduced set of files can @@ -160,8 +160,8 @@ def get_files_for_linting(allow_limited=True): diff_base = 'origin/master' elif os.getenv('TRAVIS') is None: # Only allow specified remote and branch in local dev. - remote = os.getenv('GCLOUD_REMOTE_FOR_LINT') - branch = os.getenv('GCLOUD_BRANCH_FOR_LINT') + remote = os.getenv('GOOGLE_CLOUD_REMOTE_FOR_LINT') + branch = os.getenv('GOOGLE_CLOUD_BRANCH_FOR_LINT') if remote is not None and branch is not None: diff_base = '%s/%s' % (remote, branch) diff --git a/src/google/cloud/happybase/__init__.py b/src/google/cloud/happybase/__init__.py index 8777a61..1bb2712 100644 --- a/src/google/cloud/happybase/__init__.py +++ b/src/google/cloud/happybase/__init__.py @@ -92,7 +92,7 @@ .. code:: python - from gcloud.bigtable.client import Client + from google.cloud.bigtable.client import Client client = Client(project=PROJECT_ID, admin=True) instance = client.instance(instance_id, location_id) instance.reload() @@ -125,9 +125,9 @@ Only ``max_versions`` and ``time_to_live`` are availabe in Cloud Bigtable (as - :class:`MaxVersionsGCRule ` + :class:`~google.cloud.bigtable.column_family.MaxVersionsGCRule` and - :class:`MaxAgeGCRule `). + :class:`~google.cloud.bigtable.column_family.MaxAgeGCRule`). In addition to using a dictionary for specifying column family options, we also accept instances of :class:`.GarbageCollectionRule` or subclasses. diff --git a/src/google/cloud/happybase/batch.py b/src/google/cloud/happybase/batch.py index 52aadd2..193e39f 100644 --- a/src/google/cloud/happybase/batch.py +++ b/src/google/cloud/happybase/batch.py @@ -20,8 +20,8 @@ import six -from gcloud._helpers import _datetime_from_microseconds -from gcloud.bigtable.row_filters import TimestampRange +from google.cloud._helpers import _datetime_from_microseconds +from google.cloud.bigtable.row_filters import TimestampRange _WAL_SENTINEL = object() @@ -129,7 +129,7 @@ def _get_row(self, row_key): :type row_key: str :param row_key: The row key for a row stored in the map. - :rtype: :class:`Row ` + :rtype: :class:`~google.cloud.bigtable.row.Row` :returns: The newly created or stored row that will hold mutations. """ if row_key not in self._row_map: @@ -181,7 +181,7 @@ def _delete_columns(self, columns, row_object): * an entire column family: ``fam`` or ``fam:`` * a single column: ``fam:col`` - :type row_object: :class:`Row ` + :type row_object: :class:`~google.cloud.bigtable.row.Row` :param row_object: The row which will hold the delete mutations. :raises: :class:`ValueError ` if the delete diff --git a/src/google/cloud/happybase/connection.py b/src/google/cloud/happybase/connection.py index bad7517..53dd8ad 100644 --- a/src/google/cloud/happybase/connection.py +++ b/src/google/cloud/happybase/connection.py @@ -26,13 +26,13 @@ try: from happybase.hbase.ttypes import AlreadyExists except ImportError: - from gcloud.exceptions import Conflict as AlreadyExists + from google.cloud.exceptions import Conflict as AlreadyExists -from gcloud.bigtable.client import Client -from gcloud.bigtable.column_family import GCRuleIntersection -from gcloud.bigtable.column_family import MaxAgeGCRule -from gcloud.bigtable.column_family import MaxVersionsGCRule -from gcloud.bigtable.table import Table as _LowLevelTable +from google.cloud.bigtable.client import Client +from google.cloud.bigtable.column_family import GCRuleIntersection +from google.cloud.bigtable.column_family import MaxAgeGCRule +from google.cloud.bigtable.column_family import MaxVersionsGCRule +from google.cloud.bigtable.table import Table as _LowLevelTable from google.cloud.happybase.table import Table @@ -75,7 +75,7 @@ def _get_instance(timeout=None): :type timeout: int :param timeout: (Optional) The socket timeout in milliseconds. - :rtype: :class:`gcloud.bigtable.instance.Instance` + :rtype: :class:`~google.cloud.bigtable.instance.Instance` :returns: The unique instance owned by the project inferred from the environment. :raises ValueError: if there is a failed location or any number of @@ -110,9 +110,9 @@ class Connection(object): If you pass a ``instance``, it will be :meth:`.Instance.copy`-ed before being stored on the new connection. This also copies the - :class:`Client ` that created the - :class:`Instance ` instance and the - :class:`Credentials ` stored on the + :class:`~google.cloud.bigtable.client.Client` that created the + :class:`~google.cloud.bigtable.instance.Instance` instance and the + :class:`~oauth2client.client.Credentials` stored on the client. The arguments ``host``, ``port``, ``compat``, ``transport`` and @@ -134,13 +134,13 @@ class Connection(object): :param table_prefix_separator: (Optional) Separator used with ``table_prefix``. Defaults to ``_``. - :type instance: :class:`Instance ` + :type instance: :class:`~google.cloud.bigtable.instance.Instance` :param instance: (Optional) A Cloud Bigtable instance. The instance also owns a client for making gRPC requests to the Cloud Bigtable API. If not passed in, defaults to creating client with ``admin=True`` and using the ``timeout`` here for the ``timeout_seconds`` argument to the - :class:`Client ` + :class:`~google.cloud.bigtable.client.Client` constructor. The credentials for the client will be the implicit ones loaded from the environment. Then that client is used to retrieve all the instances @@ -196,7 +196,7 @@ def _handle_legacy_args(arguments_dict): if common_args: all_args = ', '.join(common_args) message = ('The HappyBase legacy arguments %s were used. These ' - 'arguments are unused by gcloud.' % (all_args,)) + 'arguments are unused by google-cloud.' % (all_args,)) _WARN(message) for arg_name in common_args: arguments_dict.pop(arg_name) @@ -208,8 +208,8 @@ def open(self): """Open the underlying transport to Cloud Bigtable. This method opens the underlying HTTP/2 gRPC connection using a - :class:`Client ` bound to the - :class:`Instance ` owned by + :class:`~google.cloud.bigtable.client.Client` bound to the + :class:`~google.cloud.bigtable.instance.Instance` owned by this connection. """ self._instance._client.start() @@ -218,8 +218,8 @@ def close(self): """Close the underlying transport to Cloud Bigtable. This method closes the underlying HTTP/2 gRPC connection using a - :class:`Client ` bound to the - :class:`Instance ` owned by + :class:`~google.cloud.bigtable.client.Client` bound to the + :class:`~google.cloud.bigtable.instance.Instance` owned by this connection. """ self._instance._client.stop() diff --git a/src/google/cloud/happybase/table.py b/src/google/cloud/happybase/table.py index 71f5fda..757a7c3 100644 --- a/src/google/cloud/happybase/table.py +++ b/src/google/cloud/happybase/table.py @@ -20,21 +20,21 @@ import six -from gcloud._helpers import _datetime_from_microseconds -from gcloud._helpers import _microseconds_from_datetime -from gcloud._helpers import _to_bytes -from gcloud.bigtable.column_family import GCRuleIntersection -from gcloud.bigtable.column_family import MaxAgeGCRule -from gcloud.bigtable.column_family import MaxVersionsGCRule -from gcloud.bigtable.row_filters import CellsColumnLimitFilter -from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter -from gcloud.bigtable.row_filters import FamilyNameRegexFilter -from gcloud.bigtable.row_filters import RowFilterChain -from gcloud.bigtable.row_filters import RowFilterUnion -from gcloud.bigtable.row_filters import RowKeyRegexFilter -from gcloud.bigtable.row_filters import TimestampRange -from gcloud.bigtable.row_filters import TimestampRangeFilter -from gcloud.bigtable.table import Table as _LowLevelTable +from google.cloud._helpers import _datetime_from_microseconds +from google.cloud._helpers import _microseconds_from_datetime +from google.cloud._helpers import _to_bytes +from google.cloud.bigtable.column_family import GCRuleIntersection +from google.cloud.bigtable.column_family import MaxAgeGCRule +from google.cloud.bigtable.column_family import MaxVersionsGCRule +from google.cloud.bigtable.row_filters import CellsColumnLimitFilter +from google.cloud.bigtable.row_filters import ColumnQualifierRegexFilter +from google.cloud.bigtable.row_filters import FamilyNameRegexFilter +from google.cloud.bigtable.row_filters import RowFilterChain +from google.cloud.bigtable.row_filters import RowFilterUnion +from google.cloud.bigtable.row_filters import RowKeyRegexFilter +from google.cloud.bigtable.row_filters import TimestampRange +from google.cloud.bigtable.row_filters import TimestampRangeFilter +from google.cloud.bigtable.table import Table as _LowLevelTable from google.cloud.happybase.batch import _get_column_pairs from google.cloud.happybase.batch import _WAL_SENTINEL @@ -320,7 +320,7 @@ def scan(self, row_start=None, row_stop=None, row_prefix=None, used this as an HBase filter string. (See the `Thrift docs`_ for more details on those filters.) However, Google Cloud Bigtable doesn't support those filter strings so a - :class:`~gcloud.bigtable.row.RowFilter` should be used instead. + :class:`~google.cloud.bigtable.row.RowFilter` should be used instead. .. _Thrift docs: http://hbase.apache.org/0.94/book/thrift.html @@ -498,7 +498,7 @@ def batch(self, timestamp=None, batch_size=None, transaction=False, for Cloud Bigtable since it does not have a Write Ahead Log. - :rtype: :class:`Batch ` + :rtype: :class:`~google.cloud.bigtable.happybase.batch.Batch` :returns: A batch bound to this table. """ return Batch(self, timestamp=timestamp, batch_size=batch_size, @@ -631,12 +631,13 @@ def _gc_rule_to_dict(gc_rule): Only does this if the garbage collection rule is: - * :class:`gcloud.bigtable.column_family.MaxAgeGCRule` - * :class:`gcloud.bigtable.column_family.MaxVersionsGCRule` - * Composite :class:`gcloud.bigtable.column_family.GCRuleIntersection` + * :class:`~google.cloud.bigtable.column_family.MaxAgeGCRule` + * :class:`~google.cloud.bigtable.column_family.MaxVersionsGCRule` + * Composite + :class:`~google.cloud.bigtable.column_family.GCRuleIntersection` with two rules, one each of type - :class:`gcloud.bigtable.column_family.MaxAgeGCRule` and - :class:`gcloud.bigtable.column_family.MaxVersionsGCRule` + :class:`~google.cloud.bigtable.column_family.MaxAgeGCRule` and + :class:`~google.cloud.bigtable.column_family.MaxVersionsGCRule` Otherwise, just returns the input without change. @@ -646,7 +647,7 @@ def _gc_rule_to_dict(gc_rule): (if possible). :rtype: dict or - :class:`gcloud.bigtable.column_family.GarbageCollectionRule` + :class:`~google.cloud.bigtable.column_family.GarbageCollectionRule` :returns: The converted garbage collection rule. """ result = gc_rule @@ -693,7 +694,7 @@ def _string_successor(str_val): Determines shortest string that sorts after the given string when compared using regular string comparison semantics. - Modeled after implementation in ``gcloud-golang``. + Modeled after implementation in ``google-cloud-go``. Increments the last byte that is smaller than ``0xFF``, and drops everything after it. If the string only contains ``0xFF`` bytes, @@ -733,7 +734,7 @@ def _convert_to_time_range(timestamp=None): epoch). Intended to be used as the end of an HBase time range, which is exclusive. - :rtype: :class:`gcloud.bigtable.row.TimestampRange`, + :rtype: :class:`~google.cloud.bigtable.row.TimestampRange`, :data:`NoneType ` :returns: The timestamp range corresponding to the passed in ``timestamp``. @@ -751,7 +752,7 @@ def _cells_to_pairs(cells, include_timestamp=False): For example:: >>> import datetime - >>> from gcloud.bigtable.row_data import Cell + >>> from google.cloud.bigtable.row_data import Cell >>> cell1 = Cell(b'val1', datetime.datetime.utcnow()) >>> cell2 = Cell(b'val2', datetime.datetime.utcnow()) >>> _cells_to_pairs([cell1, cell2]) @@ -760,8 +761,8 @@ def _cells_to_pairs(cells, include_timestamp=False): [(b'val1', 1456361486255), (b'val2', 1456361491927)] :type cells: list - :param cells: List of :class:`gcloud.bigtable.row_data.Cell` returned - from a read request. + :param cells: List of :class:`~google.cloud.bigtable.row_data.Cell` + returned from a read request. :type include_timestamp: bool :param include_timestamp: Flag to indicate if cell timestamps should be @@ -793,7 +794,7 @@ def _partial_row_to_dict(partial_row_data, include_timestamp=False): For example:: >>> import datetime - >>> from gcloud.bigtable.row_data import Cell, PartialRowData + >>> from google.cloud.bigtable.row_data import Cell, PartialRowData >>> cell1 = Cell(b'val1', datetime.datetime.utcnow()) >>> cell2 = Cell(b'val2', datetime.datetime.utcnow()) >>> row_data = PartialRowData(b'row-key') @@ -845,7 +846,7 @@ def _filter_chain_helper(column=None, versions=None, timestamp=None, :type filters: list :param filters: (Optional) List of existing filters to be extended. - :rtype: :class:`RowFilter ` + :rtype: :class:`~google.cloud.bigtable.row.RowFilter` :returns: The chained filter created, or just a single filter if only one was needed. :raises: :class:`ValueError ` if there are no @@ -888,7 +889,7 @@ def _scan_filter_helper(row_start, row_stop, row_prefix, columns, if legacy_args: legacy_args = ', '.join(legacy_args) message = ('The HappyBase legacy arguments %s were used. These ' - 'arguments are unused by gcloud.' % (legacy_args,)) + 'arguments are unused by google-cloud.' % (legacy_args,)) _WARN(message) if kwargs: raise TypeError('Received unexpected arguments', kwargs.keys()) @@ -906,7 +907,7 @@ def _scan_filter_helper(row_start, row_stop, row_prefix, columns, if isinstance(filter_, six.string_types): raise TypeError('Specifying filters as a string is not supported ' 'by Cloud Bigtable. Use a ' - 'gcloud.bigtable.row.RowFilter instead.') + 'google.cloud.bigtable.row.RowFilter instead.') elif filter_ is not None: filters.append(filter_) @@ -929,7 +930,7 @@ def _columns_filter_helper(columns): * an entire column family: ``fam`` or ``fam:`` * a single column: ``fam:col`` - :rtype: :class:`RowFilter ` + :rtype: :class:`~google.cloud.bigtable.row.RowFilter` :returns: The union filter created containing all of the matched columns. :raises: :class:`ValueError ` if there are no filters to union. @@ -960,7 +961,7 @@ def _row_keys_filter_helper(row_keys): :type row_keys: list :param row_keys: Iterable containing row keys (as strings). - :rtype: :class:`RowFilter ` + :rtype: :class:`~google.cloud.bigtable.row.RowFilter` :returns: The union filter created containing all of the row keys. :raises: :class:`ValueError ` if there are no filters to union. diff --git a/system_tests/happybase.py b/system_tests/happybase.py index f15ebec..787300b 100644 --- a/system_tests/happybase.py +++ b/system_tests/happybase.py @@ -18,10 +18,10 @@ import unittest -from gcloud import _helpers -from gcloud.bigtable import client as client_mod -from gcloud.bigtable.happybase.connection import Connection -from gcloud.environment_vars import TESTS_PROJECT +from google.cloud import _helpers +from google.cloud.bigtable import client as client_mod +from google.cloud.bigtable.happybase.connection import Connection +from google.cloud.environment_vars import TESTS_PROJECT from retry import RetryResult from system_test_utils import unique_resource_id @@ -66,7 +66,7 @@ class Config(object): def _operation_wait(operation, max_attempts=5): """Wait until an operation has completed. - :type operation: :class:`gcloud.bigtable.instance.Operation` + :type operation: :class:`~google.cloud.bigtable.instance.Operation` :param operation: Operation that has not finished. :type max_attempts: int diff --git a/system_tests/local_test_setup.sample b/system_tests/local_test_setup.sample index 771638c..45ac9d6 100644 --- a/system_tests/local_test_setup.sample +++ b/system_tests/local_test_setup.sample @@ -1,4 +1,3 @@ export GOOGLE_APPLICATION_CREDENTIALS="app_credentials.json.sample" -export GCLOUD_TESTS_PROJECT_ID="my-project" -export GCLOUD_REMOTE_FOR_LINT="upstream" -export GCLOUD_BRANCH_FOR_LINT="master" +export GOOGLE_CLOUD_REMOTE_FOR_LINT="upstream" +export GOOGLE_CLOUD_BRANCH_FOR_LINT="master" diff --git a/system_tests/run_emulator.py b/system_tests/run_emulator.py index 46edbcb..8efed8a 100644 --- a/system_tests/run_emulator.py +++ b/system_tests/run_emulator.py @@ -25,9 +25,9 @@ import psutil -from gcloud.environment_vars import GCD_DATASET -from gcloud.environment_vars import GCD_HOST -from gcloud.environment_vars import PUBSUB_EMULATOR +from google.cloud.environment_vars import GCD_DATASET +from google.cloud.environment_vars import GCD_HOST +from google.cloud.environment_vars import PUBSUB_EMULATOR from run_system_test import run_module_tests @@ -46,7 +46,7 @@ def get_parser(): :returns: The parser for this script. """ parser = argparse.ArgumentParser( - description='Run GCloud system tests against local emulator.') + description='Run Google Cloud system tests against local emulator.') parser.add_argument('--package', dest='package', choices=('datastore', 'pubsub'), default='datastore', help='Package to be tested.') diff --git a/system_tests/run_system_test.py b/system_tests/run_system_test.py index ad832de..2ee4235 100644 --- a/system_tests/run_system_test.py +++ b/system_tests/run_system_test.py @@ -31,7 +31,7 @@ class FailedSystemTestModule(Exception): def get_parser(): parser = argparse.ArgumentParser( - description='GCloud test runner against actual project.') + description='Google Cloud test runner against actual project.') parser.add_argument('--package', dest='package', choices=TEST_MODULES.keys(), default='datastore', help='Package to be tested.') diff --git a/system_tests/system_test_utils.py b/system_tests/system_test_utils.py index 1529872..6f15bd3 100644 --- a/system_tests/system_test_utils.py +++ b/system_tests/system_test_utils.py @@ -17,8 +17,8 @@ import sys import time -from gcloud.environment_vars import CREDENTIALS as TEST_CREDENTIALS -from gcloud.environment_vars import TESTS_PROJECT +from google.cloud.environment_vars import CREDENTIALS as TEST_CREDENTIALS +from google.cloud.environment_vars import TESTS_PROJECT # From shell environ. May be None. diff --git a/tox.ini b/tox.ini index 06c607b..35756df 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ deps = pytest covercmd = py.test --quiet \ - --cov=google.cloud \ + --cov=google.cloud.happybase \ --cov=unit_tests \ --cov-config {toxinidir}/.coveragerc \ unit_tests @@ -74,7 +74,7 @@ basepython = python2.7 commands = python {toxinidir}/system_tests/attempt_system_tests.py -passenv = GOOGLE_* GCLOUD_* TRAVIS* encrypted_* +passenv = GOOGLE_* TRAVIS* encrypted_* [testenv:system-tests3] basepython = diff --git a/unit_tests/test_batch.py b/unit_tests/test_batch.py index 61b1d53..f2d4f57 100644 --- a/unit_tests/test_batch.py +++ b/unit_tests/test_batch.py @@ -45,8 +45,8 @@ def test_constructor_defaults(self): self.assertEqual(batch._mutation_count, 0) def test_constructor_explicit(self): - from gcloud._helpers import _datetime_from_microseconds - from gcloud.bigtable.row_filters import TimestampRange + from google.cloud._helpers import _datetime_from_microseconds + from google.cloud.bigtable.row_filters import TimestampRange table = object() timestamp = 144185290431 @@ -68,7 +68,7 @@ def test_constructor_explicit(self): self.assertEqual(batch._mutation_count, 0) def test_constructor_with_non_default_wal(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import batch as MUT warned = [] @@ -205,7 +205,7 @@ def test__get_row_create_new(self): self.assertEqual(batch._row_map, {row_key: mock_row}) def test_put_bad_wal(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import batch as MUT warned = [] @@ -321,7 +321,7 @@ def test__delete_columns_w_time_and_col_fam(self): self._delete_columns_test_helper(time_range=time_range) def test_delete_bad_wal(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import batch as MUT warned = [] diff --git a/unit_tests/test_connection.py b/unit_tests/test_connection.py index 5bbcf7e..219d46e 100644 --- a/unit_tests/test_connection.py +++ b/unit_tests/test_connection.py @@ -26,7 +26,7 @@ def _callFUT(self, timeout=None): def _helper(self, timeout=None, instances=(), failed_locations=()): from functools import partial - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import connection as MUT client_with_instances = partial( @@ -101,7 +101,7 @@ def test_constructor_no_autoconnect(self): self.assertEqual(connection.table_prefix_separator, '_') def test_constructor_missing_instance(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import connection as MUT instance = _Instance() @@ -143,7 +143,7 @@ def test_constructor_with_unknown_argument(self): self._makeOne(instance=instance, unknown='foo') def test_constructor_with_legacy_args(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import connection as MUT warned = [] @@ -281,7 +281,7 @@ def test_table_factory_with_ignored_prefix(self): self._table_factory_prefix_helper(use_prefix=False) def test_tables(self): - from gcloud.bigtable.table import Table + from google.cloud.bigtable.table import Table table_name1 = 'table-name1' table_name2 = 'table-name2' @@ -294,7 +294,7 @@ def test_tables(self): self.assertEqual(result, [table_name1, table_name2]) def test_tables_with_prefix(self): - from gcloud.bigtable.table import Table + from google.cloud.bigtable.table import Table table_prefix = 'prefix' table_prefix_separator = '<>' @@ -315,7 +315,7 @@ def test_tables_with_prefix(self): def test_create_table(self): import operator - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import connection as MUT instance = _Instance() # Avoid implicit environ check. @@ -395,7 +395,7 @@ def test_create_table_bad_value(self): connection.create_table(name, families) def _create_table_error_helper(self, err_val, err_type): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import connection as MUT instance = _Instance() # Avoid implicit environ check. @@ -444,7 +444,7 @@ def test_create_table_other_error(self): self._create_table_error_helper(RuntimeError, RuntimeError) def _delete_table_helper(self, disable=False): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import connection as MUT instance = _Instance() # Avoid implicit environ check. @@ -471,7 +471,7 @@ def test_delete_table(self): self._delete_table_helper() def test_delete_table_disable(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import connection as MUT warned = [] @@ -485,7 +485,7 @@ def mock_warn(msg): self.assertEqual(warned, [MUT._DISABLE_DELETE_MSG]) def test_enable_table(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import connection as MUT instance = _Instance() # Avoid implicit environ check. @@ -504,7 +504,7 @@ def mock_warn(msg): self.assertEqual(warned, [MUT._ENABLE_TMPL % (name,)]) def test_disable_table(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import connection as MUT instance = _Instance() # Avoid implicit environ check. @@ -523,7 +523,7 @@ def mock_warn(msg): self.assertEqual(warned, [MUT._DISABLE_TMPL % (name,)]) def test_is_table_enabled(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import connection as MUT instance = _Instance() # Avoid implicit environ check. @@ -543,7 +543,7 @@ def mock_warn(msg): self.assertEqual(warned, [MUT._IS_ENABLED_TMPL % (name,)]) def test_compact_table(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import connection as MUT instance = _Instance() # Avoid implicit environ check. @@ -579,7 +579,7 @@ def test_null(self): self.assertEqual(result, None) def test_dictionary_bad_key(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import connection as MUT warned = [] @@ -596,7 +596,7 @@ def mock_warn(msg): self.assertIn('badkey', warned[0]) def test_dictionary_versions_key(self): - from gcloud.bigtable.column_family import MaxVersionsGCRule + from google.cloud.bigtable.column_family import MaxVersionsGCRule versions = 42 option = {'max_versions': versions} @@ -607,7 +607,7 @@ def test_dictionary_versions_key(self): def test_dictionary_ttl_key(self): import datetime - from gcloud.bigtable.column_family import MaxAgeGCRule + from google.cloud.bigtable.column_family import MaxAgeGCRule time_to_live = 24 * 60 * 60 max_age = datetime.timedelta(days=1) @@ -619,9 +619,9 @@ def test_dictionary_ttl_key(self): def test_dictionary_both_keys(self): import datetime - from gcloud.bigtable.column_family import GCRuleIntersection - from gcloud.bigtable.column_family import MaxAgeGCRule - from gcloud.bigtable.column_family import MaxVersionsGCRule + from google.cloud.bigtable.column_family import GCRuleIntersection + from google.cloud.bigtable.column_family import MaxAgeGCRule + from google.cloud.bigtable.column_family import MaxVersionsGCRule versions = 42 time_to_live = 24 * 60 * 60 diff --git a/unit_tests/test_pool.py b/unit_tests/test_pool.py index 0f8243c..a7da569 100644 --- a/unit_tests/test_pool.py +++ b/unit_tests/test_pool.py @@ -64,7 +64,7 @@ def test_constructor_passes_kwargs(self): table_prefix_separator) def test_constructor_ignores_autoconnect(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase.connection import Connection from google.cloud.happybase import pool as MUT @@ -98,7 +98,7 @@ def open(self): self.assertFalse(connection._open_called) def test_constructor_infers_instance(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase.connection import Connection from google.cloud.happybase import pool as MUT @@ -137,7 +137,7 @@ def test_constructor_non_positive_size(self): self._makeOne(size) def _makeOneWithMockQueue(self, queue_return): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import pool as MUT # We are going to use a fake queue, so we don't want any connections diff --git a/unit_tests/test_table.py b/unit_tests/test_table.py index f64c168..a59a863 100644 --- a/unit_tests/test_table.py +++ b/unit_tests/test_table.py @@ -47,7 +47,7 @@ def _makeOne(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) def test_constructor(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT name = 'table-name' @@ -79,7 +79,7 @@ def test_constructor_null_connection(self): self.assertEqual(table._low_level_table, None) def test_families(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT name = 'table-name' @@ -122,7 +122,7 @@ def test_regions(self): table.regions() def test_row_empty_row(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT name = 'table-name' @@ -161,7 +161,7 @@ def mock_filter_chain_helper(**kwargs): self.assertEqual(mock_filters, [expected_kwargs]) def test_row_with_columns(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT name = 'table-name' @@ -209,9 +209,9 @@ def mock_filter_chain_helper(**kwargs): self.assertEqual(mock_filters, [expected_kwargs]) def test_row_with_results(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT - from gcloud.bigtable.row_data import PartialRowData + from google.cloud.bigtable.row_data import PartialRowData row_key = 'row-key' name = 'table-name' @@ -274,7 +274,7 @@ def test_rows_empty_row(self): self.assertEqual(result, []) def test_rows_with_columns(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT name = 'table-name' @@ -334,9 +334,9 @@ def mock_filter_chain_helper(**kwargs): self.assertEqual(mock_filters, [expected_kwargs]) def test_rows_with_results(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT - from gcloud.bigtable.row_data import PartialRowData + from google.cloud.bigtable.row_data import PartialRowData row_key1 = 'row-key1' row_key2 = 'row-key2' @@ -407,7 +407,7 @@ def mock_cells_to_pairs(*args, **kwargs): [((fake_cells,), to_pairs_kwargs)]) def test_cells_empty_row(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT name = 'table-name' @@ -446,9 +446,9 @@ def mock_filter_chain_helper(**kwargs): self.assertEqual(mock_filters, [expected_kwargs]) def test_cells_with_results(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT - from gcloud.bigtable.row_data import PartialRowData + from google.cloud.bigtable.row_data import PartialRowData row_key = 'row-key' name = 'table-name' @@ -508,7 +508,7 @@ def mock_cells_to_pairs(*args, **kwargs): [((fake_cells,), to_pairs_kwargs)]) def test_scan_with_batch_size(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT warned = [] @@ -529,7 +529,7 @@ def mock_warn(msg): self.assertIn('batch_size', warned[0]) def test_scan_with_scan_batching(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT warned = [] @@ -550,7 +550,7 @@ def mock_warn(msg): self.assertIn('scan_batching', warned[0]) def test_scan_with_sorted_columns(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT warned = [] @@ -596,7 +596,7 @@ def _scan_test_helper(self, row_limits=(None, None), row_prefix=None, include_timestamp=False, limit=None, rr_result=None, expected_result=None): import types - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT name = 'table-name' @@ -693,7 +693,7 @@ def test_scan_with_no_results(self): self._scan_test_helper(timestamp=timestamp, limit=limit) def test_scan_with_results(self): - from gcloud.bigtable.row_data import PartialRowData + from google.cloud.bigtable.row_data import PartialRowData row_key1 = 'row-key1' row1 = PartialRowData(row_key1) @@ -706,7 +706,7 @@ def test_scan_with_results(self): expected_result=expected_result) def test_put(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT from google.cloud.happybase.table import _WAL_SENTINEL @@ -746,7 +746,7 @@ def make_batch(*args, **kwargs): self.assertEqual(batch.delete_args, []) def test_delete(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT from google.cloud.happybase.table import _WAL_SENTINEL @@ -786,7 +786,7 @@ def make_batch(*args, **kwargs): self.assertEqual(batch.delete_args, [(row, columns)]) def test_batch(self): - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT name = 'table-name' @@ -890,7 +890,7 @@ def _counter_inc_helper(self, row, column, value, commit_result): def test_counter_set(self): import struct - from gcloud._testing import _Monkey + from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT name = 'table-name' @@ -1025,7 +1025,7 @@ def test_with_null(self): self.assertEqual(result, {}) def test_with_max_versions(self): - from gcloud.bigtable.column_family import MaxVersionsGCRule + from google.cloud.bigtable.column_family import MaxVersionsGCRule max_versions = 2 gc_rule = MaxVersionsGCRule(max_versions) @@ -1035,7 +1035,7 @@ def test_with_max_versions(self): def test_with_max_age(self): import datetime - from gcloud.bigtable.column_family import MaxAgeGCRule + from google.cloud.bigtable.column_family import MaxAgeGCRule time_to_live = 101 max_age = datetime.timedelta(seconds=time_to_live) @@ -1050,22 +1050,22 @@ def test_with_non_gc_rule(self): self.assertTrue(result is gc_rule) def test_with_gc_rule_union(self): - from gcloud.bigtable.column_family import GCRuleUnion + from google.cloud.bigtable.column_family import GCRuleUnion gc_rule = GCRuleUnion(rules=[]) result = self._callFUT(gc_rule) self.assertTrue(result is gc_rule) def test_with_intersection_other_than_two(self): - from gcloud.bigtable.column_family import GCRuleIntersection + from google.cloud.bigtable.column_family import GCRuleIntersection gc_rule = GCRuleIntersection(rules=[]) result = self._callFUT(gc_rule) self.assertTrue(result is gc_rule) def test_with_intersection_two_max_num_versions(self): - from gcloud.bigtable.column_family import GCRuleIntersection - from gcloud.bigtable.column_family import MaxVersionsGCRule + from google.cloud.bigtable.column_family import GCRuleIntersection + from google.cloud.bigtable.column_family import MaxVersionsGCRule rule1 = MaxVersionsGCRule(1) rule2 = MaxVersionsGCRule(2) @@ -1075,9 +1075,9 @@ def test_with_intersection_two_max_num_versions(self): def test_with_intersection_two_rules(self): import datetime - from gcloud.bigtable.column_family import GCRuleIntersection - from gcloud.bigtable.column_family import MaxAgeGCRule - from gcloud.bigtable.column_family import MaxVersionsGCRule + from google.cloud.bigtable.column_family import GCRuleIntersection + from google.cloud.bigtable.column_family import MaxAgeGCRule + from google.cloud.bigtable.column_family import MaxVersionsGCRule time_to_live = 101 max_age = datetime.timedelta(seconds=time_to_live) @@ -1093,7 +1093,7 @@ def test_with_intersection_two_rules(self): self.assertEqual(result, expected_result) def test_with_intersection_two_nested_rules(self): - from gcloud.bigtable.column_family import GCRuleIntersection + from google.cloud.bigtable.column_family import GCRuleIntersection rule1 = GCRuleIntersection(rules=[]) rule2 = GCRuleIntersection(rules=[]) @@ -1142,8 +1142,8 @@ def test_invalid_type(self): self._callFUT(timestamp=timestamp) def test_success(self): - from gcloud._helpers import _datetime_from_microseconds - from gcloud.bigtable.row_filters import TimestampRange + from google.cloud._helpers import _datetime_from_microseconds + from google.cloud.bigtable.row_filters import TimestampRange timestamp = 1441928298571 ts_dt = _datetime_from_microseconds(1000 * timestamp) @@ -1160,7 +1160,7 @@ def _callFUT(self, *args, **kwargs): return _cells_to_pairs(*args, **kwargs) def test_without_timestamp(self): - from gcloud.bigtable.row_data import Cell + from google.cloud.bigtable.row_data import Cell value1 = 'foo' cell1 = Cell(value=value1, timestamp=None) @@ -1171,8 +1171,8 @@ def test_without_timestamp(self): self.assertEqual(result, [value1, value2]) def test_with_timestamp(self): - from gcloud._helpers import _datetime_from_microseconds - from gcloud.bigtable.row_data import Cell + from google.cloud._helpers import _datetime_from_microseconds + from google.cloud.bigtable.row_data import Cell value1 = 'foo' ts1_millis = 1221934570148 @@ -1197,8 +1197,8 @@ def _callFUT(self, partial_row_data, include_timestamp=False): include_timestamp=include_timestamp) def test_without_timestamp(self): - from gcloud.bigtable.row_data import Cell - from gcloud.bigtable.row_data import PartialRowData + from google.cloud.bigtable.row_data import Cell + from google.cloud.bigtable.row_data import PartialRowData row_data = PartialRowData(b'row-key') val1 = b'hi-im-bytes' @@ -1215,9 +1215,9 @@ def test_without_timestamp(self): self.assertEqual(result, expected_result) def test_with_timestamp(self): - from gcloud._helpers import _datetime_from_microseconds - from gcloud.bigtable.row_data import Cell - from gcloud.bigtable.row_data import PartialRowData + from google.cloud._helpers import _datetime_from_microseconds + from google.cloud.bigtable.row_data import Cell + from google.cloud.bigtable.row_data import PartialRowData row_data = PartialRowData(b'row-key') val1 = b'hi-im-bytes' @@ -1249,7 +1249,7 @@ def test_no_filters(self): self._callFUT() def test_single_filter(self): - from gcloud.bigtable.row_filters import CellsColumnLimitFilter + from google.cloud.bigtable.row_filters import CellsColumnLimitFilter versions = 1337 result = self._callFUT(versions=versions) @@ -1259,7 +1259,7 @@ def test_single_filter(self): self.assertEqual(result.num_cells, versions) def test_existing_filters(self): - from gcloud.bigtable.row_filters import CellsColumnLimitFilter + from google.cloud.bigtable.row_filters import CellsColumnLimitFilter filters = [] versions = 1337 @@ -1274,9 +1274,10 @@ def test_existing_filters(self): def _column_helper(self, num_filters, versions=None, timestamp=None, column=None, col_fam=None, qual=None): - from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter - from gcloud.bigtable.row_filters import FamilyNameRegexFilter - from gcloud.bigtable.row_filters import RowFilterChain + from google.cloud.bigtable.row_filters import ( + ColumnQualifierRegexFilter) + from google.cloud.bigtable.row_filters import FamilyNameRegexFilter + from google.cloud.bigtable.row_filters import RowFilterChain if col_fam is None: col_fam = 'cf1' @@ -1312,7 +1313,7 @@ def test_column_unicode(self): col_fam=u'cfU', qual=u'qualN') def test_with_versions(self): - from gcloud.bigtable.row_filters import CellsColumnLimitFilter + from google.cloud.bigtable.row_filters import CellsColumnLimitFilter versions = 11 result = self._column_helper(num_filters=3, versions=versions) @@ -1324,9 +1325,9 @@ def test_with_versions(self): self.assertEqual(version_filter.num_cells, versions) def test_with_timestamp(self): - from gcloud._helpers import _datetime_from_microseconds - from gcloud.bigtable.row_filters import TimestampRange - from gcloud.bigtable.row_filters import TimestampRangeFilter + from google.cloud._helpers import _datetime_from_microseconds + from google.cloud.bigtable.row_filters import TimestampRange + from google.cloud.bigtable.row_filters import TimestampRangeFilter timestamp = 1441928298571 result = self._column_helper(num_filters=3, timestamp=timestamp) @@ -1360,7 +1361,7 @@ def test_no_columns(self): self._callFUT(columns) def test_single_column(self): - from gcloud.bigtable.row_filters import FamilyNameRegexFilter + from google.cloud.bigtable.row_filters import FamilyNameRegexFilter col_fam = 'cf1' columns = [col_fam] @@ -1369,10 +1370,11 @@ def test_single_column(self): self.assertEqual(result, expected_result) def test_column_and_column_families(self): - from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter - from gcloud.bigtable.row_filters import FamilyNameRegexFilter - from gcloud.bigtable.row_filters import RowFilterChain - from gcloud.bigtable.row_filters import RowFilterUnion + from google.cloud.bigtable.row_filters import ( + ColumnQualifierRegexFilter) + from google.cloud.bigtable.row_filters import FamilyNameRegexFilter + from google.cloud.bigtable.row_filters import RowFilterChain + from google.cloud.bigtable.row_filters import RowFilterUnion col_fam1 = 'cf1' col_fam2 = 'cf2' @@ -1408,7 +1410,7 @@ def test_no_rows(self): self._callFUT(row_keys) def test_single_row(self): - from gcloud.bigtable.row_filters import RowKeyRegexFilter + from google.cloud.bigtable.row_filters import RowKeyRegexFilter row_key = b'row-key' row_keys = [row_key] @@ -1417,8 +1419,8 @@ def test_single_row(self): self.assertEqual(result, expected_result) def test_many_rows(self): - from gcloud.bigtable.row_filters import RowFilterUnion - from gcloud.bigtable.row_filters import RowKeyRegexFilter + from google.cloud.bigtable.row_filters import RowFilterUnion + from google.cloud.bigtable.row_filters import RowKeyRegexFilter row_key1 = b'row-key1' row_key2 = b'row-key2' From 5cf24434ec5ced7cb4cd5a651aadb8b0fc42fcef Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Thu, 29 Sep 2016 09:58:06 -0700 Subject: [PATCH 3/5] Adding universal wheel to setup config. --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index e69de29..2a9acf1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 From 7369de1aa4beee12bdc93297e053d8354caaf420 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Thu, 29 Sep 2016 10:32:41 -0700 Subject: [PATCH 4/5] Upgrading system tests for new dependencies. In order to get passing system tests, Client.start() and Client.stop() had to stop being used. Also add a newline (per feedback) in setup.py. --- setup.py | 1 + src/google/cloud/happybase/connection.py | 22 ++---------- src/google/cloud/happybase/pool.py | 3 +- system_tests/happybase.py | 35 +++++++++--------- system_tests/system_test_utils.py | 23 ++++-------- unit_tests/test_connection.py | 46 ------------------------ unit_tests/test_pool.py | 7 +--- 7 files changed, 30 insertions(+), 107 deletions(-) diff --git a/setup.py b/setup.py index 05c4baa..5e0f3ca 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ ] SETUP_BASE.pop('url') + setup( name='google-cloud-happybase', version='0.19.0', diff --git a/src/google/cloud/happybase/connection.py b/src/google/cloud/happybase/connection.py index 53dd8ad..9d7c34f 100644 --- a/src/google/cloud/happybase/connection.py +++ b/src/google/cloud/happybase/connection.py @@ -85,11 +85,7 @@ def _get_instance(timeout=None): if timeout is not None: client_kwargs['timeout_seconds'] = timeout / 1000.0 client = Client(**client_kwargs) - try: - client.start() - instances, failed_locations = client.list_instances() - finally: - client.stop() + instances, failed_locations = client.list_instances() if len(failed_locations) != 0: raise ValueError('Determining instance via ListInstances encountered ' @@ -207,26 +203,14 @@ def _handle_legacy_args(arguments_dict): def open(self): """Open the underlying transport to Cloud Bigtable. - This method opens the underlying HTTP/2 gRPC connection using a - :class:`~google.cloud.bigtable.client.Client` bound to the - :class:`~google.cloud.bigtable.instance.Instance` owned by - this connection. + This method does nothing and is provided for compatibility. """ - self._instance._client.start() def close(self): """Close the underlying transport to Cloud Bigtable. - This method closes the underlying HTTP/2 gRPC connection using a - :class:`~google.cloud.bigtable.client.Client` bound to the - :class:`~google.cloud.bigtable.instance.Instance` owned by - this connection. + This method does nothing and is provided for compatibility. """ - self._instance._client.stop() - - def __del__(self): - if self._instance is not None: - self.close() def _table_name(self, name): """Construct a table name by optionally adding a table name prefix. diff --git a/src/google/cloud/happybase/pool.py b/src/google/cloud/happybase/pool.py index 8c84896..332a678 100644 --- a/src/google/cloud/happybase/pool.py +++ b/src/google/cloud/happybase/pool.py @@ -142,8 +142,7 @@ def connection(self, timeout=None): with self._lock: self._thread_connections.current = connection - # This is a no-op for connections that have already been opened - # since they just call Client.start(). + # This is a no-op. connection.open() yield connection diff --git a/system_tests/happybase.py b/system_tests/happybase.py index 787300b..f82fc29 100644 --- a/system_tests/happybase.py +++ b/system_tests/happybase.py @@ -18,10 +18,8 @@ import unittest -from google.cloud import _helpers from google.cloud.bigtable import client as client_mod -from google.cloud.bigtable.happybase.connection import Connection -from google.cloud.environment_vars import TESTS_PROJECT +from google.cloud.happybase.connection import Connection from retry import RetryResult from system_test_utils import unique_resource_id @@ -29,7 +27,6 @@ _PACK_I64 = struct.Struct('>q').pack _FIRST_ELT = operator.itemgetter(0) -_helpers.PROJECT = TESTS_PROJECT LOCATION_ID = 'us-central1-c' # NOTE: Avoid using the same name as in bigtable.py INSTANCE_ID = 'gcl-hb' + unique_resource_id('-') @@ -63,33 +60,37 @@ class Config(object): TABLE = None -def _operation_wait(operation, max_attempts=5): +def _operation_complete(result): + """Identity function. + + Meant to return a boolean, but the value passed in is + already equal to that value. + """ + return result + + +def _wait_until_complete(operation, max_attempts=5): """Wait until an operation has completed. - :type operation: :class:`~google.cloud.bigtable.instance.Operation` - :param operation: Operation that has not finished. + :type operation: :class:`google.cloud.operation.Operation` + :param operation: Operation that has not completed. :type max_attempts: int :param max_attempts: (Optional) The maximum number of times to check if - the operation has finished. Defaults to 5. + the operation has completed. Defaults to 5. :rtype: bool - :returns: Boolean indicating if the operation finished. + :returns: Boolean indicating if the operation is complete. """ - - def _operation_finished(result): - return result - - retry = RetryResult(_operation_finished, max_tries=max_attempts) - return retry(operation.finished)() + retry = RetryResult(_operation_complete, max_tries=max_attempts) + return retry(operation.poll)() def set_connection(): client = client_mod.Client(admin=True) instance = client.instance(INSTANCE_ID, LOCATION_ID) - client.start() operation = instance.create() - if not _operation_wait(operation): + if not _wait_until_complete(operation): raise RuntimeError('Instance creation exceed 5 seconds.') Config.CONNECTION = Connection(instance=instance) diff --git a/system_tests/system_test_utils.py b/system_tests/system_test_utils.py index 6f15bd3..1a3cd8c 100644 --- a/system_tests/system_test_utils.py +++ b/system_tests/system_test_utils.py @@ -18,11 +18,9 @@ import time from google.cloud.environment_vars import CREDENTIALS as TEST_CREDENTIALS -from google.cloud.environment_vars import TESTS_PROJECT # From shell environ. May be None. -PROJECT_ID = os.getenv(TESTS_PROJECT) CREDENTIALS = os.getenv(TEST_CREDENTIALS) ENVIRON_ERROR_MSG = """\ @@ -44,24 +42,15 @@ def create_scoped_required(): def check_environ(): - missing = [] - extra = '' - - if PROJECT_ID is None: - missing.append(TESTS_PROJECT) - + err_msg = None if CREDENTIALS is None: - missing.append(TEST_CREDENTIALS) + err_msg = '\nMissing variables: ' + TEST_CREDENTIALS elif not os.path.isfile(CREDENTIALS): - extra = '\nThe %s path %r is not a file.' % (TEST_CREDENTIALS, - CREDENTIALS) + err_msg = '\nThe %s path %r is not a file.' % (TEST_CREDENTIALS, + CREDENTIALS) - if missing or extra: - msg = ENVIRON_ERROR_MSG - if missing: - msg += '\nMissing variables: ' + ', '.join(missing) - if extra: - msg += extra + if err_msg is not None: + msg = ENVIRON_ERROR_MSG + err_msg print(msg, file=sys.stderr) sys.exit(1) diff --git a/unit_tests/test_connection.py b/unit_tests/test_connection.py index 219d46e..1cb5f86 100644 --- a/unit_tests/test_connection.py +++ b/unit_tests/test_connection.py @@ -44,8 +44,6 @@ def _helper(self, timeout=None, instances=(), failed_locations=()): if timeout is not None: expected_kwargs['timeout_seconds'] = timeout / 1000.0 self.assertEqual(client.kwargs, expected_kwargs) - self.assertEqual(client.start_calls, 1) - self.assertEqual(client.stop_calls, 1) def test_default(self): instance = _Instance() @@ -83,10 +81,7 @@ def _makeOne(self, *args, **kwargs): def test_constructor_defaults(self): instance = _Instance() # Avoid implicit environ check. - self.assertEqual(instance._client.start_calls, 0) connection = self._makeOne(instance=instance) - self.assertEqual(instance._client.start_calls, 1) - self.assertEqual(instance._client.stop_calls, 0) self.assertEqual(connection._instance, instance) self.assertEqual(connection.table_prefix, None) @@ -95,8 +90,6 @@ def test_constructor_defaults(self): def test_constructor_no_autoconnect(self): instance = _Instance() # Avoid implicit environ check. connection = self._makeOne(autoconnect=False, instance=instance) - self.assertEqual(instance._client.start_calls, 0) - self.assertEqual(instance._client.stop_calls, 0) self.assertEqual(connection.table_prefix, None) self.assertEqual(connection.table_prefix_separator, '_') @@ -183,37 +176,6 @@ def test_constructor_non_string_prefix_separator(self): self._makeOne(autoconnect=False, table_prefix_separator=table_prefix_separator) - def test_open(self): - instance = _Instance() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, instance=instance) - self.assertEqual(instance._client.start_calls, 0) - connection.open() - self.assertEqual(instance._client.start_calls, 1) - self.assertEqual(instance._client.stop_calls, 0) - - def test_close(self): - instance = _Instance() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, instance=instance) - self.assertEqual(instance._client.stop_calls, 0) - connection.close() - self.assertEqual(instance._client.stop_calls, 1) - self.assertEqual(instance._client.start_calls, 0) - - def test___del__with_instance(self): - instance = _Instance() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, instance=instance) - self.assertEqual(instance._client.stop_calls, 0) - connection.__del__() - self.assertEqual(instance._client.stop_calls, 1) - - def test___del__no_instance(self): - instance = _Instance() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, instance=instance) - self.assertEqual(instance._client.stop_calls, 0) - del connection._instance - connection.__del__() - self.assertEqual(instance._client.stop_calls, 0) - def test__table_name_with_prefix_set(self): table_prefix = 'table-prefix' table_prefix_separator = '<>' @@ -655,14 +617,6 @@ def __init__(self, *args, **kwargs): self.failed_locations = kwargs.pop('failed_locations', []) self.args = args self.kwargs = kwargs - self.start_calls = 0 - self.stop_calls = 0 - - def start(self): - self.start_calls += 1 - - def stop(self): - self.stop_calls += 1 def list_instances(self): return self.instances, self.failed_locations diff --git a/unit_tests/test_pool.py b/unit_tests/test_pool.py index a7da569..579c761 100644 --- a/unit_tests/test_pool.py +++ b/unit_tests/test_pool.py @@ -215,12 +215,7 @@ def test_connection_with_current_cnxn(self): class _Client(object): - - def __init__(self): - self.stop_calls = 0 - - def stop(self): - self.stop_calls += 1 + pass class _Connection(object): From 6a675319750fd68f730bd0b2ae629fb38dcad264 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Thu, 29 Sep 2016 10:01:32 -0700 Subject: [PATCH 5/5] Bringing system test scripts up to date. Also removing run_emulator since it isn't used anymore. --- system_tests/attempt_system_tests.py | 76 ++++++++++- system_tests/retry.py | 8 +- system_tests/run_emulator.py | 182 --------------------------- tox.ini | 4 +- 4 files changed, 80 insertions(+), 190 deletions(-) delete mode 100644 system_tests/run_emulator.py diff --git a/system_tests/attempt_system_tests.py b/system_tests/attempt_system_tests.py index 2ed8ffe..b683438 100644 --- a/system_tests/attempt_system_tests.py +++ b/system_tests/attempt_system_tests.py @@ -19,13 +19,41 @@ If being run as part of a Travis build for a merged commit, the encrypted `key.json` file need be decrypted before running tests. + +Before encrypting: + +* Visit ``https://github.com/settings/tokens/new``, get a token, + and store the token in travis.token file. +* Visit + ``https://console.cloud.google.com/apis/credentials?project={PROJ}`` +* Click "Create credentials >> Service account key", select your + service account and create a JSON key (we'll call it ``key.json``) + +Encrypt the file via:: + +$ travis login --github-token=$(cat travis.token) +$ travis encrypt-file \ +> --repo=GoogleCloudPlatform/google-cloud-python-happybase \ +> key.json + +After running ``travis encrypt-file``, take note of the ``openssl`` command +printed out. In particular, it'll give the name of the key and +IV (initialization vector) environment variables added to the Travis +project. + +See: +https://docs.travis-ci.com/user/encrypting-files/ """ +from __future__ import print_function +import argparse import os import subprocess import sys +from google.cloud.environment_vars import CREDENTIALS + from run_system_test import FailedSystemTestModule from run_system_test import run_module_tests @@ -37,6 +65,9 @@ SCRIPTS_DIR = os.path.dirname(__file__) ROOT_DIR = os.path.abspath(os.path.join(SCRIPTS_DIR, '..')) ENCRYPTED_KEYFILE = os.path.join(ROOT_DIR, 'system_tests', 'key.json.enc') +ENCRYPTED_KEY_ENV = 'encrypted_INVALID_key' +ENCRYPTED_INIT_VECTOR_ENV = 'encrypted_INVALID_iv' +ALL_MODULES = object() # Sentinel for argparser def check_environment(): @@ -64,9 +95,9 @@ def decrypt_keyfile(): print('Running in Travis during merge, decrypting stored ' 'key file.') - encrypted_key = os.getenv('encrypted_a1b222e8c14d_key') - encrypted_iv = os.getenv('encrypted_a1b222e8c14d_iv') - out_file = os.getenv('GOOGLE_APPLICATION_CREDENTIALS') + encrypted_key = os.getenv(ENCRYPTED_KEY_ENV) + encrypted_iv = os.getenv(ENCRYPTED_INIT_VECTOR_ENV) + out_file = os.getenv(CREDENTIALS) # Convert encrypted key file into decrypted file to be used. subprocess.call([ 'openssl', 'aes-256-cbc', @@ -101,11 +132,47 @@ def prepare_to_run(): decrypt_keyfile() +def get_parser(): + """Get an argument parser to determine a list of packages.""" + parser = argparse.ArgumentParser( + description='google-cloud tests runner.') + help_msg = ('List of packages to be tested. ' + 'If left blank, tests all packages.') + parser.add_argument('packages', nargs='*', + default=ALL_MODULES, help=help_msg) + return parser + + +def get_modules(): + """Get the list of modules names to run system tests for.""" + parser = get_parser() + args = parser.parse_args() + if args.packages is ALL_MODULES: + result = list(MODULES) + else: + result = [] + invalid = [] + for package in args.packages: + if package in MODULES: + result.append(package) + else: + invalid.append(package) + + if invalid: + msg = 'No system test for packages: ' + ', '.join(invalid) + print(msg, file=sys.stderr) + sys.exit(1) + + return result + + def main(): """Run all the system tests if necessary.""" prepare_to_run() + failed_modules = 0 - for module in MODULES: + modules = get_modules() + for module in modules: try: run_module_tests(module) except FailedSystemTestModule: @@ -113,5 +180,6 @@ def main(): sys.exit(failed_modules) + if __name__ == '__main__': main() diff --git a/system_tests/retry.py b/system_tests/retry.py index f9ab2dd..d4694f6 100644 --- a/system_tests/retry.py +++ b/system_tests/retry.py @@ -27,6 +27,10 @@ def _retry_all(_): return True +class BackoffFailed(Exception): + """Retry w/ backoffs did not complete successfully.""" + + class RetryBase(object): """Base for retrying calling a decorated function w/ exponential backoff. @@ -148,7 +152,7 @@ def wrapped_function(*args, **kwargs): time.sleep(delay) tries += 1 - return to_wrap(*args, **kwargs) + raise BackoffFailed() return wrapped_function @@ -198,6 +202,6 @@ def wrapped_function(*args, **kwargs): time.sleep(delay) tries += 1 - return to_wrap(*args, **kwargs) + raise BackoffFailed() return wrapped_function diff --git a/system_tests/run_emulator.py b/system_tests/run_emulator.py deleted file mode 100644 index 8efed8a..0000000 --- a/system_tests/run_emulator.py +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Run system tests locally with the emulator. - -First makes system calls to spawn the emulator and get the local environment -variable needed for it. Then calls the system tests. -""" - - -import argparse -import os -import subprocess - -import psutil - -from google.cloud.environment_vars import GCD_DATASET -from google.cloud.environment_vars import GCD_HOST -from google.cloud.environment_vars import PUBSUB_EMULATOR -from run_system_test import run_module_tests - - -PACKAGE_INFO = { - 'datastore': (GCD_DATASET, GCD_HOST), - 'pubsub': (PUBSUB_EMULATOR,) -} -_DS_READY_LINE = '[datastore] INFO: Dev App Server is now running\n' -_PS_READY_LINE_PREFIX = '[pubsub] INFO: Server started, listening on ' - - -def get_parser(): - """Get simple ``argparse`` parser to determine package. - - :rtype: :class:`argparse.ArgumentParser` - :returns: The parser for this script. - """ - parser = argparse.ArgumentParser( - description='Run Google Cloud system tests against local emulator.') - parser.add_argument('--package', dest='package', - choices=('datastore', 'pubsub'), - default='datastore', help='Package to be tested.') - return parser - - -def get_start_command(package): - """Get command line arguments for starting emulator. - - :type package: str - :param package: The package to start an emulator for. - - :rtype: tuple - :returns: The arguments to be used, in a tuple. - """ - return 'gcloud', 'beta', 'emulators', package, 'start' - - -def get_env_init_command(package): - """Get command line arguments for getting emulator env. info. - - :type package: str - :param package: The package to get environment info for. - - :rtype: tuple - :returns: The arguments to be used, in a tuple. - """ - return 'gcloud', 'beta', 'emulators', package, 'env-init' - - -def datastore_wait_ready(popen): - """Wait until the datastore emulator is ready to use. - - :type popen: :class:`subprocess.Popen` - :param popen: An open subprocess to interact with. - """ - emulator_ready = False - while not emulator_ready: - emulator_ready = popen.stderr.readline() == _DS_READY_LINE - - -def pubsub_wait_ready(popen): - """Wait until the pubsub emulator is ready to use. - - :type popen: :class:`subprocess.Popen` - :param popen: An open subprocess to interact with. - """ - emulator_ready = False - while not emulator_ready: - emulator_ready = popen.stderr.readline().startswith( - _PS_READY_LINE_PREFIX) - - -def wait_ready(package, popen): - """Wait until the emulator is ready to use. - - :type package: str - :param package: The package to check if ready. - - :type popen: :class:`subprocess.Popen` - :param popen: An open subprocess to interact with. - - :raises: :class:`KeyError` if the ``package`` is not among - ``datastore``, ``pubsub``. - """ - if package == 'datastore': - datastore_wait_ready(popen) - elif package == 'pubsub': - pubsub_wait_ready(popen) - else: - raise KeyError('') - - -def cleanup(pid): - """Cleanup a process (including all of its children). - - :type pid: int - :param pid: Process ID. - """ - proc = psutil.Process(pid) - for child_proc in proc.children(recursive=True): - try: - child_proc.kill() - child_proc.terminate() - except psutil.NoSuchProcess: - pass - proc.terminate() - proc.kill() - - -def run_tests_in_emulator(package): - """Spawn an emulator instance and run the system tests. - - :type package: str - :param package: The package to run system tests against. - """ - # Make sure this package has environment vars to replace. - env_vars = PACKAGE_INFO[package] - - start_command = get_start_command(package) - # Ignore stdin and stdout, don't pollute the user's output with them. - proc_start = subprocess.Popen(start_command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - try: - wait_ready(package, proc_start) - env_init_command = get_env_init_command(package) - proc_env = subprocess.Popen(env_init_command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - env_status = proc_env.wait() - if env_status != 0: - raise RuntimeError(env_status, proc_env.stderr.read()) - env_lines = proc_env.stdout.read().strip().split('\n') - # Set environment variables before running the system tests. - for env_var in env_vars: - line_prefix = 'export ' + env_var + '=' - value, = [line.split(line_prefix, 1)[1] for line in env_lines - if line.startswith(line_prefix)] - os.environ[env_var] = value - run_module_tests(package, - ignore_requirements=True) - finally: - cleanup(proc_start.pid) - - -def main(): - """Main method to run this script.""" - parser = get_parser() - args = parser.parse_args() - run_tests_in_emulator(args.package) - - -if __name__ == '__main__': - main() diff --git a/tox.ini b/tox.ini index 35756df..437cc52 100644 --- a/tox.ini +++ b/tox.ini @@ -73,12 +73,12 @@ passenv = {[testenv:system-tests]passenv} basepython = python2.7 commands = - python {toxinidir}/system_tests/attempt_system_tests.py + python {toxinidir}/system_tests/attempt_system_tests.py {posargs} passenv = GOOGLE_* TRAVIS* encrypted_* [testenv:system-tests3] basepython = python3.4 commands = - python {toxinidir}/system_tests/attempt_system_tests.py + python {toxinidir}/system_tests/attempt_system_tests.py {posargs} passenv = {[testenv:system-tests]passenv}