From 1270f034baf360ec8d653bd90c83cd537e57aadd Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 25 Oct 2024 13:56:29 -0700 Subject: [PATCH 01/12] chore(tests): clean up sample test tables --- samples/beam/hello_world_write_test.py | 15 ++---- samples/hello/async_main_test.py | 33 ++++++------ samples/hello/main_test.py | 32 ++++++------ samples/hello_happybase/main.py | 1 + samples/hello_happybase/main_test.py | 10 ++-- samples/quickstart/main_async_test.py | 5 +- samples/quickstart/main_test.py | 5 +- samples/quickstart_happybase/main_test.py | 4 +- .../data_client_snippets_async_test.py | 2 +- .../snippets/deletes/deletes_async_test.py | 22 ++------ samples/snippets/deletes/deletes_test.py | 6 +-- .../filters/filter_snippets_async_test.py | 20 ++------ samples/snippets/filters/filters_test.py | 4 +- samples/snippets/reads/reads_test.py | 5 +- samples/snippets/writes/writes_test.py | 4 +- samples/tableadmin/tableadmin_test.py | 50 ++++++++++--------- 16 files changed, 99 insertions(+), 119 deletions(-) diff --git a/samples/beam/hello_world_write_test.py b/samples/beam/hello_world_write_test.py index 4e9a47c7d..9990ecf29 100644 --- a/samples/beam/hello_world_write_test.py +++ b/samples/beam/hello_world_write_test.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import uuid from google.cloud import bigtable import pytest @@ -21,7 +20,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_PREFIX = "mobile-time-series-{}" +TABLE_ID = "mobile-time-series-beam" @pytest.fixture(scope="module", autouse=True) @@ -29,30 +28,26 @@ def table_id(): client = bigtable.Client(project=PROJECT, admin=True) instance = client.instance(BIGTABLE_INSTANCE) - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) + table_id = TABLE_ID table = instance.table(table_id) if table.exists(): table.delete() table.create(column_families={"stats_summary": None}) - yield table_id + yield table table.delete() -def test_hello_world_write(table_id): +def test_hello_world_write(table): hello_world_write.run( [ "--bigtable-project=%s" % PROJECT, "--bigtable-instance=%s" % BIGTABLE_INSTANCE, - "--bigtable-table=%s" % table_id, + "--bigtable-table=%s" % TABLE_ID, ] ) - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(table_id) - rows = table.read_rows() count = 0 for _ in rows: diff --git a/samples/hello/async_main_test.py b/samples/hello/async_main_test.py index a47ac2d33..c7592c508 100644 --- a/samples/hello/async_main_test.py +++ b/samples/hello/async_main_test.py @@ -13,27 +13,32 @@ # limitations under the License. import os -import random import asyncio +from google.cloud import bigtable from async_main import main PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_NAME_FORMAT = "hello-world-test-{}" -TABLE_NAME_RANGE = 10000 +TABLE_ID = "hello-world-test-async" def test_async_main(capsys): - table_name = TABLE_NAME_FORMAT.format(random.randrange(TABLE_NAME_RANGE)) + try: + asyncio.run(main(PROJECT, BIGTABLE_INSTANCE, TABLE_ID)) - asyncio.run(main(PROJECT, BIGTABLE_INSTANCE, table_name)) - - out, _ = capsys.readouterr() - assert "Creating the {} table.".format(table_name) in out - assert "Writing some greetings to the table." in out - assert "Getting a single greeting by row key." in out - assert "Hello World!" in out - assert "Scanning for all greetings" in out - assert "Hello Cloud Bigtable!" in out - assert "Deleting the {} table.".format(table_name) in out + out, _ = capsys.readouterr() + assert "Creating the {} table.".format(TABLE_ID) in out + assert "Writing some greetings to the table." in out + assert "Getting a single greeting by row key." in out + assert "Hello World!" in out + assert "Scanning for all greetings" in out + assert "Hello Cloud Bigtable!" in out + assert "Deleting the {} table.".format(TABLE_ID) in out + finally: + # delete table + client = bigtable.Client(PROJECT, admin=True) + instance = client.instance(BIGTABLE_INSTANCE) + table = instance.table(TABLE_ID) + if table.exists(): + table.delete() diff --git a/samples/hello/main_test.py b/samples/hello/main_test.py index 641b34d11..d28bfdaf2 100644 --- a/samples/hello/main_test.py +++ b/samples/hello/main_test.py @@ -13,26 +13,30 @@ # limitations under the License. import os -import random from main import main PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_NAME_FORMAT = "hello-world-test-{}" -TABLE_NAME_RANGE = 10000 +TABLE_ID = "hello-world-test" def test_main(capsys): - table_name = TABLE_NAME_FORMAT.format(random.randrange(TABLE_NAME_RANGE)) + try: + main(PROJECT, BIGTABLE_INSTACE, TABLE_ID) - main(PROJECT, BIGTABLE_INSTANCE, table_name) - - out, _ = capsys.readouterr() - assert "Creating the {} table.".format(table_name) in out - assert "Writing some greetings to the table." in out - assert "Getting a single greeting by row key." in out - assert "Hello World!" in out - assert "Scanning for all greetings" in out - assert "Hello Cloud Bigtable!" in out - assert "Deleting the {} table.".format(table_name) in out + out, _ = capsys.readouterr() + assert "Creating the {} table.".format(TABLE_ID) in out + assert "Writing some greetings to the table." in out + assert "Getting a single greeting by row key." in out + assert "Hello World!" in out + assert "Scanning for all greetings" in out + assert "Hello Cloud Bigtable!" in out + assert "Deleting the {} table.".format(TABLE_ID) in out + finally: + # delete table + client = bigtable.Client(PROJECT, admin=True) + instance = client.instance(BIGTABLE_INSTANCE) + table = instance.table(TABLE_ID) + if table.exists(): + table.delete() diff --git a/samples/hello_happybase/main.py b/samples/hello_happybase/main.py index 7999fd006..e85050548 100644 --- a/samples/hello_happybase/main.py +++ b/samples/hello_happybase/main.py @@ -96,6 +96,7 @@ def main(project_id, instance_id, table_name): # [END bigtable_hw_delete_table_happybase] finally: + connection.delete_table(table_name) connection.close() diff --git a/samples/hello_happybase/main_test.py b/samples/hello_happybase/main_test.py index 6a63750da..0225e6ff8 100644 --- a/samples/hello_happybase/main_test.py +++ b/samples/hello_happybase/main_test.py @@ -19,19 +19,17 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_NAME_FORMAT = "hello-world-hb-test-{}" -TABLE_NAME_RANGE = 10000 +TABLE_ID = "hello-world-hb-test" def test_main(capsys): - table_name = TABLE_NAME_FORMAT.format(random.randrange(TABLE_NAME_RANGE)) - main(PROJECT, BIGTABLE_INSTANCE, table_name) + main(PROJECT, BIGTABLE_INSTANCE, TABLE_ID) out, _ = capsys.readouterr() - assert "Creating the {} table.".format(table_name) in out + assert "Creating the {} table.".format(TABLE_ID) in out assert "Writing some greetings to the table." in out assert "Getting a single greeting by row key." in out assert "Hello World!" in out assert "Scanning for all greetings" in out assert "Hello Cloud Bigtable!" in out - assert "Deleting the {} table.".format(table_name) in out + assert "Deleting the {} table.".format(TABLE_ID) in out diff --git a/samples/quickstart/main_async_test.py b/samples/quickstart/main_async_test.py index 841cfc180..fa95d085d 100644 --- a/samples/quickstart/main_async_test.py +++ b/samples/quickstart/main_async_test.py @@ -24,7 +24,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_FORMAT = "quickstart-test-{}" +TABLE_ID = "quickstart-async-test" @pytest_asyncio.fixture @@ -39,12 +39,11 @@ async def table_id() -> AsyncGenerator[str, None]: def _create_table(): from google.cloud import bigtable - import uuid client = bigtable.Client(project=PROJECT, admin=True) instance = client.instance(BIGTABLE_INSTANCE) - table_id = TABLE_ID_FORMAT.format(uuid.uuid4().hex[:8]) + table_id = TABLE_ID table = instance.table(table_id) if table.exists(): table.delete() diff --git a/samples/quickstart/main_test.py b/samples/quickstart/main_test.py index 46d578b6b..22cccfde8 100644 --- a/samples/quickstart/main_test.py +++ b/samples/quickstart/main_test.py @@ -13,7 +13,6 @@ # limitations under the License. import os -import uuid from google.cloud import bigtable import pytest @@ -23,12 +22,12 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_FORMAT = "quickstart-test-{}" +TABLE_ID = "quickstart-test" @pytest.fixture() def table(): - table_id = TABLE_ID_FORMAT.format(uuid.uuid4().hex[:8]) + table_id = TABLE_ID client = bigtable.Client(project=PROJECT, admin=True) instance = client.instance(BIGTABLE_INSTANCE) table = instance.table(table_id) diff --git a/samples/quickstart_happybase/main_test.py b/samples/quickstart_happybase/main_test.py index dc62ebede..c662aa0e1 100644 --- a/samples/quickstart_happybase/main_test.py +++ b/samples/quickstart_happybase/main_test.py @@ -23,12 +23,12 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_FORMAT = "quickstart-hb-test-{}" +TABLE_ID = "quickstart-hb-test" @pytest.fixture() def table(): - table_id = TABLE_ID_FORMAT.format(uuid.uuid4().hex[:8]) + table_id = TABLE_ID client = bigtable.Client(project=PROJECT, admin=True) instance = client.instance(BIGTABLE_INSTANCE) table = instance.table(table_id) diff --git a/samples/snippets/data_client/data_client_snippets_async_test.py b/samples/snippets/data_client/data_client_snippets_async_test.py index 2e0fb9b81..759f71e45 100644 --- a/samples/snippets/data_client/data_client_snippets_async_test.py +++ b/samples/snippets/data_client/data_client_snippets_async_test.py @@ -31,7 +31,7 @@ def table_id(): client = bigtable.Client(project=PROJECT, admin=True) instance = client.instance(BIGTABLE_INSTANCE) - table_id = TABLE_ID_STATIC or f"data-client-{str(uuid.uuid4())[:16]}" + table_id = TABLE_ID_STATIC or "data-client" admin_table = instance.table(table_id) if not admin_table.exists(): diff --git a/samples/snippets/deletes/deletes_async_test.py b/samples/snippets/deletes/deletes_async_test.py index b708bd52e..35c6bcf66 100644 --- a/samples/snippets/deletes/deletes_async_test.py +++ b/samples/snippets/deletes/deletes_async_test.py @@ -25,42 +25,30 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_PREFIX = "mobile-time-series-{}" +TABLE_ID = "mobile-time-series-deletes-async" @pytest_asyncio.fixture async def table_id() -> AsyncGenerator[str, None]: - table_id = _create_table() + table, table_id = _create_table() await _populate_table(table_id) yield table_id - _delete_table(table_id) + table.delete() def _create_table(): from google.cloud import bigtable - import uuid client = bigtable.Client(project=PROJECT, admin=True) instance = client.instance(BIGTABLE_INSTANCE) - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) + table_id = TABLE_ID table = instance.table(table_id) if table.exists(): table.delete() table.create(column_families={"stats_summary": None, "cell_plan": None}) - client.close() - return table_id - - -def _delete_table(table_id: str): - from google.cloud import bigtable - - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(table_id) - table.delete() - client.close() + return table, table_id async def _populate_table(table_id): diff --git a/samples/snippets/deletes/deletes_test.py b/samples/snippets/deletes/deletes_test.py index bebaabafb..61c975b55 100644 --- a/samples/snippets/deletes/deletes_test.py +++ b/samples/snippets/deletes/deletes_test.py @@ -16,7 +16,6 @@ import datetime import os import time -import uuid from google.cloud import bigtable import pytest @@ -25,7 +24,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_PREFIX = "mobile-time-series-{}" +TABLE_ID = "mobile-time-series-deletes" @pytest.fixture(scope="module", autouse=True) @@ -35,7 +34,7 @@ def table_id(): client = bigtable.Client(project=PROJECT, admin=True) instance = client.instance(BIGTABLE_INSTANCE) - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) + table_id = TABLE_ID table = instance.table(table_id) if table.exists(): table.delete() @@ -93,6 +92,7 @@ def table_id(): fetched = list(table.read_rows(row_set=row_set)) yield table_id + table.delete() def assert_output_match(capsys, expected): diff --git a/samples/snippets/filters/filter_snippets_async_test.py b/samples/snippets/filters/filter_snippets_async_test.py index 76751feaf..6f0bfb13f 100644 --- a/samples/snippets/filters/filter_snippets_async_test.py +++ b/samples/snippets/filters/filter_snippets_async_test.py @@ -29,40 +29,30 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_PREFIX = "mobile-time-series-{}" +TABLE_ID = "mobile-time-series-filters-async" @pytest_asyncio.fixture async def table_id() -> AsyncGenerator[str, None]: - table_id = _create_table() + table, table_id = _create_table() await _populate_table(table_id) yield table_id - _delete_table(table_id) + table.delete() def _create_table(): from google.cloud import bigtable - import uuid client = bigtable.Client(project=PROJECT, admin=True) instance = client.instance(BIGTABLE_INSTANCE) - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) + table_id = TABLE_ID table = instance.table(table_id) if table.exists(): table.delete() table.create(column_families={"stats_summary": None, "cell_plan": None}) - return table_id - - -def _delete_table(table_id: str): - from google.cloud import bigtable - - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(table_id) - table.delete() + return table, table_id async def _populate_table(table_id): diff --git a/samples/snippets/filters/filters_test.py b/samples/snippets/filters/filters_test.py index a84932039..913fe720f 100644 --- a/samples/snippets/filters/filters_test.py +++ b/samples/snippets/filters/filters_test.py @@ -26,7 +26,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_PREFIX = "mobile-time-series-{}" +TABLE_ID = "mobile-time-series-filters" @pytest.fixture(scope="module", autouse=True) @@ -36,7 +36,7 @@ def table_id(): client = bigtable.Client(project=PROJECT, admin=True) instance = client.instance(BIGTABLE_INSTANCE) - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) + table_id = TABLE_ID table = instance.table(table_id) if table.exists(): table.delete() diff --git a/samples/snippets/reads/reads_test.py b/samples/snippets/reads/reads_test.py index da826d6fb..e6f9febbe 100644 --- a/samples/snippets/reads/reads_test.py +++ b/samples/snippets/reads/reads_test.py @@ -13,7 +13,6 @@ import datetime import os -import uuid import inspect from google.cloud import bigtable @@ -25,7 +24,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_PREFIX = "mobile-time-series-{}" +TABLE_ID = "mobile-time-series-reads" @pytest.fixture(scope="module", autouse=True) @@ -33,7 +32,7 @@ def table_id(): client = bigtable.Client(project=PROJECT, admin=True) instance = client.instance(BIGTABLE_INSTANCE) - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) + table_id = TABLE_ID table = instance.table(table_id) if table.exists(): table.delete() diff --git a/samples/snippets/writes/writes_test.py b/samples/snippets/writes/writes_test.py index 77ae883d6..f473780a2 100644 --- a/samples/snippets/writes/writes_test.py +++ b/samples/snippets/writes/writes_test.py @@ -28,7 +28,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_PREFIX = "mobile-time-series-{}" +TABLE_ID = "mobile-time-series-writes" @pytest.fixture @@ -43,7 +43,7 @@ def bigtable_instance(bigtable_client): @pytest.fixture def table_id(bigtable_instance): - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) + table_id = TABLE_ID table = bigtable_instance.table(table_id) if table.exists(): table.delete() diff --git a/samples/tableadmin/tableadmin_test.py b/samples/tableadmin/tableadmin_test.py index 3063eee9f..cc630cda9 100755 --- a/samples/tableadmin/tableadmin_test.py +++ b/samples/tableadmin/tableadmin_test.py @@ -31,36 +31,38 @@ def test_run_table_operations(capsys): - table_id = TABLE_ID_FORMAT.format(uuid.uuid4().hex[:8]) - - retry_429_503(run_table_operations)(PROJECT, BIGTABLE_INSTANCE, table_id) - out, _ = capsys.readouterr() + try: + table_id = TABLE_ID_FORMAT.format(uuid.uuid4().hex[:8]) - assert "Creating the " + table_id + " table." in out - assert "Listing tables in current project." in out - assert "Creating column family cf1 with with MaxAge GC Rule" in out - assert "Created column family cf1 with MaxAge GC Rule." in out - assert "Created column family cf2 with Max Versions GC Rule." in out - assert "Created column family cf3 with Union GC rule" in out - assert "Created column family cf4 with Intersection GC rule." in out - assert "Created column family cf5 with a Nested GC rule." in out - assert "Printing Column Family and GC Rule for all column families." in out - assert "Updating column family cf1 GC rule..." in out - assert "Updated column family cf1 GC rule" in out - assert "Print column family cf1 GC rule after update..." in out - assert "Column Family: cf1" in out - assert "max_num_versions: 1" in out - assert "Delete a column family cf2..." in out - assert "Column family cf2 deleted successfully." in out + retry_429_503(run_table_operations)(PROJECT, BIGTABLE_INSTANCE, table_id) + out, _ = capsys.readouterr() - retry_429_503(delete_table)(PROJECT, BIGTABLE_INSTANCE, table_id) + assert "Creating the " + table_id + " table." in out + assert "Listing tables in current project." in out + assert "Creating column family cf1 with with MaxAge GC Rule" in out + assert "Created column family cf1 with MaxAge GC Rule." in out + assert "Created column family cf2 with Max Versions GC Rule." in out + assert "Created column family cf3 with Union GC rule" in out + assert "Created column family cf4 with Intersection GC rule." in out + assert "Created column family cf5 with a Nested GC rule." in out + assert "Printing Column Family and GC Rule for all column families." in out + assert "Updating column family cf1 GC rule..." in out + assert "Updated column family cf1 GC rule" in out + assert "Print column family cf1 GC rule after update..." in out + assert "Column Family: cf1" in out + assert "max_num_versions: 1" in out + assert "Delete a column family cf2..." in out + assert "Column family cf2 deleted successfully." in out + finally: + retry_429_503(delete_table)(PROJECT, BIGTABLE_INSTANCE, table_id) def test_delete_table(capsys): table_id = TABLE_ID_FORMAT.format(uuid.uuid4().hex[:8]) - retry_429_503(create_table)(PROJECT, BIGTABLE_INSTANCE, table_id) - - retry_429_503(delete_table)(PROJECT, BIGTABLE_INSTANCE, table_id) + try: + retry_429_503(create_table)(PROJECT, BIGTABLE_INSTANCE, table_id) + finally: + retry_429_503(delete_table)(PROJECT, BIGTABLE_INSTANCE, table_id) out, _ = capsys.readouterr() assert "Table " + table_id + " exists." in out From 5a47880e7a1f7935c0c6d2c0a90d17a04f9e5b3c Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 28 Oct 2024 11:36:53 -0700 Subject: [PATCH 02/12] fixed lint --- samples/hello/main_test.py | 3 ++- samples/hello_happybase/main_test.py | 1 - samples/quickstart_happybase/main_test.py | 1 - .../snippets/data_client/data_client_snippets_async_test.py | 1 - samples/snippets/filters/filters_test.py | 1 - samples/snippets/writes/writes_test.py | 1 - 6 files changed, 2 insertions(+), 6 deletions(-) diff --git a/samples/hello/main_test.py b/samples/hello/main_test.py index d28bfdaf2..f489c1177 100644 --- a/samples/hello/main_test.py +++ b/samples/hello/main_test.py @@ -15,6 +15,7 @@ import os from main import main +from google.cloud import bigtable PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] @@ -23,7 +24,7 @@ def test_main(capsys): try: - main(PROJECT, BIGTABLE_INSTACE, TABLE_ID) + main(PROJECT, BIGTABLE_INSTANCE, TABLE_ID) out, _ = capsys.readouterr() assert "Creating the {} table.".format(TABLE_ID) in out diff --git a/samples/hello_happybase/main_test.py b/samples/hello_happybase/main_test.py index 0225e6ff8..841918b4e 100644 --- a/samples/hello_happybase/main_test.py +++ b/samples/hello_happybase/main_test.py @@ -13,7 +13,6 @@ # limitations under the License. import os -import random from main import main diff --git a/samples/quickstart_happybase/main_test.py b/samples/quickstart_happybase/main_test.py index c662aa0e1..9c1e5b58d 100644 --- a/samples/quickstart_happybase/main_test.py +++ b/samples/quickstart_happybase/main_test.py @@ -13,7 +13,6 @@ # limitations under the License. import os -import uuid from google.cloud import bigtable import pytest diff --git a/samples/snippets/data_client/data_client_snippets_async_test.py b/samples/snippets/data_client/data_client_snippets_async_test.py index 759f71e45..685138374 100644 --- a/samples/snippets/data_client/data_client_snippets_async_test.py +++ b/samples/snippets/data_client/data_client_snippets_async_test.py @@ -12,7 +12,6 @@ # limitations under the License. import pytest import pytest_asyncio -import uuid import os import data_client_snippets_async as data_snippets diff --git a/samples/snippets/filters/filters_test.py b/samples/snippets/filters/filters_test.py index 913fe720f..ec126b170 100644 --- a/samples/snippets/filters/filters_test.py +++ b/samples/snippets/filters/filters_test.py @@ -16,7 +16,6 @@ import inspect import os import time -import uuid from google.cloud import bigtable import pytest diff --git a/samples/snippets/writes/writes_test.py b/samples/snippets/writes/writes_test.py index f473780a2..5d7219dc4 100644 --- a/samples/snippets/writes/writes_test.py +++ b/samples/snippets/writes/writes_test.py @@ -13,7 +13,6 @@ # limitations under the License. import os -import uuid import backoff from google.api_core.exceptions import DeadlineExceeded From 8c35b8c64e6e4297540278fb4ce4bb0521608742 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 28 Oct 2024 12:13:22 -0700 Subject: [PATCH 03/12] added checks for table creation --- samples/hello/async_main.py | 12 +++++++++++- samples/hello/main.py | 12 ++++++++++++ samples/snippets/deletes/deletes_async_test.py | 14 ++++++++++++++ .../snippets/filters/filter_snippets_async_test.py | 14 ++++++++++++++ 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/samples/hello/async_main.py b/samples/hello/async_main.py index d608bb073..d7614bb47 100644 --- a/samples/hello/async_main.py +++ b/samples/hello/async_main.py @@ -33,7 +33,7 @@ from google.cloud.bigtable.data import RowMutationEntry from google.cloud.bigtable.data import SetCell from google.cloud.bigtable.data import ReadRowsQuery - +from google.api_core.exceptions import PreconditionFailed # [END bigtable_async_hw_imports] @@ -64,6 +64,16 @@ async def main(project_id, instance_id, table_id): else: print("Table {} already exists.".format(table_id)) # [END bigtable_async_hw_create_table] + # let table creation complete + attempts = 0 + table_ready = False + while not table_ready and attempts < 10: + try: + table_ready = table.exists() + except PreconditionFailed: + print("Waiting for table to become ready...") + attempts += 1 + await asyncio.sleep(5) # [START bigtable_async_hw_write_rows] print("Writing some greetings to the table.") diff --git a/samples/hello/main.py b/samples/hello/main.py index 3b7de34b0..f5f0f250a 100644 --- a/samples/hello/main.py +++ b/samples/hello/main.py @@ -25,6 +25,8 @@ """ import argparse +import time +from google.api_core.exceptions import PreconditionFailed # [START bigtable_hw_imports] import datetime @@ -59,6 +61,16 @@ def main(project_id, instance_id, table_id): else: print("Table {} already exists.".format(table_id)) # [END bigtable_hw_create_table] + # let table creation complete + attempts = 0 + table_ready = False + while not table_ready and attempts < 10: + try: + table_ready = table.exists() + except PreconditionFailed: + print("Waiting for table to become ready...") + attempts += 1 + time.sleep(5) # [START bigtable_hw_write_rows] print("Writing some greetings to the table.") diff --git a/samples/snippets/deletes/deletes_async_test.py b/samples/snippets/deletes/deletes_async_test.py index 35c6bcf66..5a1e154cc 100644 --- a/samples/snippets/deletes/deletes_async_test.py +++ b/samples/snippets/deletes/deletes_async_test.py @@ -15,11 +15,13 @@ import datetime import os +import time from typing import AsyncGenerator from google.cloud._helpers import _microseconds_from_datetime import pytest import pytest_asyncio +from google.api_core.exceptions import PreconditionFailed import deletes_snippets_async @@ -48,6 +50,18 @@ def _create_table(): table.delete() table.create(column_families={"stats_summary": None, "cell_plan": None}) + + # let table creation complete + attempts = 0 + table_ready = False + while not table_ready and attempts < 10: + try: + table_ready = table.exists() + except PreconditionFailed: + print("Waiting for table to become ready...") + attempts += 1 + time.sleep(5) + return table, table_id diff --git a/samples/snippets/filters/filter_snippets_async_test.py b/samples/snippets/filters/filter_snippets_async_test.py index 6f0bfb13f..6e3d67835 100644 --- a/samples/snippets/filters/filter_snippets_async_test.py +++ b/samples/snippets/filters/filter_snippets_async_test.py @@ -14,6 +14,7 @@ import datetime import os +import time import inspect from typing import AsyncGenerator @@ -26,6 +27,7 @@ from google.cloud._helpers import ( _microseconds_from_datetime, ) +from google.api_core.exceptions import PreconditionFailed PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] @@ -52,6 +54,18 @@ def _create_table(): table.delete() table.create(column_families={"stats_summary": None, "cell_plan": None}) + + # let table creation complete + attempts = 0 + table_ready = False + while not table_ready and attempts < 10: + try: + table_ready = table.exists() + except PreconditionFailed: + print("Waiting for table to become ready...") + attempts += 1 + time.sleep(5) + return table, table_id From 29631e8d38c58a6eb48361899556967182b1ac25 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 29 Oct 2024 15:08:21 -0700 Subject: [PATCH 04/12] fixed python3.7 error --- .kokoro/test-samples-impl.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh index 55910c8ba..53e365bc4 100755 --- a/.kokoro/test-samples-impl.sh +++ b/.kokoro/test-samples-impl.sh @@ -33,7 +33,8 @@ export PYTHONUNBUFFERED=1 env | grep KOKORO # Install nox -python3.9 -m pip install --upgrade --quiet nox +# `virtualenv==20.26.6` is added for Python 3.7 compatibility +python3.9 -m pip install --upgrade --quiet nox virtualenv==20.26.6 # Use secrets acessor service account to get secrets if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then From 429a496cadd4dd354865cd216b08cec5c9c9e7c0 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 29 Oct 2024 15:08:36 -0700 Subject: [PATCH 05/12] clear table as part of test --- samples/hello/main_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/samples/hello/main_test.py b/samples/hello/main_test.py index f489c1177..8b00a920a 100644 --- a/samples/hello/main_test.py +++ b/samples/hello/main_test.py @@ -21,6 +21,14 @@ BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] TABLE_ID = "hello-world-test" +def test_clear_tables(): + client = bigtable.Client(PROJECT, admin=True) + instance = client.instance(BIGTABLE_INSTANCE) + + for table in instance.list_tables(): + print(f"Deleting table {table.table_id}...") + table.delete() + print(f"Table {table.table_id} deleted.") def test_main(capsys): try: From db753d7fdb30cb52225b274edff94c120636d8a7 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 31 Oct 2024 12:10:24 -0700 Subject: [PATCH 06/12] reverted table clear code --- samples/hello/main_test.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/samples/hello/main_test.py b/samples/hello/main_test.py index 8b00a920a..f489c1177 100644 --- a/samples/hello/main_test.py +++ b/samples/hello/main_test.py @@ -21,14 +21,6 @@ BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] TABLE_ID = "hello-world-test" -def test_clear_tables(): - client = bigtable.Client(PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - - for table in instance.list_tables(): - print(f"Deleting table {table.table_id}...") - table.delete() - print(f"Table {table.table_id} deleted.") def test_main(capsys): try: From 9dbe9c8f2d8c43e45365a8326349360ddb4a9568 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 7 Nov 2024 12:27:35 -0800 Subject: [PATCH 07/12] moved table manegement into shared utils module --- samples/__init__.py | 0 samples/beam/__init__.py | 0 samples/beam/hello_world_write_test.py | 22 +- samples/beam/noxfile.py | 16 +- samples/hello/__init__.py | 0 samples/hello/async_main.py | 127 ++++--- samples/hello/async_main_test.py | 29 +- samples/hello/main.py | 130 ++++--- samples/hello/main_test.py | 29 +- samples/hello_happybase/__init__.py | 0 samples/hello_happybase/main.py | 7 +- samples/hello_happybase/main_test.py | 29 +- samples/quickstart/__init__.py | 0 samples/quickstart/main_async_test.py | 40 +-- samples/quickstart/main_test.py | 22 +- samples/quickstart_happybase/__init__.py | 0 samples/quickstart_happybase/main_test.py | 22 +- samples/snippets/__init__.py | 0 samples/snippets/data_client/__init__.py | 0 .../data_client_snippets_async_test.py | 24 +- samples/snippets/deletes/__init__.py | 0 .../snippets/deletes/deletes_async_test.py | 41 +-- samples/snippets/deletes/deletes_test.py | 124 ++++--- .../filters/filter_snippets_async_test.py | 39 +-- samples/snippets/filters/filters_test.py | 111 +++--- samples/snippets/reads/reads_test.py | 73 ++-- samples/snippets/writes/writes_test.py | 29 +- samples/tableadmin/__init__.py | 0 samples/tableadmin/tableadmin.py | 320 ++++++++---------- samples/tableadmin/tableadmin_test.py | 60 ++-- samples/utils.py | 70 ++++ 31 files changed, 600 insertions(+), 764 deletions(-) create mode 100644 samples/__init__.py create mode 100644 samples/beam/__init__.py create mode 100644 samples/hello/__init__.py create mode 100644 samples/hello_happybase/__init__.py create mode 100644 samples/quickstart/__init__.py create mode 100644 samples/quickstart_happybase/__init__.py create mode 100644 samples/snippets/__init__.py create mode 100644 samples/snippets/data_client/__init__.py create mode 100644 samples/snippets/deletes/__init__.py create mode 100644 samples/tableadmin/__init__.py create mode 100644 samples/utils.py diff --git a/samples/__init__.py b/samples/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/beam/__init__.py b/samples/beam/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/beam/hello_world_write_test.py b/samples/beam/hello_world_write_test.py index 9990ecf29..66c9764e2 100644 --- a/samples/beam/hello_world_write_test.py +++ b/samples/beam/hello_world_write_test.py @@ -13,10 +13,10 @@ # limitations under the License. import os -from google.cloud import bigtable import pytest -import hello_world_write +from . import hello_world_write +from ..utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] @@ -24,19 +24,11 @@ @pytest.fixture(scope="module", autouse=True) -def table_id(): - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - - table_id = TABLE_ID - table = instance.table(table_id) - if table.exists(): - table.delete() - - table.create(column_families={"stats_summary": None}) - yield table - - table.delete() +def table(): + with create_table_cm( + PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"stats_summary": None} + ) as table: + yield table def test_hello_world_write(table): diff --git a/samples/beam/noxfile.py b/samples/beam/noxfile.py index 80ffdb178..8ae94fd44 100644 --- a/samples/beam/noxfile.py +++ b/samples/beam/noxfile.py @@ -22,7 +22,6 @@ import nox - # WARNING - WARNING - WARNING - WARNING - WARNING # WARNING - WARNING - WARNING - WARNING - WARNING # DO NOT EDIT THIS FILE EVER! @@ -158,6 +157,7 @@ def blacken(session: nox.sessions.Session) -> None: # format = isort + black # + @nox.session def format(session: nox.sessions.Session) -> None: """ @@ -185,7 +185,9 @@ def _session_tests( session: nox.sessions.Session, post_install: Callable = None ) -> None: # check for presence of tests - test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob("**/test_*.py", recursive=True) + test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob( + "**/test_*.py", recursive=True + ) test_list.extend(glob.glob("**/tests", recursive=True)) if len(test_list) == 0: @@ -207,9 +209,7 @@ def _session_tests( if os.path.exists("requirements-test.txt"): if os.path.exists("constraints-test.txt"): - session.install( - "-r", "requirements-test.txt", "-c", "constraints-test.txt" - ) + session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") else: session.install("-r", "requirements-test.txt") with open("requirements-test.txt") as rtfile: @@ -222,9 +222,9 @@ def _session_tests( post_install(session) if "pytest-parallel" in packages: - concurrent_args.extend(['--workers', 'auto', '--tests-per-worker', 'auto']) + concurrent_args.extend(["--workers", "auto", "--tests-per-worker", "auto"]) elif "pytest-xdist" in packages: - concurrent_args.extend(['-n', 'auto']) + concurrent_args.extend(["-n", "auto"]) session.run( "pytest", @@ -254,7 +254,7 @@ def py(session: nox.sessions.Session) -> None: def _get_repo_root() -> Optional[str]: - """ Returns the root folder of the project. """ + """Returns the root folder of the project.""" # Get root of this repository. Assume we don't have directories nested deeper than 10 items. p = Path(os.getcwd()) for i in range(10): diff --git a/samples/hello/__init__.py b/samples/hello/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/hello/async_main.py b/samples/hello/async_main.py index d7614bb47..0130161e1 100644 --- a/samples/hello/async_main.py +++ b/samples/hello/async_main.py @@ -26,6 +26,7 @@ import argparse import asyncio +from ..utils import wait_for_table # [START bigtable_async_hw_imports] from google.cloud import bigtable @@ -33,7 +34,6 @@ from google.cloud.bigtable.data import RowMutationEntry from google.cloud.bigtable.data import SetCell from google.cloud.bigtable.data import ReadRowsQuery -from google.api_core.exceptions import PreconditionFailed # [END bigtable_async_hw_imports] @@ -64,74 +64,67 @@ async def main(project_id, instance_id, table_id): else: print("Table {} already exists.".format(table_id)) # [END bigtable_async_hw_create_table] - # let table creation complete - attempts = 0 - table_ready = False - while not table_ready and attempts < 10: - try: - table_ready = table.exists() - except PreconditionFailed: - print("Waiting for table to become ready...") - attempts += 1 - await asyncio.sleep(5) - - # [START bigtable_async_hw_write_rows] - print("Writing some greetings to the table.") - greetings = ["Hello World!", "Hello Cloud Bigtable!", "Hello Python!"] - mutations = [] - column = "greeting" - for i, value in enumerate(greetings): - # Note: This example uses sequential numeric IDs for simplicity, - # but this can result in poor performance in a production - # application. Since rows are stored in sorted order by key, - # sequential keys can result in poor distribution of operations - # across nodes. - # - # For more information about how to design a Bigtable schema for - # the best performance, see the documentation: - # - # https://cloud.google.com/bigtable/docs/schema-design - row_key = "greeting{}".format(i).encode() - row_mutation = RowMutationEntry( - row_key, SetCell(column_family_id, column, value) - ) - mutations.append(row_mutation) - await table.bulk_mutate_rows(mutations) - # [END bigtable_async_hw_write_rows] - - # [START bigtable_async_hw_create_filter] - # Create a filter to only retrieve the most recent version of the cell - # for each column across entire row. - row_filter = row_filters.CellsColumnLimitFilter(1) - # [END bigtable_async_hw_create_filter] - - # [START bigtable_async_hw_get_with_filter] - # [START bigtable_async_hw_get_by_key] - print("Getting a single greeting by row key.") - key = "greeting0".encode() - - row = await table.read_row(key, row_filter=row_filter) - cell = row.cells[0] - print(cell.value.decode("utf-8")) - # [END bigtable_async_hw_get_by_key] - # [END bigtable_async_hw_get_with_filter] - - # [START bigtable_async_hw_scan_with_filter] - # [START bigtable_async_hw_scan_all] - print("Scanning for all greetings:") - query = ReadRowsQuery(row_filter=row_filter) - async for row in await table.read_rows_stream(query): + + try: + # let table creation complete + wait_for_table(admin_table) + # [START bigtable_async_hw_write_rows] + print("Writing some greetings to the table.") + greetings = ["Hello World!", "Hello Cloud Bigtable!", "Hello Python!"] + mutations = [] + column = "greeting" + for i, value in enumerate(greetings): + # Note: This example uses sequential numeric IDs for simplicity, + # but this can result in poor performance in a production + # application. Since rows are stored in sorted order by key, + # sequential keys can result in poor distribution of operations + # across nodes. + # + # For more information about how to design a Bigtable schema for + # the best performance, see the documentation: + # + # https://cloud.google.com/bigtable/docs/schema-design + row_key = "greeting{}".format(i).encode() + row_mutation = RowMutationEntry( + row_key, SetCell(column_family_id, column, value) + ) + mutations.append(row_mutation) + await table.bulk_mutate_rows(mutations) + # [END bigtable_async_hw_write_rows] + + # [START bigtable_async_hw_create_filter] + # Create a filter to only retrieve the most recent version of the cell + # for each column across entire row. + row_filter = row_filters.CellsColumnLimitFilter(1) + # [END bigtable_async_hw_create_filter] + + # [START bigtable_async_hw_get_with_filter] + # [START bigtable_async_hw_get_by_key] + print("Getting a single greeting by row key.") + key = "greeting0".encode() + + row = await table.read_row(key, row_filter=row_filter) cell = row.cells[0] print(cell.value.decode("utf-8")) - # [END bigtable_async_hw_scan_all] - # [END bigtable_async_hw_scan_with_filter] - - # [START bigtable_async_hw_delete_table] - # the async client only supports the data API. Table deletion as an admin operation - # use admin client to create the table - print("Deleting the {} table.".format(table_id)) - admin_table.delete() - # [END bigtable_async_hw_delete_table] + # [END bigtable_async_hw_get_by_key] + # [END bigtable_async_hw_get_with_filter] + + # [START bigtable_async_hw_scan_with_filter] + # [START bigtable_async_hw_scan_all] + print("Scanning for all greetings:") + query = ReadRowsQuery(row_filter=row_filter) + async for row in await table.read_rows_stream(query): + cell = row.cells[0] + print(cell.value.decode("utf-8")) + # [END bigtable_async_hw_scan_all] + # [END bigtable_async_hw_scan_with_filter] + finally: + # [START bigtable_async_hw_delete_table] + # the async client only supports the data API. Table deletion as an admin operation + # use admin client to create the table + print("Deleting the {} table.".format(table_id)) + admin_table.delete() + # [END bigtable_async_hw_delete_table] if __name__ == "__main__": diff --git a/samples/hello/async_main_test.py b/samples/hello/async_main_test.py index c7592c508..5b005b05b 100644 --- a/samples/hello/async_main_test.py +++ b/samples/hello/async_main_test.py @@ -14,9 +14,8 @@ import os import asyncio -from google.cloud import bigtable -from async_main import main +from .async_main import main PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] @@ -24,21 +23,13 @@ def test_async_main(capsys): - try: - asyncio.run(main(PROJECT, BIGTABLE_INSTANCE, TABLE_ID)) + asyncio.run(main(PROJECT, BIGTABLE_INSTANCE, TABLE_ID)) - out, _ = capsys.readouterr() - assert "Creating the {} table.".format(TABLE_ID) in out - assert "Writing some greetings to the table." in out - assert "Getting a single greeting by row key." in out - assert "Hello World!" in out - assert "Scanning for all greetings" in out - assert "Hello Cloud Bigtable!" in out - assert "Deleting the {} table.".format(TABLE_ID) in out - finally: - # delete table - client = bigtable.Client(PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(TABLE_ID) - if table.exists(): - table.delete() + out, _ = capsys.readouterr() + assert "Creating the {} table.".format(TABLE_ID) in out + assert "Writing some greetings to the table." in out + assert "Getting a single greeting by row key." in out + assert "Hello World!" in out + assert "Scanning for all greetings" in out + assert "Hello Cloud Bigtable!" in out + assert "Deleting the {} table.".format(TABLE_ID) in out diff --git a/samples/hello/main.py b/samples/hello/main.py index f5f0f250a..3e5078608 100644 --- a/samples/hello/main.py +++ b/samples/hello/main.py @@ -25,8 +25,7 @@ """ import argparse -import time -from google.api_core.exceptions import PreconditionFailed +from ..utils import wait_for_table # [START bigtable_hw_imports] import datetime @@ -61,74 +60,69 @@ def main(project_id, instance_id, table_id): else: print("Table {} already exists.".format(table_id)) # [END bigtable_hw_create_table] - # let table creation complete - attempts = 0 - table_ready = False - while not table_ready and attempts < 10: - try: - table_ready = table.exists() - except PreconditionFailed: - print("Waiting for table to become ready...") - attempts += 1 - time.sleep(5) - - # [START bigtable_hw_write_rows] - print("Writing some greetings to the table.") - greetings = ["Hello World!", "Hello Cloud Bigtable!", "Hello Python!"] - rows = [] - column = "greeting".encode() - for i, value in enumerate(greetings): - # Note: This example uses sequential numeric IDs for simplicity, - # but this can result in poor performance in a production - # application. Since rows are stored in sorted order by key, - # sequential keys can result in poor distribution of operations - # across nodes. - # - # For more information about how to design a Bigtable schema for - # the best performance, see the documentation: - # - # https://cloud.google.com/bigtable/docs/schema-design - row_key = "greeting{}".format(i).encode() - row = table.direct_row(row_key) - row.set_cell( - column_family_id, column, value, timestamp=datetime.datetime.utcnow() - ) - rows.append(row) - table.mutate_rows(rows) - # [END bigtable_hw_write_rows] - - # [START bigtable_hw_create_filter] - # Create a filter to only retrieve the most recent version of the cell - # for each column across entire row. - row_filter = row_filters.CellsColumnLimitFilter(1) - # [END bigtable_hw_create_filter] - - # [START bigtable_hw_get_with_filter] - # [START bigtable_hw_get_by_key] - print("Getting a single greeting by row key.") - key = "greeting0".encode() - - row = table.read_row(key, row_filter) - cell = row.cells[column_family_id][column][0] - print(cell.value.decode("utf-8")) - # [END bigtable_hw_get_by_key] - # [END bigtable_hw_get_with_filter] - - # [START bigtable_hw_scan_with_filter] - # [START bigtable_hw_scan_all] - print("Scanning for all greetings:") - partial_rows = table.read_rows(filter_=row_filter) - - for row in partial_rows: + + try: + # let table creation complete + wait_for_table(table) + + # [START bigtable_hw_write_rows] + print("Writing some greetings to the table.") + greetings = ["Hello World!", "Hello Cloud Bigtable!", "Hello Python!"] + rows = [] + column = "greeting".encode() + for i, value in enumerate(greetings): + # Note: This example uses sequential numeric IDs for simplicity, + # but this can result in poor performance in a production + # application. Since rows are stored in sorted order by key, + # sequential keys can result in poor distribution of operations + # across nodes. + # + # For more information about how to design a Bigtable schema for + # the best performance, see the documentation: + # + # https://cloud.google.com/bigtable/docs/schema-design + row_key = "greeting{}".format(i).encode() + row = table.direct_row(row_key) + row.set_cell( + column_family_id, column, value, timestamp=datetime.datetime.utcnow() + ) + rows.append(row) + table.mutate_rows(rows) + # [END bigtable_hw_write_rows] + + # [START bigtable_hw_create_filter] + # Create a filter to only retrieve the most recent version of the cell + # for each column across entire row. + row_filter = row_filters.CellsColumnLimitFilter(1) + # [END bigtable_hw_create_filter] + + # [START bigtable_hw_get_with_filter] + # [START bigtable_hw_get_by_key] + print("Getting a single greeting by row key.") + key = "greeting0".encode() + + row = table.read_row(key, row_filter) cell = row.cells[column_family_id][column][0] print(cell.value.decode("utf-8")) - # [END bigtable_hw_scan_all] - # [END bigtable_hw_scan_with_filter] - - # [START bigtable_hw_delete_table] - print("Deleting the {} table.".format(table_id)) - table.delete() - # [END bigtable_hw_delete_table] + # [END bigtable_hw_get_by_key] + # [END bigtable_hw_get_with_filter] + + # [START bigtable_hw_scan_with_filter] + # [START bigtable_hw_scan_all] + print("Scanning for all greetings:") + partial_rows = table.read_rows(filter_=row_filter) + + for row in partial_rows: + cell = row.cells[column_family_id][column][0] + print(cell.value.decode("utf-8")) + # [END bigtable_hw_scan_all] + # [END bigtable_hw_scan_with_filter] + + finally: + # [START bigtable_hw_delete_table] + print("Deleting the {} table.".format(table_id)) + table.delete() + # [END bigtable_hw_delete_table] if __name__ == "__main__": diff --git a/samples/hello/main_test.py b/samples/hello/main_test.py index f489c1177..e6ca87c83 100644 --- a/samples/hello/main_test.py +++ b/samples/hello/main_test.py @@ -14,8 +14,7 @@ import os -from main import main -from google.cloud import bigtable +from .main import main PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] @@ -23,21 +22,13 @@ def test_main(capsys): - try: - main(PROJECT, BIGTABLE_INSTANCE, TABLE_ID) + main(PROJECT, BIGTABLE_INSTANCE, TABLE_ID) - out, _ = capsys.readouterr() - assert "Creating the {} table.".format(TABLE_ID) in out - assert "Writing some greetings to the table." in out - assert "Getting a single greeting by row key." in out - assert "Hello World!" in out - assert "Scanning for all greetings" in out - assert "Hello Cloud Bigtable!" in out - assert "Deleting the {} table.".format(TABLE_ID) in out - finally: - # delete table - client = bigtable.Client(PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(TABLE_ID) - if table.exists(): - table.delete() + out, _ = capsys.readouterr() + assert "Creating the {} table.".format(TABLE_ID) in out + assert "Writing some greetings to the table." in out + assert "Getting a single greeting by row key." in out + assert "Hello World!" in out + assert "Scanning for all greetings" in out + assert "Hello Cloud Bigtable!" in out + assert "Deleting the {} table.".format(TABLE_ID) in out diff --git a/samples/hello_happybase/__init__.py b/samples/hello_happybase/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/hello_happybase/main.py b/samples/hello_happybase/main.py index e85050548..570f36b63 100644 --- a/samples/hello_happybase/main.py +++ b/samples/hello_happybase/main.py @@ -25,6 +25,7 @@ """ import argparse +from ..utils import wait_for_table # [START bigtable_hw_imports_happybase] from google.cloud import bigtable @@ -51,6 +52,8 @@ def main(project_id, instance_id, table_name): ) # [END bigtable_hw_create_table_happybase] + wait_for_table(connection.table(table_name)) + # [START bigtable_hw_write_rows_happybase] print("Writing some greetings to the table.") table = connection.table(table_name) @@ -90,13 +93,11 @@ def main(project_id, instance_id, table_name): print("\t{}: {}".format(key, row[column_name.encode("utf-8")])) # [END bigtable_hw_scan_all_happybase] + finally: # [START bigtable_hw_delete_table_happybase] print("Deleting the {} table.".format(table_name)) connection.delete_table(table_name) # [END bigtable_hw_delete_table_happybase] - - finally: - connection.delete_table(table_name) connection.close() diff --git a/samples/hello_happybase/main_test.py b/samples/hello_happybase/main_test.py index 841918b4e..a950ec382 100644 --- a/samples/hello_happybase/main_test.py +++ b/samples/hello_happybase/main_test.py @@ -14,7 +14,8 @@ import os -from main import main +from .main import main +from google.cloud import bigtable PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] @@ -22,13 +23,21 @@ def test_main(capsys): - main(PROJECT, BIGTABLE_INSTANCE, TABLE_ID) + try: + main(PROJECT, BIGTABLE_INSTANCE, TABLE_ID) - out, _ = capsys.readouterr() - assert "Creating the {} table.".format(TABLE_ID) in out - assert "Writing some greetings to the table." in out - assert "Getting a single greeting by row key." in out - assert "Hello World!" in out - assert "Scanning for all greetings" in out - assert "Hello Cloud Bigtable!" in out - assert "Deleting the {} table.".format(TABLE_ID) in out + out, _ = capsys.readouterr() + assert "Creating the {} table.".format(TABLE_ID) in out + assert "Writing some greetings to the table." in out + assert "Getting a single greeting by row key." in out + assert "Hello World!" in out + assert "Scanning for all greetings" in out + assert "Hello Cloud Bigtable!" in out + assert "Deleting the {} table.".format(TABLE_ID) in out + finally: + # delete table + client = bigtable.Client(PROJECT, admin=True) + instance = client.instance(BIGTABLE_INSTANCE) + table = instance.table(TABLE_ID) + if table.exists(): + table.delete() diff --git a/samples/quickstart/__init__.py b/samples/quickstart/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/quickstart/main_async_test.py b/samples/quickstart/main_async_test.py index fa95d085d..6f179a7c2 100644 --- a/samples/quickstart/main_async_test.py +++ b/samples/quickstart/main_async_test.py @@ -19,8 +19,8 @@ import pytest import pytest_asyncio -from main_async import main - +from .main_async import main +from ..utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] @@ -29,29 +29,9 @@ @pytest_asyncio.fixture async def table_id() -> AsyncGenerator[str, None]: - table_id = _create_table() - await _populate_table(table_id) - - yield table_id - - _delete_table(table_id) - - -def _create_table(): - from google.cloud import bigtable - - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - - table_id = TABLE_ID - table = instance.table(table_id) - if table.exists(): - table.delete() - - table.create(column_families={"cf1": None}) - - client.close() - return table_id + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"cf1": None}): + await _populate_table(TABLE_ID) + yield TABLE_ID async def _populate_table(table_id: str): @@ -60,16 +40,6 @@ async def _populate_table(table_id: str): await table.mutate_row("r1", SetCell("cf1", "c1", "test-value")) -def _delete_table(table_id: str): - from google.cloud import bigtable - - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(table_id) - table.delete() - client.close() - - @pytest.mark.asyncio async def test_main(capsys, table_id): await main(PROJECT, BIGTABLE_INSTANCE, table_id) diff --git a/samples/quickstart/main_test.py b/samples/quickstart/main_test.py index 22cccfde8..53a339548 100644 --- a/samples/quickstart/main_test.py +++ b/samples/quickstart/main_test.py @@ -14,10 +14,11 @@ import os -from google.cloud import bigtable import pytest -from main import main +from .main import main + +from ..utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] @@ -27,21 +28,14 @@ @pytest.fixture() def table(): - table_id = TABLE_ID - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(table_id) column_family_id = "cf1" column_families = {column_family_id: None} - table.create(column_families=column_families) - - row = table.direct_row("r1") - row.set_cell(column_family_id, "c1", "test-value") - row.commit() - - yield table_id + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, column_families) as table: + row = table.direct_row("r1") + row.set_cell(column_family_id, "c1", "test-value") + row.commit() - table.delete() + yield TABLE_ID def test_main(capsys, table): diff --git a/samples/quickstart_happybase/__init__.py b/samples/quickstart_happybase/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/quickstart_happybase/main_test.py b/samples/quickstart_happybase/main_test.py index 9c1e5b58d..9d2b040a0 100644 --- a/samples/quickstart_happybase/main_test.py +++ b/samples/quickstart_happybase/main_test.py @@ -14,11 +14,10 @@ import os -from google.cloud import bigtable import pytest -from main import main - +from .main import main +from ..utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] @@ -27,21 +26,14 @@ @pytest.fixture() def table(): - table_id = TABLE_ID - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(table_id) column_family_id = "cf1" column_families = {column_family_id: None} - table.create(column_families=column_families) - - row = table.direct_row("r1") - row.set_cell(column_family_id, "c1", "test-value") - row.commit() - - yield table_id + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, column_families) as table: + row = table.direct_row("r1") + row.set_cell(column_family_id, "c1", "test-value") + row.commit() - table.delete() + yield TABLE_ID def test_main(capsys, table): diff --git a/samples/snippets/__init__.py b/samples/snippets/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/snippets/data_client/__init__.py b/samples/snippets/data_client/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/snippets/data_client/data_client_snippets_async_test.py b/samples/snippets/data_client/data_client_snippets_async_test.py index 685138374..467986170 100644 --- a/samples/snippets/data_client/data_client_snippets_async_test.py +++ b/samples/snippets/data_client/data_client_snippets_async_test.py @@ -14,33 +14,19 @@ import pytest_asyncio import os -import data_client_snippets_async as data_snippets +from . import data_client_snippets_async as data_snippets +from ...utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_STATIC = os.getenv( - "BIGTABLE_TABLE", None -) # if not set, a temproary table will be generated +TABLE_ID = "data-client" @pytest.fixture(scope="session") def table_id(): - from google.cloud import bigtable - - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table_id = TABLE_ID_STATIC or "data-client" - - admin_table = instance.table(table_id) - if not admin_table.exists(): - admin_table.create(column_families={"family": None, "stats_summary": None}) - - yield table_id - - if not table_id == TABLE_ID_STATIC: - # clean up table when finished - admin_table.delete() + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"family": None, "stats_summary": None}): + yield TABLE_ID @pytest_asyncio.fixture diff --git a/samples/snippets/deletes/__init__.py b/samples/snippets/deletes/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/snippets/deletes/deletes_async_test.py b/samples/snippets/deletes/deletes_async_test.py index 5a1e154cc..8773e1fcc 100644 --- a/samples/snippets/deletes/deletes_async_test.py +++ b/samples/snippets/deletes/deletes_async_test.py @@ -15,54 +15,25 @@ import datetime import os -import time from typing import AsyncGenerator from google.cloud._helpers import _microseconds_from_datetime import pytest import pytest_asyncio -from google.api_core.exceptions import PreconditionFailed -import deletes_snippets_async +from . import deletes_snippets_async +from ...utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] TABLE_ID = "mobile-time-series-deletes-async" -@pytest_asyncio.fixture +@pytest_asyncio.fixture(scope="module", autouse=True) async def table_id() -> AsyncGenerator[str, None]: - table, table_id = _create_table() - await _populate_table(table_id) - yield table_id - table.delete() - - -def _create_table(): - from google.cloud import bigtable - - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - - table_id = TABLE_ID - table = instance.table(table_id) - if table.exists(): - table.delete() - - table.create(column_families={"stats_summary": None, "cell_plan": None}) - - # let table creation complete - attempts = 0 - table_ready = False - while not table_ready and attempts < 10: - try: - table_ready = table.exists() - except PreconditionFailed: - print("Waiting for table to become ready...") - attempts += 1 - time.sleep(5) - - return table, table_id + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"stats_summary": None, "cell_plan": None}, verbose=False): + await _populate_table(TABLE_ID) + yield TABLE_ID async def _populate_table(table_id): diff --git a/samples/snippets/deletes/deletes_test.py b/samples/snippets/deletes/deletes_test.py index 61c975b55..9c0419b58 100644 --- a/samples/snippets/deletes/deletes_test.py +++ b/samples/snippets/deletes/deletes_test.py @@ -17,82 +17,72 @@ import os import time -from google.cloud import bigtable import pytest -import deletes_snippets +from . import deletes_snippets +from ...utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] TABLE_ID = "mobile-time-series-deletes" -@pytest.fixture(scope="module", autouse=True) +@pytest.fixture(scope="module") def table_id(): from google.cloud.bigtable.row_set import RowSet - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - - table_id = TABLE_ID - table = instance.table(table_id) - if table.exists(): - table.delete() - - table.create(column_families={"stats_summary": None, "cell_plan": None}) - - timestamp = datetime.datetime(2019, 5, 1) - timestamp_minus_hr = datetime.datetime(2019, 5, 1) - datetime.timedelta(hours=1) - - row_keys = [ - "phone#4c410523#20190501", - "phone#4c410523#20190502", - "phone#4c410523#20190505", - "phone#5c10102#20190501", - "phone#5c10102#20190502", - ] - - rows = [table.direct_row(row_key) for row_key in row_keys] - - rows[0].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[0].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[0].set_cell("stats_summary", "os_build", "PQ2A.190405.003", timestamp) - rows[0].set_cell("cell_plan", "data_plan_01gb", "true", timestamp_minus_hr) - rows[0].set_cell("cell_plan", "data_plan_01gb", "false", timestamp) - rows[0].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) - rows[1].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[1].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[1].set_cell("stats_summary", "os_build", "PQ2A.190405.004", timestamp) - rows[1].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) - rows[2].set_cell("stats_summary", "connected_cell", 0, timestamp) - rows[2].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[2].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - rows[2].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) - rows[3].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[3].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[3].set_cell("stats_summary", "os_build", "PQ2A.190401.002", timestamp) - rows[3].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) - rows[4].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[4].set_cell("stats_summary", "connected_wifi", 0, timestamp) - rows[4].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - rows[4].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) - - table.mutate_rows(rows) - - # Ensure mutations have propagated. - row_set = RowSet() - - for row_key in row_keys: - row_set.add_row_key(row_key) - - fetched = list(table.read_rows(row_set=row_set)) - - while len(fetched) < len(rows): - time.sleep(5) + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"stats_summary": None, "cell_plan": None}, verbose=False) as table: + timestamp = datetime.datetime(2019, 5, 1) + timestamp_minus_hr = datetime.datetime(2019, 5, 1) - datetime.timedelta(hours=1) + + row_keys = [ + "phone#4c410523#20190501", + "phone#4c410523#20190502", + "phone#4c410523#20190505", + "phone#5c10102#20190501", + "phone#5c10102#20190502", + ] + + rows = [table.direct_row(row_key) for row_key in row_keys] + + rows[0].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[0].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[0].set_cell("stats_summary", "os_build", "PQ2A.190405.003", timestamp) + rows[0].set_cell("cell_plan", "data_plan_01gb", "true", timestamp_minus_hr) + rows[0].set_cell("cell_plan", "data_plan_01gb", "false", timestamp) + rows[0].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) + rows[1].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[1].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[1].set_cell("stats_summary", "os_build", "PQ2A.190405.004", timestamp) + rows[1].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) + rows[2].set_cell("stats_summary", "connected_cell", 0, timestamp) + rows[2].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[2].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) + rows[2].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) + rows[3].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[3].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[3].set_cell("stats_summary", "os_build", "PQ2A.190401.002", timestamp) + rows[3].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) + rows[4].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[4].set_cell("stats_summary", "connected_wifi", 0, timestamp) + rows[4].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) + rows[4].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) + + table.mutate_rows(rows) + + # Ensure mutations have propagated. + row_set = RowSet() + + for row_key in row_keys: + row_set.add_row_key(row_key) + fetched = list(table.read_rows(row_set=row_set)) - yield table_id - table.delete() + while len(fetched) < len(rows): + time.sleep(5) + fetched = list(table.read_rows(row_set=row_set)) + + yield TABLE_ID def assert_output_match(capsys, expected): @@ -135,6 +125,8 @@ def test_delete_column_family(capsys, table_id): assert_output_match(capsys, "") -def test_delete_table(capsys, table_id): - deletes_snippets.delete_table(PROJECT, BIGTABLE_INSTANCE, table_id) - assert_output_match(capsys, "") +def test_delete_table(capsys): + delete_table_id = "to-delete-table" + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, delete_table_id, verbose=False): + deletes_snippets.delete_table(PROJECT, BIGTABLE_INSTANCE, delete_table_id) + assert_output_match(capsys, "") diff --git a/samples/snippets/filters/filter_snippets_async_test.py b/samples/snippets/filters/filter_snippets_async_test.py index 6e3d67835..235e64ef8 100644 --- a/samples/snippets/filters/filter_snippets_async_test.py +++ b/samples/snippets/filters/filter_snippets_async_test.py @@ -14,7 +14,6 @@ import datetime import os -import time import inspect from typing import AsyncGenerator @@ -24,49 +23,21 @@ from .snapshots.snap_filters_test import snapshots from . import filter_snippets_async +from ...utils import create_table_cm from google.cloud._helpers import ( _microseconds_from_datetime, ) -from google.api_core.exceptions import PreconditionFailed PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] TABLE_ID = "mobile-time-series-filters-async" -@pytest_asyncio.fixture +@pytest_asyncio.fixture(scope="module", autouse=True) async def table_id() -> AsyncGenerator[str, None]: - table, table_id = _create_table() - await _populate_table(table_id) - yield table_id - table.delete() - - -def _create_table(): - from google.cloud import bigtable - - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - - table_id = TABLE_ID - table = instance.table(table_id) - if table.exists(): - table.delete() - - table.create(column_families={"stats_summary": None, "cell_plan": None}) - - # let table creation complete - attempts = 0 - table_ready = False - while not table_ready and attempts < 10: - try: - table_ready = table.exists() - except PreconditionFailed: - print("Waiting for table to become ready...") - attempts += 1 - time.sleep(5) - - return table, table_id + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"stats_summary": None, "cell_plan": None}): + await _populate_table(TABLE_ID) + yield TABLE_ID async def _populate_table(table_id): diff --git a/samples/snippets/filters/filters_test.py b/samples/snippets/filters/filters_test.py index ec126b170..00f8ade37 100644 --- a/samples/snippets/filters/filters_test.py +++ b/samples/snippets/filters/filters_test.py @@ -17,11 +17,11 @@ import os import time -from google.cloud import bigtable import pytest from . import filter_snippets from .snapshots.snap_filters_test import snapshots +from ...utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] @@ -32,69 +32,60 @@ def table_id(): from google.cloud.bigtable.row_set import RowSet - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table_id = TABLE_ID - table = instance.table(table_id) - if table.exists(): - table.delete() - - table.create(column_families={"stats_summary": None, "cell_plan": None}) - - timestamp = datetime.datetime(2019, 5, 1) - timestamp_minus_hr = datetime.datetime(2019, 5, 1) - datetime.timedelta(hours=1) - - row_keys = [ - "phone#4c410523#20190501", - "phone#4c410523#20190502", - "phone#4c410523#20190505", - "phone#5c10102#20190501", - "phone#5c10102#20190502", - ] - - rows = [table.direct_row(row_key) for row_key in row_keys] - - rows[0].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[0].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[0].set_cell("stats_summary", "os_build", "PQ2A.190405.003", timestamp) - rows[0].set_cell("cell_plan", "data_plan_01gb", "true", timestamp_minus_hr) - rows[0].set_cell("cell_plan", "data_plan_01gb", "false", timestamp) - rows[0].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) - rows[1].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[1].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[1].set_cell("stats_summary", "os_build", "PQ2A.190405.004", timestamp) - rows[1].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) - rows[2].set_cell("stats_summary", "connected_cell", 0, timestamp) - rows[2].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[2].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - rows[2].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) - rows[3].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[3].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[3].set_cell("stats_summary", "os_build", "PQ2A.190401.002", timestamp) - rows[3].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) - rows[4].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[4].set_cell("stats_summary", "connected_wifi", 0, timestamp) - rows[4].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - rows[4].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) - - table.mutate_rows(rows) - - # Ensure mutations have propagated. - row_set = RowSet() - - for row_key in row_keys: - row_set.add_row_key(row_key) - - fetched = list(table.read_rows(row_set=row_set)) - - while len(fetched) < len(rows): - time.sleep(5) + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, table_id, {"stats_summary": None, "cell_plan": None}) as table: + + timestamp = datetime.datetime(2019, 5, 1) + timestamp_minus_hr = datetime.datetime(2019, 5, 1) - datetime.timedelta(hours=1) + + row_keys = [ + "phone#4c410523#20190501", + "phone#4c410523#20190502", + "phone#4c410523#20190505", + "phone#5c10102#20190501", + "phone#5c10102#20190502", + ] + + rows = [table.direct_row(row_key) for row_key in row_keys] + + rows[0].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[0].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[0].set_cell("stats_summary", "os_build", "PQ2A.190405.003", timestamp) + rows[0].set_cell("cell_plan", "data_plan_01gb", "true", timestamp_minus_hr) + rows[0].set_cell("cell_plan", "data_plan_01gb", "false", timestamp) + rows[0].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) + rows[1].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[1].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[1].set_cell("stats_summary", "os_build", "PQ2A.190405.004", timestamp) + rows[1].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) + rows[2].set_cell("stats_summary", "connected_cell", 0, timestamp) + rows[2].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[2].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) + rows[2].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) + rows[3].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[3].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[3].set_cell("stats_summary", "os_build", "PQ2A.190401.002", timestamp) + rows[3].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) + rows[4].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[4].set_cell("stats_summary", "connected_wifi", 0, timestamp) + rows[4].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) + rows[4].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) + + table.mutate_rows(rows) + + # Ensure mutations have propagated. + row_set = RowSet() + + for row_key in row_keys: + row_set.add_row_key(row_key) + fetched = list(table.read_rows(row_set=row_set)) - yield table_id + while len(fetched) < len(rows): + time.sleep(5) + fetched = list(table.read_rows(row_set=row_set)) - table.delete() + yield table_id def test_filter_limit_row_sample(capsys, table_id): diff --git a/samples/snippets/reads/reads_test.py b/samples/snippets/reads/reads_test.py index e6f9febbe..8aad1d450 100644 --- a/samples/snippets/reads/reads_test.py +++ b/samples/snippets/reads/reads_test.py @@ -15,11 +15,11 @@ import os import inspect -from google.cloud import bigtable import pytest from .snapshots.snap_reads_test import snapshots from . import read_snippets +from ...utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] @@ -29,48 +29,35 @@ @pytest.fixture(scope="module", autouse=True) def table_id(): - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - - table_id = TABLE_ID - table = instance.table(table_id) - if table.exists(): - table.delete() - - table.create(column_families={"stats_summary": None}) - - # table = instance.table(table_id) - - timestamp = datetime.datetime(2019, 5, 1) - rows = [ - table.direct_row("phone#4c410523#20190501"), - table.direct_row("phone#4c410523#20190502"), - table.direct_row("phone#4c410523#20190505"), - table.direct_row("phone#5c10102#20190501"), - table.direct_row("phone#5c10102#20190502"), - ] - - rows[0].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[0].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[0].set_cell("stats_summary", "os_build", "PQ2A.190405.003", timestamp) - rows[1].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[1].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[1].set_cell("stats_summary", "os_build", "PQ2A.190405.004", timestamp) - rows[2].set_cell("stats_summary", "connected_cell", 0, timestamp) - rows[2].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[2].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - rows[3].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[3].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[3].set_cell("stats_summary", "os_build", "PQ2A.190401.002", timestamp) - rows[4].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[4].set_cell("stats_summary", "connected_wifi", 0, timestamp) - rows[4].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - - table.mutate_rows(rows) - - yield table_id - - table.delete() + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"stats_summary": None}) as table: + timestamp = datetime.datetime(2019, 5, 1) + rows = [ + table.direct_row("phone#4c410523#20190501"), + table.direct_row("phone#4c410523#20190502"), + table.direct_row("phone#4c410523#20190505"), + table.direct_row("phone#5c10102#20190501"), + table.direct_row("phone#5c10102#20190502"), + ] + + rows[0].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[0].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[0].set_cell("stats_summary", "os_build", "PQ2A.190405.003", timestamp) + rows[1].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[1].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[1].set_cell("stats_summary", "os_build", "PQ2A.190405.004", timestamp) + rows[2].set_cell("stats_summary", "connected_cell", 0, timestamp) + rows[2].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[2].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) + rows[3].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[3].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[3].set_cell("stats_summary", "os_build", "PQ2A.190401.002", timestamp) + rows[4].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[4].set_cell("stats_summary", "connected_wifi", 0, timestamp) + rows[4].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) + + table.mutate_rows(rows) + + yield TABLE_ID def test_read_row(capsys, table_id): diff --git a/samples/snippets/writes/writes_test.py b/samples/snippets/writes/writes_test.py index 5d7219dc4..6729ae9b9 100644 --- a/samples/snippets/writes/writes_test.py +++ b/samples/snippets/writes/writes_test.py @@ -16,14 +16,13 @@ import backoff from google.api_core.exceptions import DeadlineExceeded -from google.cloud import bigtable import pytest from .write_batch import write_batch from .write_conditionally import write_conditional from .write_increment import write_increment from .write_simple import write_simple - +from ...utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] @@ -31,29 +30,9 @@ @pytest.fixture -def bigtable_client(): - return bigtable.Client(project=PROJECT, admin=True) - - -@pytest.fixture -def bigtable_instance(bigtable_client): - return bigtable_client.instance(BIGTABLE_INSTANCE) - - -@pytest.fixture -def table_id(bigtable_instance): - table_id = TABLE_ID - table = bigtable_instance.table(table_id) - if table.exists(): - table.delete() - - column_family_id = "stats_summary" - column_families = {column_family_id: None} - table.create(column_families=column_families) - - yield table_id - - table.delete() +def table_id(): + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"stats_summary": None}): + yield TABLE_ID def test_writes(capsys, table_id): diff --git a/samples/tableadmin/__init__.py b/samples/tableadmin/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/tableadmin/tableadmin.py b/samples/tableadmin/tableadmin.py index 7c28601fb..ad00e5788 100644 --- a/samples/tableadmin/tableadmin.py +++ b/samples/tableadmin/tableadmin.py @@ -35,36 +35,7 @@ from google.cloud import bigtable from google.cloud.bigtable import column_family - - -def create_table(project_id, instance_id, table_id): - """Create a Bigtable table - - :type project_id: str - :param project_id: Project id of the client. - - :type instance_id: str - :param instance_id: Instance of the client. - - :type table_id: str - :param table_id: Table id to create table. - """ - - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - # Check whether table exists in an instance. - # Create table if it does not exists. - print("Checking if table {} exists...".format(table_id)) - if table.exists(): - print("Table {} already exists.".format(table_id)) - else: - print("Creating the {} table.".format(table_id)) - table.create() - print("Created table {}.".format(table_id)) - - return client, instance, table +from ..utils import create_table_cm def run_table_operations(project_id, instance_id, table_id): @@ -80,154 +51,155 @@ def run_table_operations(project_id, instance_id, table_id): :param table_id: Table id to create table. """ - client, instance, table = create_table(project_id, instance_id, table_id) + client = bigtable.Client(project=project_id, admin=True) + instance = client.instance(instance_id) + with create_table_cm(project_id, instance_id, table_id, verbose=False) as table: + # [START bigtable_list_tables] + tables = instance.list_tables() + print("Listing tables in current project...") + if tables != []: + for tbl in tables: + print(tbl.table_id) + else: + print("No table exists in current project...") + # [END bigtable_list_tables] + + # [START bigtable_create_family_gc_max_age] + print("Creating column family cf1 with with MaxAge GC Rule...") + # Create a column family with GC policy : maximum age + # where age = current time minus cell timestamp + + # Define the GC rule to retain data with max age of 5 days + max_age_rule = column_family.MaxAgeGCRule(datetime.timedelta(days=5)) + + column_family1 = table.column_family("cf1", max_age_rule) + column_family1.create() + print("Created column family cf1 with MaxAge GC Rule.") + # [END bigtable_create_family_gc_max_age] + + # [START bigtable_create_family_gc_max_versions] + print("Creating column family cf2 with max versions GC rule...") + # Create a column family with GC policy : most recent N versions + # where 1 = most recent version + + # Define the GC policy to retain only the most recent 2 versions + max_versions_rule = column_family.MaxVersionsGCRule(2) + + column_family2 = table.column_family("cf2", max_versions_rule) + column_family2.create() + print("Created column family cf2 with Max Versions GC Rule.") + # [END bigtable_create_family_gc_max_versions] + + # [START bigtable_create_family_gc_union] + print("Creating column family cf3 with union GC rule...") + # Create a column family with GC policy to drop data that matches + # at least one condition. + # Define a GC rule to drop cells older than 5 days or not the + # most recent version + union_rule = column_family.GCRuleUnion( + [ + column_family.MaxAgeGCRule(datetime.timedelta(days=5)), + column_family.MaxVersionsGCRule(2), + ] + ) - # [START bigtable_list_tables] - tables = instance.list_tables() - print("Listing tables in current project...") - if tables != []: - for tbl in tables: - print(tbl.table_id) - else: - print("No table exists in current project...") - # [END bigtable_list_tables] - - # [START bigtable_create_family_gc_max_age] - print("Creating column family cf1 with with MaxAge GC Rule...") - # Create a column family with GC policy : maximum age - # where age = current time minus cell timestamp - - # Define the GC rule to retain data with max age of 5 days - max_age_rule = column_family.MaxAgeGCRule(datetime.timedelta(days=5)) - - column_family1 = table.column_family("cf1", max_age_rule) - column_family1.create() - print("Created column family cf1 with MaxAge GC Rule.") - # [END bigtable_create_family_gc_max_age] - - # [START bigtable_create_family_gc_max_versions] - print("Creating column family cf2 with max versions GC rule...") - # Create a column family with GC policy : most recent N versions - # where 1 = most recent version - - # Define the GC policy to retain only the most recent 2 versions - max_versions_rule = column_family.MaxVersionsGCRule(2) - - column_family2 = table.column_family("cf2", max_versions_rule) - column_family2.create() - print("Created column family cf2 with Max Versions GC Rule.") - # [END bigtable_create_family_gc_max_versions] - - # [START bigtable_create_family_gc_union] - print("Creating column family cf3 with union GC rule...") - # Create a column family with GC policy to drop data that matches - # at least one condition. - # Define a GC rule to drop cells older than 5 days or not the - # most recent version - union_rule = column_family.GCRuleUnion( - [ - column_family.MaxAgeGCRule(datetime.timedelta(days=5)), - column_family.MaxVersionsGCRule(2), - ] - ) + column_family3 = table.column_family("cf3", union_rule) + column_family3.create() + print("Created column family cf3 with Union GC rule") + # [END bigtable_create_family_gc_union] + + # [START bigtable_create_family_gc_intersection] + print("Creating column family cf4 with Intersection GC rule...") + # Create a column family with GC policy to drop data that matches + # all conditions + # GC rule: Drop cells older than 5 days AND older than the most + # recent 2 versions + intersection_rule = column_family.GCRuleIntersection( + [ + column_family.MaxAgeGCRule(datetime.timedelta(days=5)), + column_family.MaxVersionsGCRule(2), + ] + ) - column_family3 = table.column_family("cf3", union_rule) - column_family3.create() - print("Created column family cf3 with Union GC rule") - # [END bigtable_create_family_gc_union] - - # [START bigtable_create_family_gc_intersection] - print("Creating column family cf4 with Intersection GC rule...") - # Create a column family with GC policy to drop data that matches - # all conditions - # GC rule: Drop cells older than 5 days AND older than the most - # recent 2 versions - intersection_rule = column_family.GCRuleIntersection( - [ - column_family.MaxAgeGCRule(datetime.timedelta(days=5)), - column_family.MaxVersionsGCRule(2), - ] - ) + column_family4 = table.column_family("cf4", intersection_rule) + column_family4.create() + print("Created column family cf4 with Intersection GC rule.") + # [END bigtable_create_family_gc_intersection] + + # [START bigtable_create_family_gc_nested] + print("Creating column family cf5 with a Nested GC rule...") + # Create a column family with nested GC policies. + # Create a nested GC rule: + # Drop cells that are either older than the 10 recent versions + # OR + # Drop cells that are older than a month AND older than the + # 2 recent versions + rule1 = column_family.MaxVersionsGCRule(10) + rule2 = column_family.GCRuleIntersection( + [ + column_family.MaxAgeGCRule(datetime.timedelta(days=30)), + column_family.MaxVersionsGCRule(2), + ] + ) - column_family4 = table.column_family("cf4", intersection_rule) - column_family4.create() - print("Created column family cf4 with Intersection GC rule.") - # [END bigtable_create_family_gc_intersection] - - # [START bigtable_create_family_gc_nested] - print("Creating column family cf5 with a Nested GC rule...") - # Create a column family with nested GC policies. - # Create a nested GC rule: - # Drop cells that are either older than the 10 recent versions - # OR - # Drop cells that are older than a month AND older than the - # 2 recent versions - rule1 = column_family.MaxVersionsGCRule(10) - rule2 = column_family.GCRuleIntersection( - [ - column_family.MaxAgeGCRule(datetime.timedelta(days=30)), - column_family.MaxVersionsGCRule(2), - ] - ) + nested_rule = column_family.GCRuleUnion([rule1, rule2]) + + column_family5 = table.column_family("cf5", nested_rule) + column_family5.create() + print("Created column family cf5 with a Nested GC rule.") + # [END bigtable_create_family_gc_nested] + + # [START bigtable_list_column_families] + print("Printing Column Family and GC Rule for all column families...") + column_families = table.list_column_families() + for column_family_name, gc_rule in sorted(column_families.items()): + print("Column Family:", column_family_name) + print("GC Rule:") + print(gc_rule.to_pb()) + # Sample output: + # Column Family: cf4 + # GC Rule: + # gc_rule { + # intersection { + # rules { + # max_age { + # seconds: 432000 + # } + # } + # rules { + # max_num_versions: 2 + # } + # } + # } + # [END bigtable_list_column_families] + + print("Print column family cf1 GC rule before update...") + print("Column Family: cf1") + print(column_family1.to_pb()) + + # [START bigtable_update_gc_rule] + print("Updating column family cf1 GC rule...") + # Update the column family cf1 to update the GC rule + column_family1 = table.column_family("cf1", column_family.MaxVersionsGCRule(1)) + column_family1.update() + print("Updated column family cf1 GC rule\n") + # [END bigtable_update_gc_rule] + + print("Print column family cf1 GC rule after update...") + print("Column Family: cf1") + print(column_family1.to_pb()) + + # [START bigtable_delete_family] + print("Delete a column family cf2...") + # Delete a column family + column_family2.delete() + print("Column family cf2 deleted successfully.") + # [END bigtable_delete_family] - nested_rule = column_family.GCRuleUnion([rule1, rule2]) - - column_family5 = table.column_family("cf5", nested_rule) - column_family5.create() - print("Created column family cf5 with a Nested GC rule.") - # [END bigtable_create_family_gc_nested] - - # [START bigtable_list_column_families] - print("Printing Column Family and GC Rule for all column families...") - column_families = table.list_column_families() - for column_family_name, gc_rule in sorted(column_families.items()): - print("Column Family:", column_family_name) - print("GC Rule:") - print(gc_rule.to_pb()) - # Sample output: - # Column Family: cf4 - # GC Rule: - # gc_rule { - # intersection { - # rules { - # max_age { - # seconds: 432000 - # } - # } - # rules { - # max_num_versions: 2 - # } - # } - # } - # [END bigtable_list_column_families] - - print("Print column family cf1 GC rule before update...") - print("Column Family: cf1") - print(column_family1.to_pb()) - - # [START bigtable_update_gc_rule] - print("Updating column family cf1 GC rule...") - # Update the column family cf1 to update the GC rule - column_family1 = table.column_family("cf1", column_family.MaxVersionsGCRule(1)) - column_family1.update() - print("Updated column family cf1 GC rule\n") - # [END bigtable_update_gc_rule] - - print("Print column family cf1 GC rule after update...") - print("Column Family: cf1") - print(column_family1.to_pb()) - - # [START bigtable_delete_family] - print("Delete a column family cf2...") - # Delete a column family - column_family2.delete() - print("Column family cf2 deleted successfully.") - # [END bigtable_delete_family] - - print( - 'execute command "python tableadmin.py delete [project_id] \ - [instance_id] --table [tableName]" to delete the table.' - ) + print( + 'execute command "python tableadmin.py delete [project_id] \ + [instance_id] --table [tableName]" to delete the table.' + ) def delete_table(project_id, instance_id, table_id): diff --git a/samples/tableadmin/tableadmin_test.py b/samples/tableadmin/tableadmin_test.py index cc630cda9..b1b31d86d 100755 --- a/samples/tableadmin/tableadmin_test.py +++ b/samples/tableadmin/tableadmin_test.py @@ -14,55 +14,45 @@ # limitations under the License. import os -import uuid - -from google.api_core import exceptions from test_utils.retry import RetryErrors +from google.api_core import exceptions -from tableadmin import create_table -from tableadmin import delete_table -from tableadmin import run_table_operations +from .tableadmin import delete_table +from .tableadmin import run_table_operations +from ..utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_FORMAT = "tableadmin-test-{}" +TABLE_ID = "tableadmin-test" retry_429_503 = RetryErrors(exceptions.TooManyRequests, exceptions.ServiceUnavailable) def test_run_table_operations(capsys): - try: - table_id = TABLE_ID_FORMAT.format(uuid.uuid4().hex[:8]) - - retry_429_503(run_table_operations)(PROJECT, BIGTABLE_INSTANCE, table_id) - out, _ = capsys.readouterr() + retry_429_503(run_table_operations)(PROJECT, BIGTABLE_INSTANCE, TABLE_ID) + out, _ = capsys.readouterr() - assert "Creating the " + table_id + " table." in out - assert "Listing tables in current project." in out - assert "Creating column family cf1 with with MaxAge GC Rule" in out - assert "Created column family cf1 with MaxAge GC Rule." in out - assert "Created column family cf2 with Max Versions GC Rule." in out - assert "Created column family cf3 with Union GC rule" in out - assert "Created column family cf4 with Intersection GC rule." in out - assert "Created column family cf5 with a Nested GC rule." in out - assert "Printing Column Family and GC Rule for all column families." in out - assert "Updating column family cf1 GC rule..." in out - assert "Updated column family cf1 GC rule" in out - assert "Print column family cf1 GC rule after update..." in out - assert "Column Family: cf1" in out - assert "max_num_versions: 1" in out - assert "Delete a column family cf2..." in out - assert "Column family cf2 deleted successfully." in out - finally: - retry_429_503(delete_table)(PROJECT, BIGTABLE_INSTANCE, table_id) + assert "Listing tables in current project." in out + assert "Creating column family cf1 with with MaxAge GC Rule" in out + assert "Created column family cf1 with MaxAge GC Rule." in out + assert "Created column family cf2 with Max Versions GC Rule." in out + assert "Created column family cf3 with Union GC rule" in out + assert "Created column family cf4 with Intersection GC rule." in out + assert "Created column family cf5 with a Nested GC rule." in out + assert "Printing Column Family and GC Rule for all column families." in out + assert "Updating column family cf1 GC rule..." in out + assert "Updated column family cf1 GC rule" in out + assert "Print column family cf1 GC rule after update..." in out + assert "Column Family: cf1" in out + assert "max_num_versions: 1" in out + assert "Delete a column family cf2..." in out + assert "Column family cf2 deleted successfully." in out def test_delete_table(capsys): - table_id = TABLE_ID_FORMAT.format(uuid.uuid4().hex[:8]) - try: - retry_429_503(create_table)(PROJECT, BIGTABLE_INSTANCE, table_id) - finally: - retry_429_503(delete_table)(PROJECT, BIGTABLE_INSTANCE, table_id) + table_id = "table-admin-to-delete" + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, table_id, verbose=False): + delete_table(PROJECT, BIGTABLE_INSTANCE, table_id) out, _ = capsys.readouterr() assert "Table " + table_id + " exists." in out diff --git a/samples/utils.py b/samples/utils.py new file mode 100644 index 000000000..70858f38f --- /dev/null +++ b/samples/utils.py @@ -0,0 +1,70 @@ +# Copyright 2024, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from google.cloud import bigtable +from google.api_core import exceptions +from google.api_core.retry import Retry +from google.api_core.retry import if_exception_type + +delete_retry = Retry(if_exception_type(exceptions.TooManyRequests, exceptions.ServiceUnavailable)) + +class create_table_cm: + def __init__(self, *args, verbose=True, **kwargs): + self._args = args + self._kwargs = kwargs + self._verbose = verbose + + def __enter__(self): + self._table = create_table(*self._args, **self._kwargs) + if self._verbose: + print(f"created table: {self._table.table_id}") + return self._table + + def __exit__(self, *args): + if self._table.exists(): + if self._verbose: + print(f"deleting table: {self._table.table_id}") + delete_retry(self._table.delete()) + else: + if self._verbose: + print(f"table {self._table.table_id} not found") + + +def create_table(project, instance_id, table_id, column_families={}): + client = bigtable.Client(project=project, admin=True) + instance = client.instance(instance_id) + + table = instance.table(table_id) + if table.exists(): + table.delete() + + kwargs = {} + if column_families: + kwargs["column_families"] = column_families + table.create(**kwargs) + + wait_for_table(table) + + return table + +@Retry( + on_error=if_exception_type( + exceptions.PreconditionFailed, + exceptions.FailedPrecondition, + exceptions.NotFound, + ) +) +def wait_for_table(table): + if not table.exists(): + raise exceptions.NotFound \ No newline at end of file From db75791cc077093eaf831150aefc96280bd4c7fb Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 7 Nov 2024 20:43:25 +0000 Subject: [PATCH 08/12] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .kokoro/test-samples-impl.sh | 3 +-- samples/beam/noxfile.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh index 53e365bc4..55910c8ba 100755 --- a/.kokoro/test-samples-impl.sh +++ b/.kokoro/test-samples-impl.sh @@ -33,8 +33,7 @@ export PYTHONUNBUFFERED=1 env | grep KOKORO # Install nox -# `virtualenv==20.26.6` is added for Python 3.7 compatibility -python3.9 -m pip install --upgrade --quiet nox virtualenv==20.26.6 +python3.9 -m pip install --upgrade --quiet nox # Use secrets acessor service account to get secrets if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then diff --git a/samples/beam/noxfile.py b/samples/beam/noxfile.py index 8ae94fd44..80ffdb178 100644 --- a/samples/beam/noxfile.py +++ b/samples/beam/noxfile.py @@ -22,6 +22,7 @@ import nox + # WARNING - WARNING - WARNING - WARNING - WARNING # WARNING - WARNING - WARNING - WARNING - WARNING # DO NOT EDIT THIS FILE EVER! @@ -157,7 +158,6 @@ def blacken(session: nox.sessions.Session) -> None: # format = isort + black # - @nox.session def format(session: nox.sessions.Session) -> None: """ @@ -185,9 +185,7 @@ def _session_tests( session: nox.sessions.Session, post_install: Callable = None ) -> None: # check for presence of tests - test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob( - "**/test_*.py", recursive=True - ) + test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob("**/test_*.py", recursive=True) test_list.extend(glob.glob("**/tests", recursive=True)) if len(test_list) == 0: @@ -209,7 +207,9 @@ def _session_tests( if os.path.exists("requirements-test.txt"): if os.path.exists("constraints-test.txt"): - session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") + session.install( + "-r", "requirements-test.txt", "-c", "constraints-test.txt" + ) else: session.install("-r", "requirements-test.txt") with open("requirements-test.txt") as rtfile: @@ -222,9 +222,9 @@ def _session_tests( post_install(session) if "pytest-parallel" in packages: - concurrent_args.extend(["--workers", "auto", "--tests-per-worker", "auto"]) + concurrent_args.extend(['--workers', 'auto', '--tests-per-worker', 'auto']) elif "pytest-xdist" in packages: - concurrent_args.extend(["-n", "auto"]) + concurrent_args.extend(['-n', 'auto']) session.run( "pytest", @@ -254,7 +254,7 @@ def py(session: nox.sessions.Session) -> None: def _get_repo_root() -> Optional[str]: - """Returns the root folder of the project.""" + """ Returns the root folder of the project. """ # Get root of this repository. Assume we don't have directories nested deeper than 10 items. p = Path(os.getcwd()) for i in range(10): From b65b7847d5ac90ed74390d3854200020b25def97 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 7 Nov 2024 13:44:46 -0800 Subject: [PATCH 09/12] fixed table reference --- samples/hello_happybase/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/hello_happybase/main.py b/samples/hello_happybase/main.py index 570f36b63..50820febd 100644 --- a/samples/hello_happybase/main.py +++ b/samples/hello_happybase/main.py @@ -52,7 +52,7 @@ def main(project_id, instance_id, table_name): ) # [END bigtable_hw_create_table_happybase] - wait_for_table(connection.table(table_name)) + wait_for_table(instance.table(table_name)) # [START bigtable_hw_write_rows_happybase] print("Writing some greetings to the table.") From 764a40383f169f0c8251d8bb84516f67c9258e72 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 7 Nov 2024 13:48:03 -0800 Subject: [PATCH 10/12] added back uuids to test tables --- samples/beam/hello_world_write_test.py | 3 ++- samples/hello/async_main_test.py | 3 ++- samples/hello/main_test.py | 3 ++- samples/hello_happybase/main_test.py | 3 ++- samples/quickstart/main_async_test.py | 3 ++- samples/quickstart/main_test.py | 4 ++-- samples/quickstart_happybase/main_test.py | 4 ++-- .../snippets/data_client/data_client_snippets_async_test.py | 3 ++- samples/snippets/deletes/deletes_async_test.py | 3 ++- samples/snippets/deletes/deletes_test.py | 5 +++-- samples/snippets/filters/filter_snippets_async_test.py | 3 ++- samples/snippets/filters/filters_test.py | 3 ++- samples/snippets/reads/reads_test.py | 3 ++- samples/snippets/writes/writes_test.py | 3 ++- samples/tableadmin/tableadmin_test.py | 5 +++-- 15 files changed, 32 insertions(+), 19 deletions(-) diff --git a/samples/beam/hello_world_write_test.py b/samples/beam/hello_world_write_test.py index 66c9764e2..ba0e98096 100644 --- a/samples/beam/hello_world_write_test.py +++ b/samples/beam/hello_world_write_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os +import uuid import pytest @@ -20,7 +21,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID = "mobile-time-series-beam" +TABLE_ID = f"mobile-time-series-beam-{str(uuid.uuid4())[:16]}" @pytest.fixture(scope="module", autouse=True) diff --git a/samples/hello/async_main_test.py b/samples/hello/async_main_test.py index 5b005b05b..aa65a8652 100644 --- a/samples/hello/async_main_test.py +++ b/samples/hello/async_main_test.py @@ -14,12 +14,13 @@ import os import asyncio +import uuid from .async_main import main PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID = "hello-world-test-async" +TABLE_ID = f"hello-world-test-async-{str(uuid.uuid4())[:16]}" def test_async_main(capsys): diff --git a/samples/hello/main_test.py b/samples/hello/main_test.py index e6ca87c83..28814d909 100644 --- a/samples/hello/main_test.py +++ b/samples/hello/main_test.py @@ -13,12 +13,13 @@ # limitations under the License. import os +import uuid from .main import main PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID = "hello-world-test" +TABLE_ID = f"hello-world-test-{str(uuid.uuid4())[:16]}" def test_main(capsys): diff --git a/samples/hello_happybase/main_test.py b/samples/hello_happybase/main_test.py index a950ec382..252f4ccaf 100644 --- a/samples/hello_happybase/main_test.py +++ b/samples/hello_happybase/main_test.py @@ -13,13 +13,14 @@ # limitations under the License. import os +import uuid from .main import main from google.cloud import bigtable PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID = "hello-world-hb-test" +TABLE_ID = f"hello-world-hb-test-{str(uuid.uuid4())[:16]}" def test_main(capsys): diff --git a/samples/quickstart/main_async_test.py b/samples/quickstart/main_async_test.py index 6f179a7c2..0749cbd31 100644 --- a/samples/quickstart/main_async_test.py +++ b/samples/quickstart/main_async_test.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import uuid from typing import AsyncGenerator from google.cloud.bigtable.data import BigtableDataClientAsync, SetCell @@ -24,7 +25,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID = "quickstart-async-test" +TABLE_ID = f"quickstart-async-test-{str(uuid.uuid4())[:16]}" @pytest_asyncio.fixture diff --git a/samples/quickstart/main_test.py b/samples/quickstart/main_test.py index 53a339548..f58161f23 100644 --- a/samples/quickstart/main_test.py +++ b/samples/quickstart/main_test.py @@ -13,7 +13,7 @@ # limitations under the License. import os - +import uuid import pytest from .main import main @@ -23,7 +23,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID = "quickstart-test" +TABLE_ID = f"quickstart-test-{str(uuid.uuid4())[:16]}" @pytest.fixture() diff --git a/samples/quickstart_happybase/main_test.py b/samples/quickstart_happybase/main_test.py index 9d2b040a0..343ec800a 100644 --- a/samples/quickstart_happybase/main_test.py +++ b/samples/quickstart_happybase/main_test.py @@ -13,7 +13,7 @@ # limitations under the License. import os - +import uuid import pytest from .main import main @@ -21,7 +21,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID = "quickstart-hb-test" +TABLE_ID = f"quickstart-hb-test-{str(uuid.uuid4())[:16]}" @pytest.fixture() diff --git a/samples/snippets/data_client/data_client_snippets_async_test.py b/samples/snippets/data_client/data_client_snippets_async_test.py index 467986170..8dfff50d1 100644 --- a/samples/snippets/data_client/data_client_snippets_async_test.py +++ b/samples/snippets/data_client/data_client_snippets_async_test.py @@ -13,6 +13,7 @@ import pytest import pytest_asyncio import os +import uuid from . import data_client_snippets_async as data_snippets from ...utils import create_table_cm @@ -20,7 +21,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID = "data-client" +TABLE_ID = f"data-client-{str(uuid.uuid4())[:16]}" @pytest.fixture(scope="session") diff --git a/samples/snippets/deletes/deletes_async_test.py b/samples/snippets/deletes/deletes_async_test.py index 8773e1fcc..9408a8320 100644 --- a/samples/snippets/deletes/deletes_async_test.py +++ b/samples/snippets/deletes/deletes_async_test.py @@ -15,6 +15,7 @@ import datetime import os +import uuid from typing import AsyncGenerator from google.cloud._helpers import _microseconds_from_datetime @@ -26,7 +27,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID = "mobile-time-series-deletes-async" +TABLE_ID = f"mobile-time-series-deletes-async-{str(uuid.uuid4())[:16]}" @pytest_asyncio.fixture(scope="module", autouse=True) diff --git a/samples/snippets/deletes/deletes_test.py b/samples/snippets/deletes/deletes_test.py index 9c0419b58..3284c37da 100644 --- a/samples/snippets/deletes/deletes_test.py +++ b/samples/snippets/deletes/deletes_test.py @@ -16,6 +16,7 @@ import datetime import os import time +import uuid import pytest @@ -24,7 +25,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID = "mobile-time-series-deletes" +TABLE_ID = f"mobile-time-series-deletes-{str(uuid.uuid4())[:16]}" @pytest.fixture(scope="module") @@ -126,7 +127,7 @@ def test_delete_column_family(capsys, table_id): def test_delete_table(capsys): - delete_table_id = "to-delete-table" + delete_table_id = f"to-delete-table-{str(uuid.uuid4())[:16]}" with create_table_cm(PROJECT, BIGTABLE_INSTANCE, delete_table_id, verbose=False): deletes_snippets.delete_table(PROJECT, BIGTABLE_INSTANCE, delete_table_id) assert_output_match(capsys, "") diff --git a/samples/snippets/filters/filter_snippets_async_test.py b/samples/snippets/filters/filter_snippets_async_test.py index 235e64ef8..124db8157 100644 --- a/samples/snippets/filters/filter_snippets_async_test.py +++ b/samples/snippets/filters/filter_snippets_async_test.py @@ -14,6 +14,7 @@ import datetime import os +import uuid import inspect from typing import AsyncGenerator @@ -30,7 +31,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID = "mobile-time-series-filters-async" +TABLE_ID = f"mobile-time-series-filters-async-{str(uuid.uuid4())[:16]}" @pytest_asyncio.fixture(scope="module", autouse=True) diff --git a/samples/snippets/filters/filters_test.py b/samples/snippets/filters/filters_test.py index 00f8ade37..fe99886bd 100644 --- a/samples/snippets/filters/filters_test.py +++ b/samples/snippets/filters/filters_test.py @@ -16,6 +16,7 @@ import inspect import os import time +import uuid import pytest @@ -25,7 +26,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID = "mobile-time-series-filters" +TABLE_ID = f"mobile-time-series-filters-{str(uuid.uuid4())[:16]}" @pytest.fixture(scope="module", autouse=True) diff --git a/samples/snippets/reads/reads_test.py b/samples/snippets/reads/reads_test.py index 8aad1d450..0078ce598 100644 --- a/samples/snippets/reads/reads_test.py +++ b/samples/snippets/reads/reads_test.py @@ -14,6 +14,7 @@ import datetime import os import inspect +import uuid import pytest @@ -24,7 +25,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID = "mobile-time-series-reads" +TABLE_ID = f"mobile-time-series-reads-{str(uuid.uuid4())[:16]}" @pytest.fixture(scope="module", autouse=True) diff --git a/samples/snippets/writes/writes_test.py b/samples/snippets/writes/writes_test.py index 6729ae9b9..2c7a3d62b 100644 --- a/samples/snippets/writes/writes_test.py +++ b/samples/snippets/writes/writes_test.py @@ -17,6 +17,7 @@ import backoff from google.api_core.exceptions import DeadlineExceeded import pytest +import uuid from .write_batch import write_batch from .write_conditionally import write_conditional @@ -26,7 +27,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID = "mobile-time-series-writes" +TABLE_ID = f"mobile-time-series-writes-{str(uuid.uuid4())[:16]}" @pytest.fixture diff --git a/samples/tableadmin/tableadmin_test.py b/samples/tableadmin/tableadmin_test.py index b1b31d86d..0ffdc75c9 100755 --- a/samples/tableadmin/tableadmin_test.py +++ b/samples/tableadmin/tableadmin_test.py @@ -16,6 +16,7 @@ import os from test_utils.retry import RetryErrors from google.api_core import exceptions +import uuid from .tableadmin import delete_table from .tableadmin import run_table_operations @@ -23,7 +24,7 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID = "tableadmin-test" +TABLE_ID = f"tableadmin-test-{str(uuid.uuid4())[:16]}" retry_429_503 = RetryErrors(exceptions.TooManyRequests, exceptions.ServiceUnavailable) @@ -50,7 +51,7 @@ def test_run_table_operations(capsys): def test_delete_table(capsys): - table_id = "table-admin-to-delete" + table_id = f"table-admin-to-delete-{str(uuid.uuid4())[:16]}" with create_table_cm(PROJECT, BIGTABLE_INSTANCE, table_id, verbose=False): delete_table(PROJECT, BIGTABLE_INSTANCE, table_id) out, _ = capsys.readouterr() From df8fee12c4f28c349ce23a322764117ad74c98f9 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 8 Nov 2024 10:45:47 -0800 Subject: [PATCH 11/12] added docstrings to util.py --- samples/utils.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/samples/utils.py b/samples/utils.py index 70858f38f..6bdb884c9 100644 --- a/samples/utils.py +++ b/samples/utils.py @@ -10,6 +10,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +""" +Provides helper logic used across samples +""" from google.cloud import bigtable @@ -20,6 +23,10 @@ delete_retry = Retry(if_exception_type(exceptions.TooManyRequests, exceptions.ServiceUnavailable)) class create_table_cm: + """ + Create a new table using a context manager, to ensure that table.delete() is called to clean up + the table, even if an exception is thrown + """ def __init__(self, *args, verbose=True, **kwargs): self._args = args self._kwargs = kwargs @@ -42,6 +49,9 @@ def __exit__(self, *args): def create_table(project, instance_id, table_id, column_families={}): + """ + Creates a new table, and blocks until it reaches a ready state + """ client = bigtable.Client(project=project, admin=True) instance = client.instance(instance_id) @@ -64,7 +74,14 @@ def create_table(project, instance_id, table_id, column_families={}): exceptions.FailedPrecondition, exceptions.NotFound, ) + timeout=120, ) def wait_for_table(table): + """ + raises an exception if the table does not exist or is not ready to use + + Because this method is wrapped with an api_core.Retry decorator, it will + retry with backoff if the table is not ready + """ if not table.exists(): raise exceptions.NotFound \ No newline at end of file From c5e88f8094d30e70e00daa04930d1f8aa24b85d4 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 8 Nov 2024 10:57:40 -0800 Subject: [PATCH 12/12] added missing , --- samples/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/utils.py b/samples/utils.py index 6bdb884c9..eb0ca68f9 100644 --- a/samples/utils.py +++ b/samples/utils.py @@ -73,7 +73,7 @@ def create_table(project, instance_id, table_id, column_families={}): exceptions.PreconditionFailed, exceptions.FailedPrecondition, exceptions.NotFound, - ) + ), timeout=120, ) def wait_for_table(table):