From 52c80d652de45dad860d954ab55207326c5eb0e6 Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Wed, 16 Feb 2022 16:38:52 +1100 Subject: [PATCH] cli: add --sync to db info to show syncs When looking at containers and accounts it's sometimes nice to know who they've been replicating with. This patch adds a `--sync|-s` option to swift-{container|account}-info which will also dump the incoming and outgoing sync tables: $ swift-container-info /srv/node3/sdb3/containers/294/624/49b9ff074c502ec5e429e7af99a30624/49b9ff074c502ec5e429e7af99a30624.db -s Path: /AUTH_test/new Account: AUTH_test Container: new Deleted: False Container Hash: 49b9ff074c502ec5e429e7af99a30624 Metadata: Created at: 2022-02-16T05:34:05.988480 (1644989645.98848) Put Timestamp: 2022-02-16T05:34:05.981320 (1644989645.98132) Delete Timestamp: 1970-01-01T00:00:00.000000 (0) Status Timestamp: 2022-02-16T05:34:05.981320 (1644989645.98132) Object Count: 1 Bytes Used: 7 Storage Policy: default (0) Reported Put Timestamp: 1970-01-01T00:00:00.000000 (0) Reported Delete Timestamp: 1970-01-01T00:00:00.000000 (0) Reported Object Count: 0 Reported Bytes Used: 0 Chexor: 962368324c2ca023c56669d03ed92807 UUID: f33184e7-56d5-4c74-9d2e-5417c187d722-sdb3 X-Container-Sync-Point2: -1 X-Container-Sync-Point1: -1 No system metadata found in db file No user metadata found in db file Sharding Metadata: Type: root State: unsharded Incoming Syncs: Sync Point Remote ID Updated At 1 ce7268a1-f5d0-4b83-b993-af17b602a0ff-sdb1 2022-02-16T05:38:22.000000 (1644989902) 1 2af5abc0-7f70-4e2f-8f94-737aeaada7f4-sdb4 2022-02-16T05:38:22.000000 (1644989902) Outgoing Syncs: Sync Point Remote ID Updated At Partition 294 Hash 49b9ff074c502ec5e429e7af99a30624 As a follow up to the device in DB ID patch we can see that the replicas at sdb1 and sdb4 have replicated with this node. Change-Id: I23d786e82c6710bea7660a9acf8bbbd113b5b727 --- bin/swift-container-info | 3 +++ swift/cli/info.py | 29 ++++++++++++++++++++- swift/common/db.py | 20 +++++++++------ test/unit/cli/test_info.py | 36 +++++++++++++++++++++++++- test/unit/common/test_db.py | 51 +++++++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 10 deletions(-) diff --git a/bin/swift-container-info b/bin/swift-container-info index 46e5d6f11e..7d9efecba3 100755 --- a/bin/swift-container-info +++ b/bin/swift-container-info @@ -47,6 +47,9 @@ if __name__ == '__main__': '-v', '--verbose', default=False, action="store_true", help="Show all shard ranges. By default, only the number of shard " "ranges is displayed if there are many shards.") + parser.add_option( + '--sync', '-s', default=False, action="store_true", + help="Output the contents of the incoming/outging sync tables") options, args = parser.parse_args() diff --git a/swift/cli/info.py b/swift/cli/info.py index f65daf3689..4d2e777983 100644 --- a/swift/cli/info.py +++ b/swift/cli/info.py @@ -198,6 +198,28 @@ def print_ring_locations(ring, datadir, account, container=None, obj=None, 'real value is set in the config file on each storage node.') +def get_max_len_sync_item(syncs, item, title): + def map_func(element): + return str(element[item]) + return max(list(map(len, map(map_func, syncs))) + [len(title)]) + + +def print_db_syncs(incoming, syncs): + max_sync_point_len = get_max_len_sync_item(syncs, 'sync_point', + "Sync Point") + max_remote_len = get_max_len_sync_item(syncs, 'remote_id', "Remote ID") + print('%s Syncs:' % ('Incoming' if incoming else 'Outgoing')) + print(' %s\t%s\t%s' % ("Sync Point".ljust(max_sync_point_len), + "Remote ID".ljust(max_remote_len), + "Updated At")) + for sync in syncs: + print(' %s\t%s\t%s (%s)' % ( + str(sync['sync_point']).ljust(max_sync_point_len), + sync['remote_id'].ljust(max_remote_len), + Timestamp(sync['updated_at']).isoformat, + sync['updated_at'])) + + def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False, verbose=False): """ @@ -439,7 +461,7 @@ def print_obj_metadata(metadata, drop_prefixes=False): def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False, - drop_prefixes=False, verbose=False): + drop_prefixes=False, verbose=False, sync=False): if db_type not in ('account', 'container'): print("Unrecognized DB type: internal error") raise InfoSystemExit() @@ -473,6 +495,11 @@ def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False, info['shard_ranges'] = sranges print_db_info_metadata( db_type, info, broker.metadata, drop_prefixes, verbose) + if sync: + # Print incoming / outgoing sync tables. + for incoming in (True, False): + print_db_syncs(incoming, broker.get_syncs(incoming, + include_timestamp=True)) try: ring = Ring(swift_dir, ring_name=db_type) except Exception: diff --git a/swift/common/db.py b/swift/common/db.py index e539f09d97..fab1103fee 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -724,22 +724,26 @@ class DatabaseBroker(object): return -1 return row['sync_point'] - def get_syncs(self, incoming=True): + def get_syncs(self, incoming=True, include_timestamp=False): """ Get a serialized copy of the sync table. :param incoming: if True, get the last incoming sync, otherwise get the last outgoing sync - :returns: list of {'remote_id', 'sync_point'} + :param include_timestamp: If True include the updated_at timestamp + :returns: list of {'remote_id', 'sync_point'} or + {'remote_id', 'sync_point', 'updated_at'} + if include_timestamp is True. """ with self.get() as conn: + columns = 'remote_id, sync_point' + if include_timestamp: + columns += ', updated_at' curs = conn.execute(''' - SELECT remote_id, sync_point FROM %s_sync - ''' % ('incoming' if incoming else 'outgoing')) - result = [] - for row in curs: - result.append({'remote_id': row[0], 'sync_point': row[1]}) - return result + SELECT %s FROM %s_sync + ''' % (columns, 'incoming' if incoming else 'outgoing')) + curs.row_factory = dict_factory + return [r for r in curs] def get_max_row(self, table=None): if not table: diff --git a/test/unit/cli/test_info.py b/test/unit/cli/test_info.py index 94b9c05966..a2a17c8647 100644 --- a/test/unit/cli/test_info.py +++ b/test/unit/cli/test_info.py @@ -29,7 +29,7 @@ from swift.common.storage_policy import StoragePolicy, POLICIES from swift.cli.info import (print_db_info_metadata, print_ring_locations, print_info, print_obj_metadata, print_obj, InfoSystemExit, print_item_locations, - parse_get_node_args) + parse_get_node_args, print_db_syncs) from swift.account.server import AccountController from swift.container.server import ContainerController from swift.container.backend import UNSHARDED, SHARDED @@ -666,6 +666,40 @@ Shard Ranges (3): self.assertIn(exp_cont_msg, out.getvalue()) self.assertIn(exp_obj_msg, out.getvalue()) + def test_print_db_syncs(self): + # first the empty case + for incoming in (True, False): + out = StringIO() + with mock.patch('sys.stdout', out): + print_db_syncs(incoming, []) + if incoming: + exp_heading = 'Incoming Syncs:' + else: + exp_heading = 'Outgoing Syncs:' + exp_heading += '\n Sync Point\tRemote ID\tUpdated At' + self.assertIn(exp_heading, out.getvalue()) + + # now add some syncs + ts0 = utils.Timestamp(1) + ts1 = utils.Timestamp(2) + syncs = [{'sync_point': 0, 'remote_id': 'remote_0', + 'updated_at': str(int(ts0))}, + {'sync_point': 1, 'remote_id': 'remote_1', + 'updated_at': str(int(ts1))}] + + template_output = """%s:\n Sync Point\tRemote ID\tUpdated At + 0 \tremote_0 \t%s (%s) + 1 \tremote_1 \t%s (%s) +""" + for incoming in (True, False): + out = StringIO() + with mock.patch('sys.stdout', out): + print_db_syncs(incoming, syncs) + output = template_output % ( + 'Incoming Syncs' if incoming else 'Outgoing Syncs', + ts0.isoformat, str(int(ts0)), ts1.isoformat, str(int(ts1))) + self.assertEqual(output, out.getvalue()) + def test_print_item_locations_account_container_object_dashed_ring(self): out = StringIO() account = 'account' diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index c4d94ff5a6..22484cc533 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -21,6 +21,8 @@ import unittest from tempfile import mkdtemp from shutil import rmtree, copy from uuid import uuid4 + +import mock import six.moves.cPickle as pickle import base64 @@ -1043,6 +1045,55 @@ class TestDatabaseBroker(TestDbBase): self.assertEqual(broker.get_items_since(3, 2), []) self.assertEqual(broker.get_items_since(999, 2), []) + def test_get_syncs(self): + broker = DatabaseBroker(self.db_path) + broker.db_type = 'test' + broker.db_contains_type = 'test' + uuid1 = str(uuid4()) + + def _initialize(conn, timestamp, **kwargs): + conn.execute('CREATE TABLE test (one TEXT)') + conn.execute('CREATE TABLE test_stat (id TEXT)') + conn.execute('INSERT INTO test_stat (id) VALUES (?)', (uuid1,)) + conn.execute('INSERT INTO test (one) VALUES ("1")') + conn.commit() + pass + broker._initialize = _initialize + broker.initialize(normalize_timestamp('1')) + + for incoming in (True, False): + # Can't mock out timestamp now, because the update_at in the sync + # tables are cuase by a trigger inside sqlite which uses it's own + # now method. So instead track the time before and after to make + # sure we're getting the right timestamps. + ts0 = Timestamp.now() + broker.merge_syncs([ + {'sync_point': 0, 'remote_id': 'remote_0'}, + {'sync_point': 1, 'remote_id': 'remote_1'}], incoming) + + time.sleep(0.005) + broker.merge_syncs([ + {'sync_point': 2, 'remote_id': 'remote_2'}], incoming) + + ts1 = Timestamp.now() + expected_syncs = [{'sync_point': 0, 'remote_id': 'remote_0'}, + {'sync_point': 1, 'remote_id': 'remote_1'}, + {'sync_point': 2, 'remote_id': 'remote_2'}] + + self.assertEqual(expected_syncs, broker.get_syncs(incoming)) + + # if we want the updated_at timestamps too then: + expected_syncs[0]['updated_at'] = mock.ANY + expected_syncs[1]['updated_at'] = mock.ANY + expected_syncs[2]['updated_at'] = mock.ANY + actual_syncs = broker.get_syncs(incoming, include_timestamp=True) + self.assertEqual(expected_syncs, actual_syncs) + # Note that most of the time, we expect these all to be == + # but we've been known to see sizeable delays in the gate at times + self.assertTrue(all([ + str(int(ts0)) <= s['updated_at'] <= str(int(ts1)) + for s in actual_syncs])) + def test_get_sync(self): broker = DatabaseBroker(self.db_path) broker.db_type = 'test'