From 3c11f9d24807ef4a7df9daebcd8d4f3b1480f809 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Fri, 12 Jan 2024 15:01:41 -0500 Subject: [PATCH 1/2] Fix tests --- system_test/e2e/multi_node.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/system_test/e2e/multi_node.py b/system_test/e2e/multi_node.py index f90a9075..e6cda904 100644 --- a/system_test/e2e/multi_node.py +++ b/system_test/e2e/multi_node.py @@ -530,15 +530,19 @@ class TestEndToEndBackupRestore(unittest.TestCase): class TestEndToEndSnapRestoreCluster(unittest.TestCase): def wait_for_snap(self, n): + ''' + Wait for at least n snapshots to be taken. + ''' timeout = 10 t = 0 while True: if t > timeout: raise Exception('timeout') - if self.n0.num_snapshots() is n: + if self.n0.num_snapshots() >= n: break time.sleep(1) t+=1 + return self.n0.num_snapshots() def poll_query(self, node, exp): t = 0 @@ -546,7 +550,7 @@ class TestEndToEndSnapRestoreCluster(unittest.TestCase): if t > TIMEOUT: raise Exception('timeout waiting for node %s to have all data' % node.node_id) j = node.query('SELECT count(*) FROM foo', level='none') - if j == d_("{'results': [{'values': [[502]], 'types': ['integer'], 'columns': ['count(*)']}]}"): + if j == exp: break time.sleep(1) t+=1 @@ -559,11 +563,10 @@ class TestEndToEndSnapRestoreCluster(unittest.TestCase): self.n0.execute('CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)') # Let's get multiple snapshots done. - for j in range(1, 4): - for i in range(0,100): - self.n0.execute('INSERT INTO foo(name) VALUES("fiona")') - self.n0.wait_for_all_fsm() - self.wait_for_snap(j) + for i in range(0,300): + self.n0.execute('INSERT INTO foo(name) VALUES("fiona")') + self.n0.wait_for_all_fsm() + num_snaps = self.wait_for_snap(1) # Add two more nodes to the cluster self.n1 = Node(RQLITED_PATH, '1') @@ -579,22 +582,19 @@ class TestEndToEndSnapRestoreCluster(unittest.TestCase): # Ensure those new nodes have the full correct state. self.n1.wait_for_fsm_index(self.n0.fsm_index()) - j = self.n1.query('SELECT count(*) FROM foo', level='none') - self.assertEqual(j, d_("{'results': [{'values': [[301]], 'types': ['integer'], 'columns': ['count(*)']}]}")) + self.poll_query(self.n1, d_("{'results': [{'values': [[301]], 'types': ['integer'], 'columns': ['count(*)']}]}")) self.n2.wait_for_fsm_index(self.n0.fsm_index()) - j = self.n2.query('SELECT count(*) FROM foo', level='none') - self.assertEqual(j, d_("{'results': [{'values': [[301]], 'types': ['integer'], 'columns': ['count(*)']}]}")) + self.poll_query(self.n2, d_("{'results': [{'values': [[301]], 'types': ['integer'], 'columns': ['count(*)']}]}")) # Kill one of the nodes, and make more changes, enough to trigger more snaps. self.n2.stop() # Let's get more snapshots done. - for j in range(3, 5): - for i in range(0,100): - self.n0.execute('INSERT INTO foo(name) VALUES("fiona")') - self.n0.wait_for_all_fsm() - self.wait_for_snap(j) + for j in range(0, 200): + self.n0.execute('INSERT INTO foo(name) VALUES("fiona")') + self.n0.wait_for_all_fsm() + self.wait_for_snap(num_snaps+1) # Restart killed node, check it has full state. self.n2.start() From 430f41521fbf9b084abb8b3d71d770acaec46f6e Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Fri, 12 Jan 2024 15:02:50 -0500 Subject: [PATCH 2/2] CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ebcd853..3bf60432 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## 8.16.5 (unreleased) ### Implementation changes and bug fixes - [PR #1592](https://github.com/rqlite/rqlite/pull/1592): Refactor and enhance upload logging. +- [PR #1593](https://github.com/rqlite/rqlite/pull/1593): Tighthen snapshot-join end-to-end testing. ## 8.16.4 (January 12th 2024) ### Implementation changes and bug fixes