From 8dbd2678dd15067c6042f427d2cc3a863c0127d4 Mon Sep 17 00:00:00 2001
From: Philip O'Toole <philip.otoole@yahoo.com>
Date: Fri, 29 Jan 2021 09:06:26 -0500
Subject: [PATCH 1/2] More end-to-end snapshot and restore testing

---
 system_test/full_system_test.py | 56 ++++++++++++++++++++++++++++++---
 1 file changed, 52 insertions(+), 4 deletions(-)

diff --git a/system_test/full_system_test.py b/system_test/full_system_test.py
index 516103ff..0666ae23 100755
--- a/system_test/full_system_test.py
+++ b/system_test/full_system_test.py
@@ -160,9 +160,6 @@ class Node(object):
   def last_snapshot_index(self):
     return int(self.status()['store']['raft']['last_snapshot_index'])
 
-  def num_snapshots(self):
-    return int(self.expvar()['store']['num_snapshots'])
-
   def num_join_requests(self):
     return int(self.expvar()['http']['joins'])
 
@@ -615,7 +612,58 @@ class TestEndToEndBackupRestore(unittest.TestCase):
     deprovision_node(self.node1)
     os.remove(self.db_file)
 
-class TestEndToEndSnapRestore(unittest.TestCase):
+class TestEndToEndSnapRestoreSingle(unittest.TestCase):
+  def setUp(self):
+    self.n0 = Node(RQLITED_PATH, '0',  raft_snap_threshold=10, raft_snap_int="1s")
+    self.n0.start()
+    self.n0.wait_for_leader()
+
+  def waitForSnapIndex(self, n):
+    timeout = 10
+    t = 0
+    while True:
+      if t > timeout:
+        raise Exception('timeout')
+      if self.n0.last_snapshot_index() >= n:
+        break
+      time.sleep(1)
+      t+=1
+
+  def test_snap_and_restart(self):
+    '''Check that an node restarts correctly after multiple snapshots'''
+
+    # Let's get multiple snapshots done.
+    self.n0.execute('CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)')
+
+    for i in range(0,200):
+      self.n0.execute('INSERT INTO foo(name) VALUES("fiona")')
+    self.n0.wait_for_all_applied()
+    self.waitForSnapIndex(175)
+
+    # Ensure node has the full correct state.
+    j = self.n0.query('SELECT count(*) FROM foo', level='none')
+    self.assertEqual(str(j), "{u'results': [{u'values': [[200]], u'types': [u''], u'columns': [u'count(*)']}]}")
+
+    # Restart nodes, and make sure it comes back with the correct state
+    self.n0.stop()
+    self.n0.start()
+    self.n0.wait_for_leader()
+    self.n0.wait_for_all_applied()
+    self.assertEqual(self.n0.expvar()['store']['num_restores'], 1)
+
+    j = self.n0.query('SELECT count(*) FROM foo', level='none')
+    self.assertEqual(str(j), "{u'results': [{u'values': [[200]], u'types': [u''], u'columns': [u'count(*)']}]}")
+
+  def tearDown(self):
+    deprovision_node(self.n0)
+
+class TestEndToEndSnapRestoreSingleOnDisk(TestEndToEndSnapRestoreSingle):
+  def setUp(self):
+    self.n0 = Node(RQLITED_PATH, '0',  raft_snap_threshold=10, raft_snap_int="1s", on_disk=True)
+    self.n0.start()
+    self.n0.wait_for_leader()
+
+class TestEndToEndSnapRestoreCluster(unittest.TestCase):
   def waitForSnap(self, n):
     timeout = 10
     t = 0

From 2e19df93dec51204856ddd126f165e5f74e9db5f Mon Sep 17 00:00:00 2001
From: Philip O'Toole <philip.otoole@yahoo.com>
Date: Fri, 29 Jan 2021 09:07:36 -0500
Subject: [PATCH 2/2] Correct comment

---
 system_test/full_system_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/system_test/full_system_test.py b/system_test/full_system_test.py
index 0666ae23..825fc876 100755
--- a/system_test/full_system_test.py
+++ b/system_test/full_system_test.py
@@ -644,7 +644,7 @@ class TestEndToEndSnapRestoreSingle(unittest.TestCase):
     j = self.n0.query('SELECT count(*) FROM foo', level='none')
     self.assertEqual(str(j), "{u'results': [{u'values': [[200]], u'types': [u''], u'columns': [u'count(*)']}]}")
 
-    # Restart nodes, and make sure it comes back with the correct state
+    # Restart node, and make sure it comes back with the correct state
     self.n0.stop()
     self.n0.start()
     self.n0.wait_for_leader()