SQL refactoring and testing

Author: cis-muzahid

src/bitmessagemain.py

@@ -19,8 +19,10 @@
 app_dir = pathmagic.setup()
 
 import depends
+
 depends.check_dependencies()
 
+import helper_sql
 import getopt
 import multiprocessing
 # Used to capture a Ctrl-C keypress so that Bitmessage can shutdown gracefully.
@@ -81,6 +83,7 @@ def signal_handler(signum, frame):
 
 class Main(object):
     """Main PyBitmessage class"""
+
     def start(self):
         """Start main application"""
         # pylint: disable=too-many-statements,too-many-branches,too-many-locals
@@ -261,8 +264,8 @@ def start(self):
             while state.shutdown == 0:
                 time.sleep(1)
                 if (
-                    state.testmode
-                    and time.time() - state.last_api_response >= 30
+                        state.testmode
+                        and time.time() - state.last_api_response >= 30
                 ):
                     self.stop()
         elif not state.enableGUI:
@@ -385,7 +388,6 @@ def main():
 if __name__ == "__main__":
     main()
 
-
 # So far, the creation of and management of the Bitmessage protocol and this
 # client is a one-man operation. Bitcoin tips are quite appreciated.
 # 1H5XaDA6fYENLbknwZyjiYXYPQaFjjLX2u

src/class_sqlThread.py

@@ -0,0 +1,3 @@
+ALTER TABLE inventory ADD first20bytesofencryptedmessage blob DEFAULT '';
+
+UPDATE settings SET value = 2 WHERE key = 'version';

src/sql/init_version_1.sql

@@ -13,3 +13,5 @@ CREATE TABLE `addressbook` (
 INSERT INTO addressbook SELECT label, address FROM old_addressbook;
 
 DROP TABLE old_addressbook;
+
+UPDATE settings SET value = 11 WHERE key = 'version';

src/sql/init_version_10.sql

@@ -27,3 +27,5 @@ CREATE TABLE `inventory` (
 INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;
 
 DROP TABLE inventory_backup;
+
+UPDATE settings SET value = 3 WHERE key = 'version';

src/sql/init_version_2.sql

@@ -3,3 +3,5 @@
 --
 
 ALTER TABLE inventory ADD tag blob DEFAULT '';
+
+UPDATE settings SET value = 4 WHERE key = 'version';

src/sql/init_version_3.sql

@@ -15,3 +15,5 @@ CREATE TABLE `pubkeys` (
 ) ;
 
 DELETE FROM inventory WHERE objecttype = 'pubkey';
+
+UPDATE settings SET value = 5 WHERE key = 'version';

src/sql/init_version_4.sql

@@ -10,3 +10,5 @@ CREATE TABLE `objectprocessorqueue` (
     `data` blob,
     UNIQUE(objecttype, data) ON CONFLICT REPLACE
 ) ;
+
+UPDATE settings SET value = 6 WHERE key = 'version';

src/sql/init_version_5.sql

@@ -23,3 +23,5 @@ CREATE TABLE `objectprocessorqueue` (
     `data` blob,
     UNIQUE(objecttype, data) ON CONFLICT REPLACE
 ) ;
+
+UPDATE settings SET value = 7 WHERE key = 'version';

src/sql/init_version_6.sql

@@ -9,3 +9,5 @@ DELETE FROM inventory WHERE objecttype = 1;
 DELETE FROM pubkeys;
 
 UPDATE sent SET status='msgqueued' WHERE status='doingmsgpow' or status='badkey';
+
+UPDATE settings SET value = 8 WHERE key = 'version';

src/sql/init_version_7.sql

@@ -5,3 +5,5 @@
 --
 
 ALTER TABLE inbox ADD sighash blob DEFAULT '';
+
+UPDATE settings SET value = 9 WHERE key = 'version';

src/sql/init_version_8.sql

@@ -27,7 +27,7 @@ CREATE TABLE `sent` (
     `ackdata` blob,
     `senttime` integer,
     `lastactiontime` integer,
-    `sleeptill` int,
+    `sleeptill` integer,
     `status` text,
     `retrynumber` integer,
     `folder` text,
@@ -72,3 +72,5 @@ CREATE TABLE `pubkeys` (
 INSERT INTO pubkeys SELECT address, addressversion, transmitdata, `time`, usedpersonally FROM pubkeys_backup;
 
 DROP TABLE pubkeys_backup;
+
+UPDATE settings SET value = 10 WHERE key = 'version';

src/sql/init_version_9.sql

@@ -0,0 +1,92 @@
+CREATE TABLE `inbox` (
+   `msgid` blob,
+   `toaddress` text,
+   `fromaddress` text,
+   `subject` text,
+   `received` text,
+   `message` text,
+   `folder` text,
+   `encodingtype` int,
+   `read` bool,
+UNIQUE(msgid) ON CONFLICT REPLACE
+);
+
+CREATE TABLE `sent` (
+   `msgid` blob,
+   `toaddress` text,
+   `toripe` blob,
+   `fromaddress` text,
+   `subject` text,
+   `message` text,
+   `ackdata` blob,
+   `lastactiontime` integer,
+   `status` text,
+   `pubkeyretrynumber` integer,
+   `msgretrynumber` integer,
+   `folder` text,
+   `encodingtype` int
+);
+
+CREATE TABLE `subscriptions` (
+   `label` text,
+   `address` text,
+   `enabled` bool
+);
+
+CREATE TABLE `addressbook` (
+   `label` text,
+   `address` text
+);
+
+CREATE TABLE `blacklist` (
+   `label` text,
+   `address` text,
+   `enabled` bool
+);
+
+CREATE TABLE `whitelist` (
+   `label` text,
+   `address` text,
+   `enabled` bool
+);
+
+CREATE TABLE `pubkeys` (
+   `hash` blob,
+   `transmitdata` blob,
+   `time` int,
+   `usedpersonally` text,
+UNIQUE(hash) ON CONFLICT REPLACE
+);
+
+CREATE TABLE `inventory` (
+   `hash` blob,
+   `objecttype` text,
+   `streamnumber` int,
+   `payload` blob,
+   `receivedtime` integer,
+UNIQUE(hash) ON CONFLICT REPLACE
+);
+
+CREATE TABLE `knownnodes` (
+   `timelastseen` int,
+   `stream` int,
+   `services` blob,
+   `host` blob,
+   `port` blob,
+UNIQUE(host, stream, port) ON CONFLICT REPLACE
+);
+
+CREATE TABLE `settings` (
+   `key` blob,
+   `value` blob,
+UNIQUE(key) ON CONFLICT REPLACE
+);
+
+INSERT INTO subscriptions VALUES ('Bitmessage new releases/announcements', 'BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw', 1);
+
+INSERT INTO settings VALUES('version', 1);
+
+INSERT INTO settings VALUES('lastvacuumtime', CAST(strftime('%s', 'now') AS STR) );
+
+INSERT INTO inventory VALUES( '', 'pubkey', 1, '', 1);
+

src/sql/initialize_schema_v1.sql

@@ -23,7 +23,7 @@
 
 from bmconfigparser import config
 from helper_msgcoding import MsgEncode, MsgDecode
-from helper_sql import sqlQuery
+from helper_sql import sqlQuery, sqlExecute
 from network import asyncore_pollchoose as asyncore, knownnodes
 from network.bmproto import BMProto
 from network.connectionpool import BMConnectionPool
@@ -412,6 +412,22 @@ def test_adding_two_same_case_sensitive_addresses(self):
         self.delete_address_from_addressbook(address1)
         self.delete_address_from_addressbook(address2)
 
+    def test_sqlscripts(self):
+        """ Test sql statements"""
+
+        sqlExecute('create table if not exists testtbl (id integer)')
+        tables = list(sqlQuery("select name from sqlite_master where type is 'table'"))
+        res = [item for item in tables if 'testtbl' in item]
+        self.assertEqual(res[0][0], 'testtbl')
+
+        queryreturn = sqlExecute("INSERT INTO testtbl VALUES(101);")
+        self.assertEqual(queryreturn, 1)
+
+        queryreturn = sqlQuery('''SELECT * FROM testtbl''')
+        self.assertEqual(queryreturn[0][0], 101)
+
+        sqlQuery("DROP TABLE testtbl")
+
 
 def run():
     """Starts all tests intended for core run"""

src/tests/core.py

@@ -0,0 +1,3 @@
+INSERT INTO addressbook VALUES ('', '');
+
+INSERT INTO addressbook VALUES ('', '');

src/tests/sql/init_version_10.sql

@@ -0,0 +1,7 @@
+INSERT INTO inventory VALUES( '', 1, 1, '', 1, '');
+
+INSERT INTO pubkeys VALUES( '', 1, '', 1, '');
+
+INSERT INTO sent VALUES( '', '', '', '', '', '', '', 1, 'doingmsgpow', 1, 1, '', 1);
+
+INSERT INTO sent VALUES( '', '', '', '', '', '', '', 1, 'badkey', 1, 1, '', 1);

src/tests/sql/init_version_2.sql

@@ -0,0 +1 @@
+INSERT INTO pubkeys VALUES( x'0001010101010101010101010101010101010101', 3, '', 1, '');