Read from and write to SQLite database in binary mode

src/addresses.py

@@ -187,7 +187,7 @@ def decodeAddress(address):
         integer = decodeBase58(address)
     if integer == 0:
         status = 'invalidcharacters'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
     # after converting to hex, the string will be prepended
     # with a 0x and appended with a L in python2
     hexdata = hex(integer)[2:].rstrip('L')
@@ -200,31 +200,31 @@ def decodeAddress(address):
 
     if checksum != double_sha512(data[:-4])[0:4]:
         status = 'checksumfailed'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
 
     try:
         addressVersionNumber, bytesUsedByVersionNumber = decodeVarint(data[:9])
     except varintDecodeError as e:
         logger.error(str(e))
         status = 'varintmalformed'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
 
     if addressVersionNumber > 4:
         logger.error('cannot decode address version numbers this high')
         status = 'versiontoohigh'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
     elif addressVersionNumber == 0:
         logger.error('cannot decode address version numbers of zero.')
         status = 'versiontoohigh'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
 
     try:
         streamNumber, bytesUsedByStreamNumber = \
             decodeVarint(data[bytesUsedByVersionNumber:])
     except varintDecodeError as e:
         logger.error(str(e))
         status = 'varintmalformed'
-        return status, 0, 0, ''
+        return status, 0, 0, b''
 
     status = 'success'
     if addressVersionNumber == 1:
@@ -242,21 +242,21 @@ def decodeAddress(address):
             return status, addressVersionNumber, streamNumber, \
                 b'\x00\x00' + embeddedRipeData
         elif len(embeddedRipeData) < 18:
-            return 'ripetooshort', 0, 0, ''
+            return 'ripetooshort', 0, 0, b''
         elif len(embeddedRipeData) > 20:
-            return 'ripetoolong', 0, 0, ''
-        return 'otherproblem', 0, 0, ''
+            return 'ripetoolong', 0, 0, b''
+        return 'otherproblem', 0, 0, b''
     elif addressVersionNumber == 4:
         embeddedRipeData = \
             data[bytesUsedByVersionNumber + bytesUsedByStreamNumber:-4]
         if embeddedRipeData[0:1] == b'\x00':
             # In order to enforce address non-malleability, encoded
             # RIPE data must have NULL bytes removed from the front
-            return 'encodingproblem', 0, 0, ''
+            return 'encodingproblem', 0, 0, b''
         elif len(embeddedRipeData) > 20:
-            return 'ripetoolong', 0, 0, ''
+            return 'ripetoolong', 0, 0, b''
         elif len(embeddedRipeData) < 4:
-            return 'ripetooshort', 0, 0, ''
+            return 'ripetooshort', 0, 0, b''
         x00string = b'\x00' * (20 - len(embeddedRipeData))
         return status, addressVersionNumber, streamNumber, \
             x00string + embeddedRipeData

src/api.py

@@ -70,6 +70,7 @@
 
 import six
 from six.moves import configparser, http_client, xmlrpc_server
+from dbcompat import dbstr
 
 import helper_inbox
 import helper_sent
@@ -531,12 +532,12 @@ def _dump_inbox_message(  # pylint: disable=too-many-arguments
         message = shared.fixPotentiallyInvalidUTF8Data(message)
         return {
             'msgid': hexlify(msgid),
-            'toAddress': toAddress,
-            'fromAddress': fromAddress,
+            'toAddress': toAddress.decode("utf-8", "replace"),
+            'fromAddress': fromAddress.decode("utf-8", "replace"),
             'subject': base64.b64encode(subject),
             'message': base64.b64encode(message),
             'encodingType': encodingtype,
-            'receivedTime': received,
+            'receivedTime': received.decode("utf-8", "replace"),
             'read': read
         }
 
@@ -598,11 +599,12 @@ def HandleListAddressBookEntries(self, label=None):
         """
         queryreturn = sqlQuery(
             "SELECT label, address from addressbook WHERE label = ?",
-            label
+            dbstr(label)
         ) if label else sqlQuery("SELECT label, address from addressbook")
         data = []
         for label, address in queryreturn:
             label = shared.fixPotentiallyInvalidUTF8Data(label)
+            address = address.decode("utf-8", "replace")
             data.append({
                 'label': base64.b64encode(label),
                 'address': address
@@ -618,12 +620,12 @@ def HandleAddAddressBookEntry(self, address, label):
         self._verifyAddress(address)
         # TODO: add unique together constraint in the table
         queryreturn = sqlQuery(
-            "SELECT address FROM addressbook WHERE address=?", address)
+            "SELECT address FROM addressbook WHERE address=?", dbstr(address))
         if queryreturn != []:
             raise APIError(
                 16, 'You already have this address in your address book.')
 
-        sqlExecute("INSERT INTO addressbook VALUES(?,?)", label, address)
+        sqlExecute("INSERT INTO addressbook VALUES(?,?)", dbstr(label), dbstr(address))
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderMessagelistToLabels', ''))
         queues.UISignalQueue.put(('rerenderAddressBook', ''))
@@ -635,7 +637,7 @@ def HandleDeleteAddressBookEntry(self, address):
         """Delete an entry from address book."""
         address = addBMIfNotPresent(address)
         self._verifyAddress(address)
-        sqlExecute('DELETE FROM addressbook WHERE address=?', address)
+        sqlExecute('DELETE FROM addressbook WHERE address=?', dbstr(address))
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderMessagelistToLabels', ''))
         queues.UISignalQueue.put(('rerenderAddressBook', ''))
@@ -919,6 +921,7 @@ def HandleGetAllInboxMessages(self):
             " ORDER BY received"
         )
         return {"inboxMessages": [
+
             self._dump_inbox_message(*data) for data in queryreturn
         ]}
 
@@ -1018,7 +1021,7 @@ def HandleInboxMessagesByReceiver(self, toAddress):
         queryreturn = sqlQuery(
             "SELECT msgid, toaddress, fromaddress, subject, received,"
             " message, encodingtype, read FROM inbox WHERE folder='inbox'"
-            " AND toAddress=?", toAddress)
+            " AND toAddress=?", dbstr(toAddress))
         return {"inboxMessages": [
             self._dump_inbox_message(*data) for data in queryreturn
         ]}
@@ -1055,7 +1058,7 @@ def HandleGetSentMessagesByAddress(self, fromAddress):
             "SELECT msgid, toaddress, fromaddress, subject, lastactiontime,"
             " message, encodingtype, status, ackdata FROM sent"
             " WHERE folder='sent' AND fromAddress=? ORDER BY lastactiontime",
-            fromAddress
+            dbstr(fromAddress)
         )
         return {"sentMessages": [
             self._dump_sent_message(*data) for data in queryreturn
@@ -1150,9 +1153,9 @@ def HandleSendMessage(
 
         toLabel = ''
         queryreturn = sqlQuery(
-            "SELECT label FROM addressbook WHERE address=?", toAddress)
+            "SELECT label FROM addressbook WHERE address=?", dbstr(toAddress))
         try:
-            toLabel = queryreturn[0][0]
+            toLabel = queryreturn[0][0].decode("utf-8", "replace")
         except IndexError:
             pass
 
@@ -1219,7 +1222,7 @@ def HandleGetStatus(self, ackdata):
         queryreturn = sqlQuery(
             "SELECT status FROM sent where ackdata=?", ackdata)
         try:
-            return queryreturn[0][0]
+            return queryreturn[0][0].decode("utf-8", "replace")
         except IndexError:
             return 'notfound'
 
@@ -1238,11 +1241,11 @@ def HandleAddSubscription(self, address, label=''):
         # First we must check to see if the address is already in the
         # subscriptions list.
         queryreturn = sqlQuery(
-            "SELECT * FROM subscriptions WHERE address=?", address)
+            "SELECT * FROM subscriptions WHERE address=?", dbstr(address))
         if queryreturn:
             raise APIError(16, 'You are already subscribed to that address.')
         sqlExecute(
-            "INSERT INTO subscriptions VALUES (?,?,?)", label, address, True)
+            "INSERT INTO subscriptions VALUES (?,?,?)", dbstr(label), dbstr(address), True)
         shared.reloadBroadcastSendersForWhichImWatching()
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderSubscriptions', ''))
@@ -1256,7 +1259,7 @@ def HandleDeleteSubscription(self, address):
         """
 
         address = addBMIfNotPresent(address)
-        sqlExecute("DELETE FROM subscriptions WHERE address=?", address)
+        sqlExecute("DELETE FROM subscriptions WHERE address=?", dbstr(address))
         shared.reloadBroadcastSendersForWhichImWatching()
         queues.UISignalQueue.put(('rerenderMessagelistFromLabels', ''))
         queues.UISignalQueue.put(('rerenderSubscriptions', ''))
@@ -1274,6 +1277,7 @@ def ListSubscriptions(self):
         data = []
         for label, address, enabled in queryreturn:
             label = shared.fixPotentiallyInvalidUTF8Data(label)
+            address = address.decode("utf-8", "replace")
             data.append({
                 'label': base64.b64encode(label),
                 'address': address,