diff --git a/.gitignore b/.gitignore index 8546f0d1..8d5fa805 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ contracting.egg-info build .github .idea -logs \ No newline at end of file +logs +*.so +.ccls diff --git a/contracting/client.py b/contracting/client.py index b89c606e..d896ec66 100644 --- a/contracting/client.py +++ b/contracting/client.py @@ -175,37 +175,49 @@ def __init__(self, signer='sys', self.submission_filename = submission_filename self.environment = environment - # Seed the genesis contracts into the instance - with open(self.submission_filename) as f: - contract = f.read() + # Get submission contract from file + if submission_filename is not None: + # Seed the genesis contracts into the instance + with open(self.submission_filename) as f: + contract = f.read() - self.raw_driver.set_contract(name='submission', - code=contract) + self.raw_driver.set_contract(name='submission', + code=contract) - self.raw_driver.commit() + self.raw_driver.commit() + # Get submission contract from state self.submission_contract = self.get_contract('submission') - def set_submission_contract(self, filename=None): + def set_submission_contract(self, filename=None, commit=True): + state_contract = self.get_contract('submission') + if filename is None: filename = self.submission_filename - with open(filename) as f: - contract = f.read() + if filename is None and state_contract is None: + raise AssertionError("No submission contract provided or found in state.") - self.raw_driver.delete_contract(name='submission') - self.raw_driver.set_contract(name='submission', - code=contract) + if filename is not None: + with open(filename) as f: + contract = f.read() - self.raw_driver.commit() + self.raw_driver.delete_contract(name='submission') + self.raw_driver.set_contract(name='submission', + code=contract) + if commit: + self.raw_driver.commit() self.submission_contract = self.get_contract('submission') + def flush(self): # flushes db and resubmits genesis contracts self.raw_driver.flush() self.raw_driver.clear_pending_state() - self.set_submission_contract() + + if self.submission_filename is not None: + self.set_submission_contract() # Returns abstract contract which has partial methods mapped to each exported function. def get_contract(self, name): @@ -273,6 +285,8 @@ def compile(self, f): def submit(self, f, name=None, metering=None, owner=None, constructor_args={}, signer=None): + assert self.submission_contract is not None, "No submission contract set. Try set_submission_contract first." + if isinstance(f, FunctionType): f, n = self.closure_to_code_string(f) if name is None: diff --git a/contracting/compilation/linter.py b/contracting/compilation/linter.py index 4be77745..35d33d00 100644 --- a/contracting/compilation/linter.py +++ b/contracting/compilation/linter.py @@ -47,6 +47,11 @@ def no_nested_imports(self, node): def visit_Name(self, node): self.not_system_variable(node.id, node.lineno) + if node.id == 'rt': + self._is_success = False + str = "Line {}: ".format(node.lineno) + VIOLATION_TRIGGERS[13] + self._violations.append(str) + if node.id in ILLEGAL_BUILTINS and node.id != 'float': self._is_success = False str = "Line {}: ".format(node.lineno) + VIOLATION_TRIGGERS[13] diff --git a/contracting/config.py b/contracting/config.py index ceb13090..6f821ac3 100644 --- a/contracting/config.py +++ b/contracting/config.py @@ -5,9 +5,11 @@ DELIMITER = ':' INDEX_SEPARATOR = '.' +HDF5_GROUP_SEPARATOR = '/' DECIMAL_PRECISION = 64 +SUBMISSION_CONTRACT_NAME = 'submission' PRIVATE_METHOD_PREFIX = '__' EXPORT_DECORATOR_STRING = 'export' INIT_DECORATOR_STRING = 'construct' @@ -24,3 +26,5 @@ WRITE_COST_PER_BYTE = 25 STAMPS_PER_TAU = 20 + +BLOCK_NUM_DEFAULT = -1 diff --git a/contracting/db/driver.py b/contracting/db/driver.py index b3ec3b0c..c9e100ba 100644 --- a/contracting/db/driver.py +++ b/contracting/db/driver.py @@ -12,14 +12,17 @@ from pathlib import Path import shutil import hashlib -import lmdb + import motor.motor_asyncio import asyncio +import logging +from contracting.db.hdf5 import h5c FILE_EXT = '.d' HASH_EXT = '.x' STORAGE_HOME = Path().home().joinpath('.lamden') +FSDRIVER_HOME = STORAGE_HOME.joinpath('state') # DB maps bytes to bytes # Driver maps string to python object @@ -31,7 +34,6 @@ COMPILED_KEY = '__compiled__' DEVELOPER_KEY = '__developer__' - class Driver: def __init__(self, db='lamden', collection='state'): self.client = pymongo.MongoClient() @@ -155,7 +157,7 @@ def __init__(self): super().__init__() self.db = {} - def get(self, item): + def get(self, item: str): key = item.encode() value = self.db.get(key) return decode(value) @@ -205,222 +207,83 @@ def __delitem__(self, key: str): except KeyError: pass - class FSDriver: - OS_KEY_LIMIT = (256 - 1) - len(FILE_EXT) - - def __init__(self, root='fs'): - self.root = os.path.join(Path.home(), root) + def __init__(self, root=FSDRIVER_HOME): + self.root = root + self.root.mkdir(exist_ok=True, parents=True) - def get(self, item: str): + def __parse_key(self, key): try: - filename = self._key_to_file(item) - with open(filename, 'r') as f: - v = f.read() + filename, variable = key.split(config.INDEX_SEPARATOR, 1) + variable = variable.replace(config.DELIMITER, config.HDF5_GROUP_SEPARATOR) + except: + filename = '__misc' + variable = key.replace(config.DELIMITER, config.HDF5_GROUP_SEPARATOR) - except FileNotFoundError: - return None + return filename, variable - return decode(v) + def __filename_to_path(self, filename): + return str(self.root.joinpath(filename)) - def set(self, key, value): - if value is None: - self.__delitem__(key) - else: - v = encode(value) - filename = self._key_to_file(key) + def __get_files(self): + return sorted(os.listdir(self.root)) - os.makedirs(filename.parents[0], exist_ok=True) + def __get_keys_from_file(self, filename): + return [filename + config.INDEX_SEPARATOR + g.replace(config.HDF5_GROUP_SEPARATOR, config.DELIMITER) for g in h5c.get_groups(self.__filename_to_path(filename))] - with open(filename, 'w') as f: - f.write(v) + def __getitem__(self, key): + return self.get(key) - def flush(self): - try: - shutil.rmtree(self.root) - except FileNotFoundError: - pass - - def delete(self, key: str): - self.__delitem__(key) - - def iter(self, prefix: str='', length=0): - keys = [] - - for (r, dirs, files) in sorted(os.walk(self.root, topdown=True)): - files.sort() - base = r[len(self.root):] - - for f in files: - if f.endswith(FILE_EXT) and f.startswith(prefix): - keys.append(self.path_to_key(os.path.join(base, f))) - - if 0 < length <= len(keys): - break - - if 0 < length <= len(keys): - break - - keys.sort() - - return keys + def __setitem__(self, key, value): + self.set(key, value) - def _iter(self, prefix: str='', length=0): - keys = [] + def __delitem__(self, key): + self.delete(key) - for (r, dirs, files) in os.walk(os.path.join(self.root, prefix), topdown=True): - base = r[len(self.root):] + def get(self, item: str): + filename, variable = self.__parse_key(item) - for f in files: - if f.endswith(FILE_EXT): - keys.append(self.path_to_key(os.path.join(base, f))) + return decode(h5c.get_value(self.__filename_to_path(filename), variable)) - if 0 < length <= len(keys): - break + def get_block(self, item: str): + filename, variable = self.__parse_key(item) + block_num = h5c.get_block(self.__filename_to_path(filename), variable) - if 0 < length <= len(keys): - break + return config.BLOCK_NUM_DEFAULT if block_num is None else int(block_num) - keys.sort() - - return keys + def set(self, key, value, block_num=None): + filename, variable = self.__parse_key(key) + h5c.set( + self.__filename_to_path(filename), + variable, + encode(value) if value is not None else None, + str(block_num) if block_num is not None else None + ) - def keys(self): - return self.iter() - - def __getitem__(self, item: str): - value = self.get(item) - if value is None: - raise KeyError - return value - - def __setitem__(self, key: str, value): - self.set(key, value) - - def __delitem__(self, key: str): - filename = self._key_to_file(key) + def flush(self): try: - os.remove(filename) + shutil.rmtree(self.root) except FileNotFoundError: pass + self.root.mkdir(exist_ok=True, parents=True) - @staticmethod - def hash_key(key: str): - h = hashlib.sha3_256() - h.update(key.encode()) - - return h.hexdigest()[:-2] + HASH_EXT - - def _key_to_file(self, key: str): - filename = key.replace(':', '/') - filename = filename.replace('.', '/') - filename += FILE_EXT - filename = os.path.join(self.root, filename) - return Path(filename) - - def path_to_key(self, path): - if path.endswith(FILE_EXT): - path = path[:-len(FILE_EXT)] - - pth = path.split('/') - - pth = [p for p in pth if p != ''] - - contract = pth.pop(0) - - if len(pth) == 0: - return contract - - variable = pth.pop(0) - - key = '.'.join((contract, variable)) - - if len(pth) == 0: - return key - - key = ':'.join((key, *pth)) - - return key - - -class LMDBDriver: - def __init__(self, filename=STORAGE_HOME.joinpath('state')): - self.filename = filename - self.filename.mkdir(exist_ok=True, parents=True) - - self.db_writer = lmdb.open(path=str(self.filename), map_size=int(1e12), readonly=False) - self.db_reader = lmdb.open(path=str(self.filename), map_size=int(1e12), readonly=True, lock=False) - - def get(self, item: str): - with self.db_reader.begin() as tx: - v = tx.get(item.encode()) - - if v is None: - return None - - return decode(v) - - def set(self, key, value): - if value is None: - self.__delitem__(key) - else: - v = encode(value) - with self.db_writer.begin(write=True) as tx: - tx.put(key.encode(), v.encode()) - - def flush(self): - with self.db_writer.begin(write=True) as tx: - cursor = tx.cursor() - for key, _ in cursor: - tx.delete(key) - - def delete(self, key: str): - self.__delitem__(key) + def delete(self, key): + filename, variable = self.__parse_key(key) + h5c.delete(self.__filename_to_path(filename), variable) - def iter(self, prefix: str, length=0): + def iter(self, prefix='', length=0): keys = [] + for filename in self.__get_files(): + if filename.startswith(prefix): + keys.extend(self.__get_keys_from_file(filename)) + if length > 0 and len(keys) >= length: + break + keys.sort() - with self.db_reader.begin() as tx: - cursor = tx.cursor() - - if not cursor.set_range(prefix.encode()): - return [] - - else: - for key, _ in cursor: - if not key.startswith(prefix.encode()): - break - - keys.append(key.decode()) - - if len(keys) >= length > 0: - break - - return keys + return keys if length == 0 else keys[:length] def keys(self): - keys = [] - - with self.db_reader.begin() as tx: - cursor = tx.cursor() - for key, _ in cursor: - keys.append(key.decode()) - - return keys - - def __getitem__(self, item: str): - value = self.get(item) - if value is None: - raise KeyError - return value - - def __setitem__(self, key: str, value): - self.set(key, value) - - def __delitem__(self, key: str): - with self.db_writer.begin(write=True) as tx: - tx.delete(key.encode()) - + return self.iter() class WebDriver(InMemDriver): def __init__(self, masternode='http://masternode-01.lamden.io'): @@ -439,66 +302,91 @@ def get(self, item: str): r = requests.get(f'{self.masternode}/contracts/{contract}/{variable}?key={keys}') return decode(r.json()['value']) - class CacheDriver: - def __init__(self, driver: Driver=Driver()): - self.driver = driver - self.cache = {} + def __init__(self, driver=None): + self.pending_writes = {} # L2 cache + self.cache = {} # L1 cache + self.driver = driver or FSDriver() # L0 cache - self.reads = set() - self.pending_writes = {} + self.pending_reads = {} self.pending_deltas = {} - def soft_apply(self, hcl: str, state_changes: dict): - deltas = {} + def find(self, key: str): + value = self.pending_writes.get(key) + if value is not None: + return value - for k, v in state_changes.items(): - current = self.get(k) - deltas[k] = (current, v) + value = self.cache.get(key) + if value is not None: + return value - self.set(k, v) + value = self.driver.get(key) + if value is not None: + return value - self.pending_deltas[hcl] = deltas + return None - def get(self, key: str, mark=True): - # Try to get from cache - v = self.cache.get(key) - if v is not None: - rt.deduct_read(*encode_kv(key, v)) - return v + def get(self, key: str, save: bool = True): - # If it doesn't exist, get from db, add to cache - dv = self.driver.get(key) - rt.deduct_read(*encode_kv(key, dv)) + value = self.find(key) - self.cache[key] = dv + if save: + if self.pending_reads.get(key) is None: + self.pending_reads[key] = value - # Add key to reads - if mark: - self.reads.add(key) + if value is not None: + rt.deduct_read(*encode_kv(key, value)) - return dv + return value - def set(self, key, value, mark=True): + def set(self, key, value): rt.deduct_write(*encode_kv(key, value)) + if self.pending_reads.get(key) is None: + self.get(key) + if type(value) == decimal.Decimal or type(value) == float: value = ContractingDecimal(str(value)) - self.cache[key] = value - if mark: - self.pending_writes[key] = value + self.pending_writes[key] = value - def delete(self, key, mark=True): - self.set(key, None, mark=mark) + def delete(self, key): + self.set(key, None) + + #TODO: Fix bug where rolling back on a key written to twice rolls back to the initial state instead of the immediate previous value + def soft_apply(self, hcl: str): + deltas = {} - def commit(self): for k, v in self.pending_writes.items(): - if v is None: - self.driver.delete(k) - else: - self.driver.set(k, v) + current = self.pending_reads.get(k) + deltas[k] = (current, v) + + self.cache[k] = v + + self.pending_deltas[hcl] = { + 'writes': deltas, + 'reads': self.pending_reads + } + + # Clear the top cache + self.pending_reads = {} + self.pending_writes.clear() + + def soft_apply_rewards(self, hcl: str): + deltas = {} + + for k, v in self.pending_writes.items(): + current = self.pending_reads.get(k) + deltas[k] = (current, v) + + self.cache[k] = v + + self.pending_deltas[hcl]['rewards'] = deltas + + # Clear the top cache + self.pending_reads = {} + self.pending_writes.clear() def hard_apply(self, hlc): # see if the HCL even exists @@ -511,13 +399,9 @@ def hard_apply(self, hlc): for _hlc, _deltas in sorted(self.pending_deltas.items()): # Run through all state changes, taking the second value, which is the post delta - for key, delta in _deltas.items(): + for key, delta in _deltas['writes'].items(): self.driver.set(key, delta[1]) - - try: - self.cache.pop(key) - except KeyError: - pass + # self.cache[key] = delta[1] # Add the key ( to_delete.append(_hlc) @@ -527,30 +411,70 @@ def hard_apply(self, hlc): # Remove the deltas from the set [self.pending_deltas.pop(key) for key in to_delete] - def rollback(self): - # Run through the state changes in reverse, reversing the newest to the oldest - for _hlc, _deltas in reversed(sorted(self.pending_deltas.items())): - # Run through all state changes, taking the first value, which is the pre delta - for key, delta in _deltas.items(): - self.set(key, delta[0]) + # Same as hard apply but for only the most recent changes and the cache + def commit(self): + self.cache.update(self.pending_writes) - self.pending_deltas.clear() + for k, v in self.cache.items(): + if v is None: + self.driver.delete(k) + else: + self.driver.set(k, v) - def clear_pending_state(self): self.cache.clear() - self.reads.clear() self.pending_writes.clear() + self.pending_reads = {} + + def rollback(self, hlc=None): + if hlc is None: + # Returns to disk state which should be whatever it was prior to any write sessions + self.cache.clear() + self.pending_reads = {} + self.pending_writes.clear() + self.pending_deltas.clear() + else: + to_delete = [] + for _hlc, _deltas in sorted(self.pending_deltas.items())[::-1]: + # Clears the current reads/writes, and the reads/writes that get made when rolling back from the + # last HLC + self.pending_reads = {} + self.pending_writes.clear() + + + if _hlc < hlc: + # if we are less than the HLC then top processing anymore, this is our rollback point + break + else: + # if we are still greater than or equal to then mark this as delete and rollback its changes + to_delete.append(_hlc) + # Run through all state changes, taking the second value, which is the post delta + for key, delta in _deltas['writes'].items(): + # self.set(key, delta[0]) + self.cache[key] = delta[0] + + # Remove the deltas from the set + [self.pending_deltas.pop(key) for key in to_delete] + + def clear_pending_state(self): + self.rollback() class ContractDriver(CacheDriver): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.delimiter = '.' + self.log = logging.getLogger('Driver') def items(self, prefix=''): # Get all of the items in the cache currently _items = {} keys = set() + + for k, v in self.pending_writes.items(): + if k.startswith(prefix) and v is not None: + _items[k] = v + keys.add(k) + for k, v in self.cache.items(): if k.startswith(prefix) and v is not None: _items[k] = v @@ -579,11 +503,11 @@ def make_key(self, contract, variable, args=[]): def get_var(self, contract, variable, arguments=[], mark=True): key = self.make_key(contract, variable, arguments) - return self.get(key, mark=mark) + return self.get(key) def set_var(self, contract, variable, arguments=[], value=None, mark=True): key = self.make_key(contract, variable, arguments) - self.set(key, value, mark=mark) + self.set(key, value) def get_contract(self, name): return self.get_var(name, CODE_KEY) @@ -628,18 +552,45 @@ def flush(self): def get_contract_keys(self, name): return self.keys(name) - # Set cache to None - # Set pending writes to none - # def delete(self, key): - # # if self.cache.get(key) is not None: - # # del self.cache[key] - # # - # # if self.pending_writes.get(key) is not None: - # # del self.pending_writes[key] - # # - # # self.driver.delete(key) - # self.cache[key] = None - # self.pending_writes[key] = None + def rollback_drivers(self, hlc_timestamp): + # Roll back the current state to the point of the last block consensus + self.log.debug(f"Length of Pending Deltas BEFORE {len(self.driver.pending_deltas.keys())}") + self.log.debug(f"rollback to hlc_timestamp: {hlc_timestamp}") + + if hlc_timestamp is None: + # Returns to disk state which should be whatever it was prior to any write sessions + self.cache.clear() + self.reads = set() + self.pending_writes.clear() + self.pending_deltas.clear() + else: + to_delete = [] + for _hlc, _deltas in sorted(self.pending_deltas.items())[::-1]: + # Clears the current reads/writes, and the reads/writes that get made when rolling back from the + # last HLC + self.reads = set() + self.pending_writes.clear() + + + if _hlc < hlc_timestamp: + self.log.debug(f"{_hlc} is less than {hlc_timestamp}, breaking!") + # if we are less than the HLC then top processing anymore, this is our rollback point + break + else: + # if we are still greater than or equal to then mark this as delete and rollback its changes + to_delete.append(_hlc) + # Run through all state changes, taking the second value, which is the post delta + for key, delta in _deltas['writes'].items(): + # self.set(key, delta[0]) + self.cache[key] = delta[0] + + # Remove the deltas from the set + self.log.debug(to_delete) + [self.pending_deltas.pop(key) for key in to_delete] + + #self.driver.rollback(hlc=hlc_timestamp) + + self.log.debug(f"Length of Pending Deltas AFTER {len(self.driver.pending_deltas.keys())}") class AsyncContractDriver: @@ -700,18 +651,3 @@ def get_compiled(self, name): def get_contract_keys(self, name): return self.keys(name) - - # Set cache to None - # Set pending writes to none - # def delete(self, key): - # # if self.cache.get(key) is not None: - # # del self.cache[key] - # # - # # if self.pending_writes.get(key) is not None: - # # del self.pending_writes[key] - # # - # # self.driver.delete(key) - # self.cache[key] = None - # self.pending_writes[key] = None - - diff --git a/contracting/db/encoder.py b/contracting/db/encoder.py index 0bdbe37f..40661bcf 100644 --- a/contracting/db/encoder.py +++ b/contracting/db/encoder.py @@ -170,6 +170,9 @@ def convert(k, v): def convert_dict(d): + if not isinstance(d, dict): + return d + d2 = dict() for k, v in d.items(): if k in TYPES: diff --git a/contracting/db/hdf5/__init__.py b/contracting/db/hdf5/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c new file mode 100644 index 00000000..9c8d1d8c --- /dev/null +++ b/contracting/db/hdf5/h5c.c @@ -0,0 +1,259 @@ +#define PY_SSIZE_T_CLEAN +#include +#include +#include +#include +#include + +// HDF5 Reference Manual: https://support.hdfgroup.org/HDF5/doc/RM/RM_H5Front.html + +#define ATTR_LEN_MAX 64000 // http://davis.lbl.gov/Manuals/HDF5-1.8.7/UG/13_Attributes.html#SpecIssues +#define ATTR_VALUE "value" +#define ATTR_BLOCK "block" +#define LOCK_SUFFIX "-lock" + +static char dirname_buf[PATH_MAX + 1]; + +static void +lock_acquire(char *filepath) +{ + strcat(dirname_buf, filepath); + strcat(dirname_buf, LOCK_SUFFIX); + while(mkdir(dirname_buf, S_IRWXU) != 0) + ; + memset(dirname_buf, 0, sizeof(dirname_buf)); +} + +static void +lock_release(char *filepath) +{ + strcat(dirname_buf, filepath); + strcat(dirname_buf, LOCK_SUFFIX); + rmdir(dirname_buf); + memset(dirname_buf, 0, sizeof(dirname_buf)); +} + +static void +write_attr(hid_t gid, char *name, char *value) +{ + H5Adelete(gid, name); + if(value) + { + hid_t atype = H5Tcopy(H5T_C_S1); + H5Tset_size(atype, strlen(value)); + hid_t aid = H5Acreate(gid, name, atype, H5Screate(H5S_SCALAR), H5P_DEFAULT, H5P_DEFAULT); + H5Awrite(aid, atype, value); + H5Aclose(aid); + H5Tclose(atype); + } +} + +static PyObject * +set(PyObject *self, PyObject *args) +{ +#ifndef DEBUG + H5Eset_auto2(H5P_DEFAULT, NULL, NULL); +#endif + + char *filepath, *group, *value, *blocknum; + if(!PyArg_ParseTuple(args, "sszz", &filepath, &group, &value, &blocknum)) + return NULL; + + lock_acquire(filepath); + + hid_t fid = H5Fopen(filepath, H5F_ACC_RDWR, H5P_DEFAULT); + if(fid < 0) + { + fid = H5Fcreate(filepath, H5F_ACC_EXCL, H5P_DEFAULT, H5P_DEFAULT); + if(fid < 0) + { + lock_release(filepath); + return PyErr_Format(PyExc_OSError, "failed to open/create file \"%s\"", filepath); + } + } + + hid_t gid = H5Gopen(fid, group, H5P_DEFAULT); + if(gid < 0) + { + hid_t lcpl = H5Pcreate(H5P_LINK_CREATE); + H5Pset_create_intermediate_group(lcpl, 1); + gid = H5Gcreate(fid, group, lcpl, H5P_DEFAULT, H5P_DEFAULT); + H5Pclose(lcpl); + } + + write_attr(gid, ATTR_VALUE, value); + write_attr(gid, ATTR_BLOCK, blocknum); + + H5Gclose(gid); + H5Fclose(fid); + + lock_release(filepath); + + Py_RETURN_NONE; +} + +static PyObject * +get_attr(char *filepath, char *group, char *name) +{ + static char buf[ATTR_LEN_MAX + 1]; + + lock_acquire(filepath); + + hid_t fid = H5Fopen(filepath, H5F_ACC_RDONLY, H5P_DEFAULT); + if(fid < 0) + { + lock_release(filepath); + Py_RETURN_NONE; + } + + hid_t aid = H5Aopen_by_name(fid, group, name, H5P_DEFAULT, H5P_DEFAULT); + if(aid < 0) + { + H5Fclose(fid); + lock_release(filepath); + Py_RETURN_NONE; + } + + hid_t atype = H5Tcopy(H5T_C_S1); + H5Tset_size(atype, sizeof(buf)); + memset(buf, 0, sizeof(buf)); + if(H5Aread(aid, atype, buf) < 0) + { + H5Tclose(atype); + H5Aclose(aid); + H5Fclose(fid); + lock_release(filepath); + Py_RETURN_NONE; + } + + H5Tclose(atype); + H5Aclose(aid); + H5Fclose(fid); + + lock_release(filepath); + + return PyUnicode_FromString(buf); +} + +static PyObject * +get_value(PyObject *self, PyObject *args) +{ +#ifndef DEBUG + H5Eset_auto2(H5P_DEFAULT, NULL, NULL); +#endif + + char *filepath, *group; + if(!PyArg_ParseTuple(args, "ss", &filepath, &group)) + return NULL; + + return get_attr(filepath, group, ATTR_VALUE); +} + +static PyObject * +get_block(PyObject *self, PyObject *args) +{ +#ifndef DEBUG + H5Eset_auto2(H5P_DEFAULT, NULL, NULL); +#endif + + char *filepath, *group; + if(!PyArg_ParseTuple(args, "ss", &filepath, &group)) + return NULL; + + return get_attr(filepath, group, ATTR_BLOCK); +} + +static PyObject * +delete(PyObject *self, PyObject *args) +{ +#ifndef DEBUG + H5Eset_auto2(H5P_DEFAULT, NULL, NULL); +#endif + + char *filepath, *group; + if(!PyArg_ParseTuple(args, "ss", &filepath, &group)) + return NULL; + + lock_acquire(filepath); + + hid_t fid = H5Fopen(filepath, H5F_ACC_RDWR, H5P_DEFAULT); + hid_t gid = H5Gopen(fid, group, H5P_DEFAULT); + H5Adelete(gid, ATTR_VALUE); + H5Adelete(gid, ATTR_BLOCK); + + H5Gclose(gid); + H5Fclose(fid); + + lock_release(filepath); + + Py_RETURN_NONE; +} + +static herr_t +store_group_name(hid_t id, const char *name, const H5O_info_t *object_info, void *op_data) +{ + if(strcmp(name, ".") == 0 || object_info->num_attrs == 0) + return 0; + return PyList_Append((PyObject *) op_data, PyUnicode_FromString(name)); +} + +static PyObject * +get_groups(PyObject *self, PyObject *args) +{ +#ifndef DEBUG + H5Eset_auto2(H5P_DEFAULT, NULL, NULL); +#endif + + char *filepath; + if(!PyArg_ParseTuple(args, "s", &filepath)) + return NULL; + + lock_acquire(filepath); + + hid_t fid = H5Fopen(filepath, H5F_ACC_RDONLY, H5P_DEFAULT); + if(fid < 0) + { + lock_release(filepath); + Py_RETURN_NONE; + } + + PyObject *group_names = PyList_New(0); +#if (H5_VERS_MAJOR == 1 && H5_VERS_MINOR >= 12) || H5_VERS_MAJOR > 1 + if(H5Ovisit(fid, H5_INDEX_NAME, H5_ITER_NATIVE, store_group_name, group_names, H5O_INFO_ALL) < 0) +#else + if(H5Ovisit(fid, H5_INDEX_NAME, H5_ITER_NATIVE, store_group_name, group_names) < 0) +#endif + { + H5Fclose(fid); + lock_release(filepath); + Py_RETURN_NONE; + } + + H5Fclose(fid); + + lock_release(filepath); + + return group_names; +} + +static PyMethodDef methods[] = { + {"set", set, METH_VARARGS, "Set value"}, + {"get_value", get_value, METH_VARARGS, "Get value"}, + {"get_block", get_block, METH_VARARGS, "Get block"}, + {"delete", delete, METH_VARARGS, "Delete value & block"}, + {"get_groups", get_groups, METH_VARARGS, "Get groups"} +}; + +static struct PyModuleDef h5cmodule = { + PyModuleDef_HEAD_INIT, + "h5c", + NULL, + -1, + methods +}; + +PyMODINIT_FUNC +PyInit_h5c(void) +{ + return PyModule_Create(&h5cmodule); +} diff --git a/contracting/execution/executor.py b/contracting/execution/executor.py index 19b791df..2210cf1a 100644 --- a/contracting/execution/executor.py +++ b/contracting/execution/executor.py @@ -68,9 +68,19 @@ def execute(self, sender, contract_name, function_name, kwargs, sender) balance = driver.get(balances_key) + + if type(balance) == dict: + balance = ContractingDecimal(balance.get('__fixed__')) + if balance is None: balance = 0 + log.debug({ + 'balance': balance, + 'stamp_cost': stamp_cost, + 'stamps': stamps + }) + assert balance * stamp_cost >= stamps, 'Sender does not have enough stamps for the transaction. \ Balance at key {} is {}'.format(balances_key, balance) @@ -83,16 +93,22 @@ def execute(self, sender, contract_name, function_name, kwargs, 'signer': sender, 'caller': sender, 'this': contract_name, - 'owner': driver.get_owner(contract_name) + 'entry': (contract_name, function_name), + 'owner': driver.get_owner(contract_name), + 'submission_name': None } + module = importlib.import_module(contract_name) + func = getattr(module, function_name) + if runtime.rt.context.owner is not None and runtime.rt.context.owner != runtime.rt.context.caller: raise Exception(f'Caller {runtime.rt.context.caller} is not the owner {runtime.rt.context.owner}!') decimal.setcontext(CONTEXT) - module = importlib.import_module(contract_name) - func = getattr(module, function_name) + ## add the contract name to the context on a submission call + if contract_name == config.SUBMISSION_CONTRACT_NAME: + runtime.rt.context._base_state['submission_name'] = kwargs.get('name') for k, v in kwargs.items(): if type(v) == float: @@ -155,7 +171,7 @@ def execute(self, sender, contract_name, function_name, kwargs, 'result': result, 'stamps_used': stamps_used, 'writes': deepcopy(driver.pending_writes), - 'reads': driver.reads + 'reads': driver.pending_reads } disable_restricted_imports() diff --git a/contracting/execution/runtime.py b/contracting/execution/runtime.py index f19a4f3a..94c2a039 100644 --- a/contracting/execution/runtime.py +++ b/contracting/execution/runtime.py @@ -48,12 +48,22 @@ def signer(self): def owner(self): return self._get_state()['owner'] + @property + def entry(self): + return self._get_state()['entry'] + + @property + def submission_name(self): + return self._get_state()['submission_name'] + _context = Context({ 'this': None, 'caller': None, 'owner': None, - 'signer': None + 'signer': None, + 'entry': None, + 'submission_name': None }) WRITE_MAX = 1024 * 64 diff --git a/contracting/stdlib/bridge/access.py b/contracting/stdlib/bridge/access.py index 2d1d24f1..b21aa4ed 100644 --- a/contracting/stdlib/bridge/access.py +++ b/contracting/stdlib/bridge/access.py @@ -17,7 +17,9 @@ def __enter__(self, *args, **kwargs): 'owner': driver.get_owner(self.contract), 'caller': current_state['this'], 'signer': current_state['signer'], - 'this': self.contract + 'this': self.contract, + 'entry': current_state['entry'], + 'submission_name': current_state['submission_name'], } rt.context._add_state(state) diff --git a/contracting/stdlib/bridge/random.py b/contracting/stdlib/bridge/random.py index 27f38673..5b8fe428 100644 --- a/contracting/stdlib/bridge/random.py +++ b/contracting/stdlib/bridge/random.py @@ -28,12 +28,15 @@ def seed(aux_salt=None): block_hash = rt.env.get('block_hash') or '0' __input_hash = rt.env.get('__input_hash') or '0' - # Auxillary salt is used to create completely unique random seeds based on some other properties (optional) - auxillary_salt = '' + # Auxiliary salt is used to create completely unique random seeds based on some other properties (optional) + auxiliary_salt = '' if aux_salt is not None and rt.env.get(aux_salt): - auxillary_salt = str(rt.env.get(aux_salt)) + auxiliary_salt = str(rt.env.get(aux_salt)) + else: + if rt.env.get("AUXILIARY_SALT"): + auxiliary_salt = str(rt.env.get("AUXILIARY_SALT")) - s = block_height + block_hash + __input_hash + auxillary_salt + s = block_height + block_hash + __input_hash + auxiliary_salt random.seed(s) Seeded.s = True diff --git a/setup.py b/setup.py index 77b23362..f5f531f4 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,15 @@ +from distutils import errors +from distutils.command.build_ext import build_ext, CCompilerError, DistutilsExecError, DistutilsPlatformError from setuptools import setup, find_packages from setuptools.extension import Extension -from distutils.command.build_ext import build_ext, CCompilerError, DistutilsExecError, DistutilsPlatformError -from distutils import errors -import sys +from sys import platform +import sys, subprocess, pathlib major = 0 -__version__ = '1.0.5.2' +__version__ = '1.1.6' -requirements = ['astor', 'pymongo', 'autopep8', 'stdlib_list'] +requirements = ['astor==0.8.1', 'pymongo==3.12.3', 'autopep8==1.5.7', "stdlib_list==0.8.0", 'motor==2.5.1'] ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError) @@ -42,6 +43,28 @@ def build_extension(self, ext): raise BuildFailed() raise +def pkgconfig(package): + flag_map = {'-I': 'include_dirs', '-L': 'library_dirs', '-l': 'libraries'} + res = {} + + if platform == "linux" or platform == "linux2": + output = subprocess.getoutput('pkg-config --cflags --libs {}'.format(package)) + for token in output.strip().split(): + res.setdefault(flag_map.get(token[:2]), []).append(token[2:]) + + elif platform == "darwin": + res['libraries'] = [f'{package}'] + output = subprocess.getoutput(f'brew --prefix {package}') + if 'Error:' in output or not pathlib.Path(output).is_dir(): + raise ModuleNotFoundError(f'{output}\nInstall "{package}"" package using brew. "brew install {package}"') + res['include_dirs'] = [output + '/include/'] + res['library_dirs'] = [output + '/lib/'] + + elif platform == "win32": + raise NotImplemented("Cannot install on Windows") + + return res + setup( name='contracting', version=__version__, @@ -58,6 +81,7 @@ def build_extension(self, ext): include_package_data=True, ext_modules=[ Extension('contracting.execution.metering.tracer', sources=['contracting/execution/metering/tracer.c']), + Extension('contracting.db.hdf5.h5c', sources=['contracting/db/hdf5/h5c.c'], **pkgconfig('hdf5')) ], cmdclass={ 'build_ext': ve_build_ext, diff --git a/tests/integration/test_misc_contracts.py b/tests/integration/test_misc_contracts.py index ae8a675e..25f5d158 100644 --- a/tests/integration/test_misc_contracts.py +++ b/tests/integration/test_misc_contracts.py @@ -8,7 +8,7 @@ def too_many_writes(): @export def single(): - v.set('a' * (32 * 1024 + 1)) + v.set('a' * (64 * 1024 + 1)) @export def multiple(): diff --git a/tests/integration/test_rich_ctx_calling.py b/tests/integration/test_rich_ctx_calling.py index be5838f6..e0211eef 100644 --- a/tests/integration/test_rich_ctx_calling.py +++ b/tests/integration/test_rich_ctx_calling.py @@ -1,7 +1,6 @@ from unittest import TestCase from contracting.client import ContractingClient - def con_module1(): @export def get_context2(): @@ -10,7 +9,9 @@ def get_context2(): 'owner': ctx.owner, 'this': ctx.this, 'signer': ctx.signer, - 'caller': ctx.caller + 'caller': ctx.caller, + 'entry': ctx.entry, + 'submission_name': ctx.submission_name } @@ -30,7 +31,9 @@ def call_me_again_again(): 'owner': ctx.owner, 'this': ctx.this, 'signer': ctx.signer, - 'caller': ctx.caller + 'caller': ctx.caller, + 'entry': ctx.entry, + 'submission_name': ctx.submission_name }) @@ -45,7 +48,9 @@ def called_from_a_far(): 'owner': ctx.owner, 'this': ctx.this, 'signer': ctx.signer, - 'caller': ctx.caller + 'caller': ctx.caller, + 'entry': ctx.entry, + 'submission_name': ctx.submission_name }] @export @@ -53,16 +58,37 @@ def con_called_from_a_far_stacked(): m = importlib.import_module('con_all_in_one') return m.call() +def con_submission_name_test(): + submission_name = Variable() + + @construct + def seed(): + submission_name.set(ctx.submission_name) + + @export + def get_submission_context(): + return submission_name.get() + + @export + def get_entry_context(): + con_name, func_name = ctx.entry + + return { + 'entry_contract': con_name, + 'entry_function': func_name + } + + class TestRandomsContract(TestCase): def setUp(self): self.c = ContractingClient(signer='stu') self.c.flush() - self.c.submit(con_module1) - - self.c.submit(con_all_in_one) - self.c.submit(con_dynamic_import) + self.c.submit(f=con_module1) + self.c.submit(f=con_all_in_one) + self.c.submit(f=con_dynamic_import) + self.c.submit(f=con_submission_name_test) def tearDown(self): self.c.flush() @@ -72,10 +98,12 @@ def test_ctx2(self): res = module.get_context2() expected = { 'name': 'get_context2', + 'entry': ('con_module1', 'get_context2'), 'owner': None, 'this': 'con_module1', 'signer': 'stu', - 'caller': 'stu' + 'caller': 'stu', + 'submission_name': None } self.assertDictEqual(res, expected) @@ -85,10 +113,12 @@ def test_multi_call_doesnt_affect_parameters(self): expected = { 'name': 'call_me_again_again', + 'entry': ('con_all_in_one', 'call_me'), 'owner': None, 'this': 'con_all_in_one', 'signer': 'stu', - 'caller': 'stu' + 'caller': 'stu', + 'submission_name': None } self.assertDictEqual(res, expected) @@ -99,19 +129,36 @@ def test_dynamic_call(self): expected1 = { 'name': 'call_me_again_again', + 'entry': ('con_dynamic_import', 'called_from_a_far'), 'owner': None, 'this': 'con_all_in_one', 'signer': 'stu', - 'caller': 'con_dynamic_import' + 'caller': 'con_dynamic_import', + 'submission_name': None } expected2 = { 'name': 'called_from_a_far', + 'entry': ('con_dynamic_import', 'called_from_a_far'), 'owner': None, 'this': 'con_dynamic_import', 'signer': 'stu', - 'caller': 'stu' + 'caller': 'stu', + 'submission_name': None } self.assertDictEqual(res1, expected1) self.assertDictEqual(res2, expected2) + + def test_submission_name_in_construct_function(self): + contract = self.c.get_contract('con_submission_name_test') + submission_name = contract.get_submission_context() + + self.assertEqual("con_submission_name_test", submission_name) + + def test_entry_context(self): + contract = self.c.get_contract('con_submission_name_test') + details = contract.get_entry_context() + + self.assertEqual("con_submission_name_test", details.get('entry_contract')) + self.assertEqual("get_entry_context", details.get('entry_function')) diff --git a/tests/integration/test_seneca_client_randoms.py b/tests/integration/test_seneca_client_randoms.py index cc51b672..b9ebc0ad 100644 --- a/tests/integration/test_seneca_client_randoms.py +++ b/tests/integration/test_seneca_client_randoms.py @@ -100,10 +100,11 @@ def test_random_num_one_vs_two(self): self.assertEqual(k2, random.randrange(1000)) + ''' TEST CASE IS IRRELEVANT as getrandbits will never sync with system random. def test_random_getrandbits(self): b = self.random_contract.random_bits(k=20) - random.seed('000') + random.seed('000'z) cards = [1, 2, 3, 4, 5, 6, 7, 8] random.shuffle(cards) @@ -111,6 +112,7 @@ def test_random_getrandbits(self): random.shuffle(cards) self.assertEqual(b, random.getrandbits(20)) + ''' def test_random_range_int(self): a = self.random_contract.int_in_range(a=100, b=50000) @@ -131,3 +133,14 @@ def test_random_choice(self): cc = random.choices(c, k=2) self.assertListEqual(cities, cc) + + def test_auxilary_salt(self): + cards_1 = self.random_contract.shuffle_cards(environment={ + 'AUXILIARY_SALT': 'ffd8ded9ced929a41dae83b1f22a6a31b52f79bbf4cdabe6a27d9646dd2bd725fc29c8bc122cb9e37a2904da00e34df499ee7a897505d1de3f0511f9f9c1150c'}) + cards_2 = self.random_contract.shuffle_cards(environment={ + 'AUXILIARY_SALT': 'ffd8ded9ced929a41dae83b1f22a6a31b52f79bbf4cdabe6a27d9646dd2bd725fc29c8bc122cb9e37a2904da00e34df499ee7a897505d1de3f0511f9f9c1150c'}) + cards_3 = self.random_contract.shuffle_cards(environment={ + 'AUXILIARY_SALT': 'f79bbded9ced929a41dae83b1f22a6a31b52f79bbf4cdabe6a27d9646dd2bd725fc29c8bc122cb9e37a2904da00e34df499ee7a897505d1de3f0511f9f9c1150c'}) + + self.assertEqual(cards_1, cards_2) + self.assertNotEqual(cards_1, cards_3) diff --git a/tests/integration/test_stamp_deduction.py b/tests/integration/test_stamp_deduction.py index 61188f81..9736bb1d 100644 --- a/tests/integration/test_stamp_deduction.py +++ b/tests/integration/test_stamp_deduction.py @@ -65,10 +65,8 @@ def test_too_few_stamps_fails_and_deducts_properly(self): print(prior_balance) - small_amount_of_stamps = 1 * STAMPS_PER_TAU - output = self.e.execute('stu', 'currency', 'transfer', kwargs={'amount': 100, 'to': 'colin'}, - stamps=small_amount_of_stamps, auto_commit=True) + stamps=1, auto_commit=True) print(output) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 7140660c..781c7b73 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1,16 +1,27 @@ from unittest import TestCase from contracting.client import ContractingClient - +from contracting.db.driver import InMemDriver, ContractDriver +import os +from pathlib import Path class TestClient(TestCase): def setUp(self): - self.client = ContractingClient() - self.client.flush() + self.client = None + self.raw_driver = InMemDriver() + self.contract_driver = ContractDriver(driver=self.raw_driver) + + submission_file_path = f'{Path.cwd().parent.parent}/contracting/contracts/submission.s.py' + with open(submission_file_path) as f: + self.submission_contract_file = f.read() def tearDown(self): - self.client.flush() + if self.client: + self.client.flush() def test_set_submission_updates_contract_file(self): + self.client = ContractingClient(driver=self.contract_driver) + self.client.flush() + submission_1_code = self.client.raw_driver.get('submission.__code__') self.client.flush() @@ -19,3 +30,72 @@ def test_set_submission_updates_contract_file(self): submission_2_code = self.client.raw_driver.get('submission.__code__') self.assertNotEqual(submission_1_code, submission_2_code) + + def test_can_create_instance_without_submission_contract(self): + self.client = ContractingClient(submission_filename=None, driver=self.contract_driver) + + self.assertIsNotNone(self.client) + + + def test_gets_submission_contract_from_state_if_no_filename_provided(self): + self.contract_driver.set_contract(name='submission', code=self.submission_contract_file) + self.contract_driver.commit() + + self.client = ContractingClient(submission_filename=None, driver=self.contract_driver) + + self.assertIsNotNone(self.client.submission_contract) + + def test_set_submission_contract__sets_from_submission_filename_property(self): + self.client = ContractingClient(driver=self.contract_driver) + + self.client.raw_driver.flush() + self.client.raw_driver.clear_pending_state() + self.client.submission_contract = None + + contract = self.client.raw_driver.get_contract('submission') + self.assertIsNone(contract) + self.assertIsNone(self.client.submission_contract) + + self.client.set_submission_contract() + + contract = self.client.raw_driver.get_contract('submission') + self.assertIsNotNone(contract) + self.assertIsNotNone(self.client.submission_contract) + + def test_set_submission_contract__sets_from_submission_from_state(self): + self.client = ContractingClient(driver=self.contract_driver) + + self.client.raw_driver.flush() + self.client.raw_driver.clear_pending_state() + self.client.submission_contract = None + + contract = self.client.raw_driver.get_contract('submission') + self.assertIsNone(contract) + self.assertIsNone(self.client.submission_contract) + + self.contract_driver.set_contract(name='submission', code=self.submission_contract_file) + self.contract_driver.commit() + + self.client.set_submission_contract() + + contract = self.client.raw_driver.get_contract('submission') + self.assertIsNotNone(contract) + self.assertIsNotNone(self.client.submission_contract) + + def test_set_submission_contract__no_contract_provided_or_found_raises_AssertionError(self): + self.client = ContractingClient(driver=self.contract_driver) + + self.client.raw_driver.flush() + self.client.raw_driver.clear_pending_state() + self.client.submission_filename = None + + with self.assertRaises(AssertionError): + self.client.set_submission_contract() + + def test_submit__raises_AssertionError_if_no_submission_contract_set(self): + self.client = ContractingClient(submission_filename=None, driver=self.contract_driver) + + with self.assertRaises(AssertionError): + self.client.submit(f="") + + diff --git a/tests/unit/test_linter.py b/tests/unit/test_linter.py index 66459c0c..a3567556 100644 --- a/tests/unit/test_linter.py +++ b/tests/unit/test_linter.py @@ -439,7 +439,7 @@ def greeting(name: str) -> str: c = ast.parse(code) chk = self.l.check(c) - self.assertEqual(chk, ['Line 2 : S18- Illegal use of return annotation : str']) + self.assertEqual(['Line 2 : S18- Illegal use of return annotation : str'], chk) def test_contract_annotation(self): diff --git a/tests/unit/test_new_cache_driver.py b/tests/unit/test_new_cache_driver.py index 36b9c47c..04434330 100644 --- a/tests/unit/test_new_cache_driver.py +++ b/tests/unit/test_new_cache_driver.py @@ -11,22 +11,12 @@ def setUp(self): def test_get_adds_to_read(self): self.c.get('thing') - self.assertTrue('thing' in self.c.reads) + self.assertTrue('thing' in self.c.pending_reads) def test_set_adds_to_cache_and_pending_writes(self): self.c.set('thing', 1234) - self.assertEqual(self.c.cache['thing'], 1234) self.assertEqual(self.c.pending_writes['thing'], 1234) - def test_object_added_to_cache_if_read_from_db(self): - self.assertIsNone(self.c.cache.get('thing')) - - self.d.set('thing', 8999) - - self.c.get('thing') - - self.assertEqual(self.c.cache['thing'], 8999) - def test_object_in_cache_returns_from_cache(self): self.d.set('thing', 8999) self.c.get('thing') @@ -58,28 +48,24 @@ def test_clear_pending_state_resets_all_variables(self): self.c.set('thing2', 1235) self.c.get('something') - self.assertTrue(len(self.c.cache) > 0) - self.assertTrue(len(self.c.reads) > 0) + self.assertTrue(len(self.c.pending_reads) > 0) self.assertTrue(len(self.c.pending_writes) > 0) - self.c.clear_pending_state() + self.c.rollback() - self.assertFalse(len(self.c.cache) > 0) - self.assertFalse(len(self.c.reads) > 0) + self.assertFalse(len(self.c.pending_reads) > 0) self.assertFalse(len(self.c.pending_writes) > 0) def test_soft_apply_adds_changes_to_pending_deltas(self): - self.c.set('thing1', 9999) - - state_changes = { - 'thing1': 8888 - } + self.c.driver.set('thing1', 9999) - self.c.soft_apply('0', state_changes) + self.c.set('thing1', 8888) + self.c.soft_apply('0') expected_deltas = { '0': { - 'thing1': (9999, 8888) + 'writes': {'thing1': (9999, 8888)}, + 'reads': {'thing1': 9999} } } @@ -89,11 +75,8 @@ def test_soft_apply_applies_the_changes_to_the_driver_but_not_hard_driver(self): self.c.set('thing1', 9999) self.c.commit() - state_changes = { - 'thing1': 8888 - } - - self.c.soft_apply('0', state_changes) + self.c.set('thing1', 8888) + self.c.soft_apply('0') res = self.c.get('thing1') @@ -104,11 +87,9 @@ def test_hard_apply_applies_hcl_if_exists(self): self.c.set('thing1', 9999) self.c.commit() - state_changes = { - 'thing1': 8888 - } + self.c.set('thing1', 8888) - self.c.soft_apply('0', state_changes) + self.c.soft_apply('0') self.c.hard_apply('0') res = self.c.get('thing1') @@ -117,94 +98,121 @@ def test_hard_apply_applies_hcl_if_exists(self): self.assertEqual(self.c.driver.get('thing1'), 8888) - def test_hard_apply_only_applies_changes_up_to_delta(self): + def test_rollback_applies_hcl_if_exists(self): self.c.set('thing1', 9999) self.c.commit() - state_changes = { - 'thing1': 8888 - } + self.c.set('thing1', 8888) - self.c.soft_apply('0', state_changes) + self.c.soft_apply('0') + self.c.rollback('0') - state_changes = { - 'thing1': 7777 - } + res = self.c.get('thing1') - self.c.soft_apply('1', state_changes) + self.assertEqual(res, 9999) - state_changes = { - 'thing1': 6666 - } + self.assertEqual(self.c.driver.get('thing1'), 9999) - self.c.soft_apply('2', state_changes) + def test_rollback_twice_returns(self): + self.c.set('thing1', 9999) + self.c.commit() - self.c.hard_apply('1') + self.c.set('thing1', 8888) + self.c.soft_apply('0') + + self.c.set('thing1', 7777) + self.c.soft_apply('1') + + self.c.set('thing1', 6666) + self.c.soft_apply('2') + + self.c.rollback('1') res = self.c.get('thing1') - self.assertEqual(res, 7777) + self.assertEqual(res, 8888) - self.assertEqual(self.c.driver.get('thing1'), 7777) + self.assertEqual(self.c.driver.get('thing1'), 9999) - def test_hard_apply_removes_hcls(self): + def test_rollback_removes_hlcs(self): self.c.set('thing1', 9999) self.c.commit() - state_changes = { - 'thing1': 8888 - } + self.c.set('thing1', 8888) + self.c.soft_apply('0') - self.c.soft_apply('0', state_changes) + self.c.set('thing1', 7777) + self.c.soft_apply('1') - state_changes = { - 'thing1': 7777 - } + self.c.set('thing1', 6666) + self.c.soft_apply('2') - self.c.soft_apply('1', state_changes) + self.c.rollback('1') - state_changes = { - 'thing1': 6666 - } + self.assertIsNone(self.c.pending_deltas.get('2')) + self.assertIsNone(self.c.pending_deltas.get('1')) + + def test_hard_apply_only_applies_changes_up_to_delta(self): + self.c.set('thing1', 9999) + self.c.commit() + + self.c.set('thing1', 8888) + self.c.soft_apply('0') + + self.c.set('thing1', 7777) + self.c.soft_apply('1') - self.c.soft_apply('2', state_changes) + self.c.set('thing1', 6666) + self.c.soft_apply('2') + + self.c.set('thing1', 5555) + self.c.soft_apply('3') + + self.c.hard_apply('1') + + res = self.c.get('thing1') + + self.assertEqual(res, 5555) + + self.assertEqual(self.c.driver.get('thing1'), 7777) + + def test_hard_apply_removes_hcls(self): + self.c.set('thing1', 9999) + self.c.commit() + + self.c.set('thing1', 8888) + self.c.soft_apply('0') + + self.c.set('thing1', 7777) + self.c.soft_apply('1') + + self.c.set('thing1', 6666) + self.c.soft_apply('2') self.c.hard_apply('0') - hcls = { - '1': { - 'thing1': (8888, 7777) - }, - '2': { - 'thing1': (7777, 6666) - } - } + hlcs = {'1': + {'writes': {'thing1': (8888, 7777)}, 'reads': {'thing1': 8888}}, + '2': + {'writes': {'thing1': (7777, 6666)}, 'reads': {'thing1': 7777}} + } - self.assertDictEqual(self.c.pending_deltas, hcls) + self.assertDictEqual(self.c.pending_deltas, hlcs) def test_rollback_returns_to_initial_state(self): self.c.set('thing1', 9999) self.c.commit() - state_changes = { - 'thing1': 8888 - } - - self.c.soft_apply('0', state_changes) + self.c.set('thing1', 8888) + self.c.soft_apply('0') self.assertEqual(self.c.get('thing1'), 8888) - state_changes = { - 'thing1': 7777 - } - - self.c.soft_apply('1', state_changes) + self.c.set('thing1', 7777) + self.c.soft_apply('1') self.assertEqual(self.c.get('thing1'), 7777) - state_changes = { - 'thing1': 6666 - } - - self.c.soft_apply('2', state_changes) + self.c.set('thing1', 6666) + self.c.soft_apply('2') self.assertEqual(self.c.get('thing1'), 6666) self.c.rollback() @@ -216,27 +224,46 @@ def test_rollback_removes_hlcs(self): self.c.set('thing1', 9999) self.c.commit() - state_changes = { - 'thing1': 8888 - } - - self.c.soft_apply('0', state_changes) + self.c.set('thing1', 8888) + self.c.soft_apply('0') self.assertEqual(self.c.get('thing1'), 8888) - state_changes = { - 'thing1': 7777 - } - - self.c.soft_apply('1', state_changes) + self.c.set('thing1', 7777) + self.c.soft_apply('1') self.assertEqual(self.c.get('thing1'), 7777) - state_changes = { - 'thing1': 6666 - } - - self.c.soft_apply('2', state_changes) + self.c.set('thing1', 6666) + self.c.soft_apply('2') self.assertEqual(self.c.get('thing1'), 6666) self.c.rollback() self.assertDictEqual(self.c.pending_deltas, {}) + + def test_find_returns_none(self): + x = self.c.find('none') + self.assertIsNone(x) + + def test_find_returns_driver(self): + self.c.driver.set('none', 123) + + x = self.c.find('none') + + self.assertEqual(x, 123) + + def test_find_returns_cache(self): + self.c.driver.set('none', 123) + self.c.cache['none'] = 999 + + x = self.c.find('none') + + self.assertEqual(x, 999) + + def test_find_returns_pending_writes(self): + self.c.driver.set('none', 123) + self.c.cache['none'] = 999 + self.c.pending_writes['none'] = 5555 + + x = self.c.find('none') + + self.assertEqual(x, 5555) diff --git a/tests/unit/test_new_contract_driver.py b/tests/unit/test_new_contract_driver.py index 04a8180a..7be80e76 100644 --- a/tests/unit/test_new_contract_driver.py +++ b/tests/unit/test_new_contract_driver.py @@ -5,6 +5,7 @@ import marshal from datetime import datetime + class TestContractDriver(TestCase): def setUp(self): self.d = Driver() @@ -129,6 +130,8 @@ def test_items_in_both_cache_and_db_works(self): for k, v in kvs_1.items(): self.c.set(k, v) + self.c.commit() + for k, v in kvs_2.items(): self.c.driver.set(k, v) diff --git a/tests/unit/test_new_driver.py b/tests/unit/test_new_driver.py index 6a8a8974..819ba3b4 100644 --- a/tests/unit/test_new_driver.py +++ b/tests/unit/test_new_driver.py @@ -1,9 +1,10 @@ -from unittest import TestCase -from contracting.db.driver import Driver, InMemDriver, FSDriver, LMDBDriver +from contracting import config +from contracting.db.driver import Driver, InMemDriver, FSDriver from contracting.db.encoder import MONGO_MAX_INT -from contracting.stdlib.bridge.time import Datetime, Timedelta from contracting.stdlib.bridge.decimal import ContractingDecimal +from contracting.stdlib.bridge.time import Datetime, Timedelta from decimal import Decimal +from unittest import TestCase import random SAMPLE_STRING = 'beef' @@ -530,304 +531,68 @@ class TestFSDriver(TestCase): # Flush this sucker every test def setUp(self): self.d = FSDriver() - self.d.flush() + self.block_num = 33 def tearDown(self): self.d.flush() def test_get_set(self): for v in TEST_DATA: - self.d.set('b', v) + self.d.set('b.b', v) - b = self.d.get('b') + b = self.d.get('b.b') self.assertEqual(v, b) if not isinstance(v, dict) else self.assertDictEqual(v, b) + self.assertEqual(self.d.get_block('b.b'), config.BLOCK_NUM_DEFAULT) - def test_delete(self): - for v in TEST_DATA: - self.d.set('b', v) - - b = self.d.get('b') - self.assertEqual(v, b) if not isinstance(v, dict) else self.assertDictEqual(v, b) - - self.d.delete('b') - - b = self.d.get('b') - self.assertIsNone(b) - - def test_iter(self): - - prefix_1_keys = [ - 'b77aa343e339bed781c7c2be1267cd597', - 'bc22ede6e6fb4046d78bf2f9d1f8afdb6', - 'b93dbb37d993846d70b8a92779cbfbfe9', - 'be1a2783019de6ea7ef169cc55e48a3ae', - 'b1fe8db32b9185d628f4c346f0455023e', - 'bef918f83b6a0d1e4f980013342807cf8', - 'b004cb9235acb5f689d20904692bc026e', - 'b869ff9519d67354816af90867f4a5425', - 'bcd8e9100dcb601f65e849c147e3e972e', - 'b0111919a698f9816862b4ae662a6ed06', - ] - - prefix_2_keys = [ - 'x37fbab0bd2e60563c79469e5be41e515', - 'x30c6eb2ad176773b5ce6d590d2472dfe', - 'x3d4fc9480f0a07b28aa7646d5066b54d', - 'x387c3d4ab7f0c1c6ef549198fc14b525', - 'x5c74dc83e132e435e8512599e1075bc0', - 'x1472425d0d9bb5ff511e132896d54b13', - 'x2cedb5c52163c22a0b5f179001959dd2', - 'x6223f65e553280cd25cadeac6657555c', - 'xeae18af37c223dde92a71fef55e64afe', - 'xb8810784ffb360cd3ffc57b1d088e537', - ] - - keys = prefix_1_keys + prefix_2_keys - random.shuffle(keys) - - for k in keys: - self.d.set(k, k) - - p1 = [] - - for k in self.d.iter(prefix='b'): - p1.append(k) - - prefix_1_keys.sort() - p1.sort() - - self.assertListEqual(prefix_1_keys, p1) - - p2 = [] - - for k in self.d.iter(prefix='x'): - p2.append(k) - - prefix_2_keys.sort() - p2.sort() - - self.assertListEqual(prefix_2_keys, p2) - - def test_set_object_returns_properly(self): - thing = { - 'a': 123, - 'b': False, - 'x': None - } - - self.d.set('thing', thing) - - t = self.d.get('thing') - self.assertDictEqual(thing, t) - - def test_set_none_deletes(self): - t = 123 - - self.d.set('t', t) - - self.assertEqual(self.d.get('t'), 123) - - self.d.set('t', None) - - self.assertEqual(self.d.get('t'), None) - - def test_delete_sets_to_none(self): - t = 123 - - self.d.set('t', t) - - self.assertEqual(self.d.get('t'), 123) - - self.d.delete('t') - - self.assertEqual(self.d.get('t'), None) - - def test_getitem_works_like_get(self): - thing = { - 'a': 123, - 'b': False, - 'x': None - } - - self.d.set('thing', thing) - - t = self.d['thing'] - self.assertDictEqual(thing, t) - - def test_setitem_works_like_set(self): - thing = { - 'a': 123, - 'b': False, - 'x': None - } - - self.d['thing'] = thing - - t = self.d['thing'] - self.assertDictEqual(thing, t) - - def test_delitem_works_like_del(self): - t = 123 - - self.d.set('t', t) - - self.assertEqual(self.d.get('t'), 123) - - del self.d['t'] - - self.assertEqual(self.d.get('t'), None) - - def test_iter_with_length_returns_list_of_size_l(self): - prefix_1_keys = [ - 'b77aa343e339bed781c7c2be1267cd597', - 'bc22ede6e6fb4046d78bf2f9d1f8afdb6', - 'b93dbb37d993846d70b8a92779cbfbfe9', - 'be1a2783019de6ea7ef169cc55e48a3ae', - 'b1fe8db32b9185d628f4c346f0455023e', - 'bef918f83b6a0d1e4f980013342807cf8', - 'b004cb9235acb5f689d20904692bc026e', - 'b869ff9519d67354816af90867f4a5425', - 'bcd8e9100dcb601f65e849c147e3e972e', - 'b0111919a698f9816862b4ae662a6ed06', - ] - - prefix_2_keys = [ - 'x37fbab0bd2e60563c79469e5be41e515', - 'x30c6eb2ad176773b5ce6d590d2472dfe', - 'x3d4fc9480f0a07b28aa7646d5066b54d', - 'x387c3d4ab7f0c1c6ef549198fc14b525', - 'x5c74dc83e132e435e8512599e1075bc0', - 'x1472425d0d9bb5ff511e132896d54b13', - 'x2cedb5c52163c22a0b5f179001959dd2', - 'x6223f65e553280cd25cadeac6657555c', - 'xeae18af37c223dde92a71fef55e64afe', - 'xb8810784ffb360cd3ffc57b1d088e537', - ] - - keys = prefix_1_keys + prefix_2_keys - random.shuffle(keys) - - for k in keys: - self.d.set(k, k) - - p1 = [] - - for k in self.d.iter(prefix='b', length=3): - p1.append(k) - - prefix_1_keys.sort() - p1.sort() - - self.assertListEqual(prefix_1_keys[:3], p1) - - p2 = [] - - for k in self.d.iter(prefix='x', length=5): - p2.append(k) - - prefix_2_keys.sort() - p2.sort() - - self.assertListEqual(prefix_2_keys[:5], p2) - - def test_key_error_if_getitem_doesnt_exist(self): - with self.assertRaises(KeyError): - print(self.d['thing']) - - def test_keys_returns_all_keys(self): - prefix_1_keys = [ - 'b77aa343e339bed781c7c2be1267cd597', - 'bc22ede6e6fb4046d78bf2f9d1f8afdb6', - 'b93dbb37d993846d70b8a92779cbfbfe9', - 'be1a2783019de6ea7ef169cc55e48a3ae', - 'b1fe8db32b9185d628f4c346f0455023e', - 'bef918f83b6a0d1e4f980013342807cf8', - 'b004cb9235acb5f689d20904692bc026e', - 'b869ff9519d67354816af90867f4a5425', - 'bcd8e9100dcb601f65e849c147e3e972e', - 'b0111919a698f9816862b4ae662a6ed06', - ] - - prefix_2_keys = [ - 'x37fbab0bd2e60563c79469e5be41e515', - 'x30c6eb2ad176773b5ce6d590d2472dfe', - 'x3d4fc9480f0a07b28aa7646d5066b54d', - 'x387c3d4ab7f0c1c6ef549198fc14b525', - 'x5c74dc83e132e435e8512599e1075bc0', - 'x1472425d0d9bb5ff511e132896d54b13', - 'x2cedb5c52163c22a0b5f179001959dd2', - 'x6223f65e553280cd25cadeac6657555c', - 'xeae18af37c223dde92a71fef55e64afe', - 'xb8810784ffb360cd3ffc57b1d088e537', - ] - - keys = prefix_1_keys + prefix_2_keys - random.shuffle(keys) - - for k in keys: - self.d.set(k, k) - - keys.sort() - - got_keys = self.d.keys() - - self.assertListEqual(keys, got_keys) - - -class TestLMDBDriver(TestCase): - # Flush this sucker every test - def setUp(self): - self.d = LMDBDriver() - self.d.flush() - - def tearDown(self): - self.d.flush() - - def test_get_set(self): + def test_get_set_with_block_num(self): for v in TEST_DATA: - self.d.set('b', v) + self.d.set('b.b', v, block_num=self.block_num) - b = self.d.get('b') + b = self.d.get('b.b') self.assertEqual(v, b) if not isinstance(v, dict) else self.assertDictEqual(v, b) + self.assertEqual(self.d.get_block('b.b'), self.block_num) def test_delete(self): for v in TEST_DATA: - self.d.set('b', v) + self.d.set('b.b', v) - b = self.d.get('b') + b = self.d.get('b.b') self.assertEqual(v, b) if not isinstance(v, dict) else self.assertDictEqual(v, b) + self.assertEqual(self.d.get_block('b.b'), config.BLOCK_NUM_DEFAULT) - self.d.delete('b') + self.d.delete('b.b') - b = self.d.get('b') + b = self.d.get('b.b') self.assertIsNone(b) + self.assertEqual(self.d.get_block('b.b'), config.BLOCK_NUM_DEFAULT) - def test_iter(self): + def test_keys_with_prefix(self): prefix_1_keys = [ - 'b77aa343e339bed781c7c2be1267cd597', - 'bc22ede6e6fb4046d78bf2f9d1f8afdb6', - 'b93dbb37d993846d70b8a92779cbfbfe9', - 'be1a2783019de6ea7ef169cc55e48a3ae', - 'b1fe8db32b9185d628f4c346f0455023e', - 'bef918f83b6a0d1e4f980013342807cf8', - 'b004cb9235acb5f689d20904692bc026e', - 'b869ff9519d67354816af90867f4a5425', - 'bcd8e9100dcb601f65e849c147e3e972e', - 'b0111919a698f9816862b4ae662a6ed06', + 'b77aa343e339bed78.1c7c2be1267cd597', + 'bc22ede6e6fb4046d.78bf2f9d1f8afdb6', + 'b93dbb37d993846d7.0b8a92779cbfbfe9', + 'be1a2783019de6ea7.ef169cc55e48a3ae', + 'b1fe8db32b9185d62.8f4c346f0455023e', + 'bef918f83b6a0d1e4.f980013342807cf8', + 'b004cb9235acb5f68.9d20904692bc026e', + 'b869ff9519d673548.16af90867f4a5425', + 'bcd8e9100dcb601f6.5e849c147e3e972e', + 'b0111919a698f9816.862b4ae662a6ed06', ] prefix_2_keys = [ - 'x37fbab0bd2e60563c79469e5be41e515', - 'x30c6eb2ad176773b5ce6d590d2472dfe', - 'x3d4fc9480f0a07b28aa7646d5066b54d', - 'x387c3d4ab7f0c1c6ef549198fc14b525', - 'x5c74dc83e132e435e8512599e1075bc0', - 'x1472425d0d9bb5ff511e132896d54b13', - 'x2cedb5c52163c22a0b5f179001959dd2', - 'x6223f65e553280cd25cadeac6657555c', - 'xeae18af37c223dde92a71fef55e64afe', - 'xb8810784ffb360cd3ffc57b1d088e537', + 'x37fbab0bd2e60563.c79469e5be41e515', + 'x37fbab0bd2e60563.c79469e5be41e515:something', + 'x30c6eb2ad176773b.5ce6d590d2472dfe', + 'x3d4fc9480f0a07b2.8aa7646d5066b54d', + 'x387c3d4ab7f0c1c6.ef549198fc14b525', + 'x5c74dc83e132e435.e8512599e1075bc0', + 'x1472425d0d9bb5ff.511e132896d54b13', + 'x2cedb5c52163c22a.0b5f179001959dd2', + 'x6223f65e553280cd.25cadeac6657555c', + 'xeae18af37c223dde.92a71fef55e64afe', + 'xb8810784ffb360cd.3ffc57b1d088e537', ] keys = prefix_1_keys + prefix_2_keys @@ -863,32 +628,32 @@ def test_set_object_returns_properly(self): 'x': None } - self.d.set('thing', thing) + self.d.set('thing.thing', thing) - t = self.d.get('thing') + t = self.d.get('thing.thing') self.assertDictEqual(thing, t) def test_set_none_deletes(self): t = 123 - self.d.set('t', t) + self.d.set('t.t', t) - self.assertEqual(self.d.get('t'), 123) + self.assertEqual(self.d.get('t.t'), 123) - self.d.set('t', None) + self.d.set('t.t', None) - self.assertEqual(self.d.get('t'), None) + self.assertEqual(self.d.get('t.t'), None) def test_delete_sets_to_none(self): t = 123 - self.d.set('t', t) + self.d.set('t.t', t) - self.assertEqual(self.d.get('t'), 123) + self.assertEqual(self.d.get('t.t'), 123) - self.d.delete('t') + self.d.delete('t.t') - self.assertEqual(self.d.get('t'), None) + self.assertEqual(self.d.get('t.t'), None) def test_getitem_works_like_get(self): thing = { @@ -897,9 +662,9 @@ def test_getitem_works_like_get(self): 'x': None } - self.d.set('thing', thing) + self.d.set('thing.thing', thing) - t = self.d['thing'] + t = self.d['thing.thing'] self.assertDictEqual(thing, t) def test_setitem_works_like_set(self): @@ -909,47 +674,47 @@ def test_setitem_works_like_set(self): 'x': None } - self.d['thing'] = thing + self.d['thing.thing'] = thing - t = self.d['thing'] + t = self.d['thing.thing'] self.assertDictEqual(thing, t) def test_delitem_works_like_del(self): t = 123 - self.d.set('t', t) + self.d.set('t.t', t) - self.assertEqual(self.d.get('t'), 123) + self.assertEqual(self.d.get('t.t'), 123) - del self.d['t'] + del self.d['t.t'] - self.assertEqual(self.d.get('t'), None) + self.assertEqual(self.d.get('t.t'), None) - def test_iter_with_length_returns_list_of_size_l(self): + def test_keys_with_length_returns_list_of_size_l(self): prefix_1_keys = [ - 'b77aa343e339bed781c7c2be1267cd597', - 'bc22ede6e6fb4046d78bf2f9d1f8afdb6', - 'b93dbb37d993846d70b8a92779cbfbfe9', - 'be1a2783019de6ea7ef169cc55e48a3ae', - 'b1fe8db32b9185d628f4c346f0455023e', - 'bef918f83b6a0d1e4f980013342807cf8', - 'b004cb9235acb5f689d20904692bc026e', - 'b869ff9519d67354816af90867f4a5425', - 'bcd8e9100dcb601f65e849c147e3e972e', - 'b0111919a698f9816862b4ae662a6ed06', + 'b77aa343e339bed7.81c7c2be1267cd597', + 'bc22ede6e6fb4046.d78bf2f9d1f8afdb6', + 'b93dbb37d993846d.70b8a92779cbfbfe9', + 'be1a2783019de6ea.7ef169cc55e48a3ae', + 'b1fe8db32b9185d6.28f4c346f0455023e', + 'bef918f83b6a0d1e.4f980013342807cf8', + 'b004cb9235acb5f6.89d20904692bc026e', + 'b869ff9519d67354.816af90867f4a5425', + 'bcd8e9100dcb601f.65e849c147e3e972e', + 'b0111919a698f981.6862b4ae662a6ed06', ] prefix_2_keys = [ - 'x37fbab0bd2e60563c79469e5be41e515', - 'x30c6eb2ad176773b5ce6d590d2472dfe', - 'x3d4fc9480f0a07b28aa7646d5066b54d', - 'x387c3d4ab7f0c1c6ef549198fc14b525', - 'x5c74dc83e132e435e8512599e1075bc0', - 'x1472425d0d9bb5ff511e132896d54b13', - 'x2cedb5c52163c22a0b5f179001959dd2', - 'x6223f65e553280cd25cadeac6657555c', - 'xeae18af37c223dde92a71fef55e64afe', - 'xb8810784ffb360cd3ffc57b1d088e537', + 'x37fbab0bd2e6056.3c79469e5be41e515', + 'x30c6eb2ad176773.b5ce6d590d2472dfe', + 'x3d4fc9480f0a07b.28aa7646d5066b54d', + 'x387c3d4ab7f0c1c.6ef549198fc14b525', + 'x5c74dc83e132e43.5e8512599e1075bc0', + 'x1472425d0d9bb5f.f511e132896d54b13', + 'x2cedb5c52163c22.a0b5f179001959dd2', + 'x6223f65e553280c.d25cadeac6657555c', + 'xeae18af37c223dd.e92a71fef55e64afe', + 'xb8810784ffb360c.d3ffc57b1d088e537', ] keys = prefix_1_keys + prefix_2_keys @@ -978,35 +743,34 @@ def test_iter_with_length_returns_list_of_size_l(self): self.assertListEqual(prefix_2_keys[:5], p2) - def test_key_error_if_getitem_doesnt_exist(self): - with self.assertRaises(KeyError): - print(self.d['thing']) + def test_key_none_if_getitem_doesnt_exist(self): + self.assertIsNone(self.d['thing.thing']) def test_keys_returns_all_keys(self): prefix_1_keys = [ - 'b77aa343e339bed781c7c2be1267cd597', - 'bc22ede6e6fb4046d78bf2f9d1f8afdb6', - 'b93dbb37d993846d70b8a92779cbfbfe9', - 'be1a2783019de6ea7ef169cc55e48a3ae', - 'b1fe8db32b9185d628f4c346f0455023e', - 'bef918f83b6a0d1e4f980013342807cf8', - 'b004cb9235acb5f689d20904692bc026e', - 'b869ff9519d67354816af90867f4a5425', - 'bcd8e9100dcb601f65e849c147e3e972e', - 'b0111919a698f9816862b4ae662a6ed06', + 'b77aa343e339bed7.81c7c2be1267cd597', + 'bc22ede6e6fb4046.d78bf2f9d1f8afdb6', + 'b93dbb37d993846d.70b8a92779cbfbfe9', + 'be1a2783019de6ea.7ef169cc55e48a3ae', + 'b1fe8db32b9185d6.28f4c346f0455023e', + 'bef918f83b6a0d1e.4f980013342807cf8', + 'b004cb9235acb5f6.89d20904692bc026e', + 'b869ff9519d67354.816af90867f4a5425', + 'bcd8e9100dcb601f.65e849c147e3e972e', + 'b0111919a698f981.6862b4ae662a6ed06', ] prefix_2_keys = [ - 'x37fbab0bd2e60563c79469e5be41e515', - 'x30c6eb2ad176773b5ce6d590d2472dfe', - 'x3d4fc9480f0a07b28aa7646d5066b54d', - 'x387c3d4ab7f0c1c6ef549198fc14b525', - 'x5c74dc83e132e435e8512599e1075bc0', - 'x1472425d0d9bb5ff511e132896d54b13', - 'x2cedb5c52163c22a0b5f179001959dd2', - 'x6223f65e553280cd25cadeac6657555c', - 'xeae18af37c223dde92a71fef55e64afe', - 'xb8810784ffb360cd3ffc57b1d088e537', + 'x37fbab0bd2e6056.3c79469e5be41e515', + 'x30c6eb2ad176773.b5ce6d590d2472dfe', + 'x3d4fc9480f0a07b.28aa7646d5066b54d', + 'x387c3d4ab7f0c1c.6ef549198fc14b525', + 'x5c74dc83e132e43.5e8512599e1075bc0', + 'x1472425d0d9bb5f.f511e132896d54b13', + 'x2cedb5c52163c22.a0b5f179001959dd2', + 'x6223f65e553280c.d25cadeac6657555c', + 'xeae18af37c223dd.e92a71fef55e64afe', + 'xb8810784ffb360c.d3ffc57b1d088e537', ] keys = prefix_1_keys + prefix_2_keys