diff --git a/.gitignore b/.gitignore index 57e683c..5e31945 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc /build +__pycache__ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f59b9fe --- /dev/null +++ b/Makefile @@ -0,0 +1,6 @@ +clean: + find . -name '*~' | xargs rm -f + find . -name '*.pyc' | xargs rm -f + rm -rf build/ + rm -rf tests/__pycache__ + rm -rf kyototycoon/__pycache__ diff --git a/README b/README deleted file mode 100644 index 68fb5c7..0000000 --- a/README +++ /dev/null @@ -1,35 +0,0 @@ -ABOUT ------ -Python client library for Kyoto Tycoon. For more information on -Kyoto Tycoon please refer to the official project website. - - http://fallabs.com/kyototycoon/ - -python-kyototycoon's interface follows the preferred interface -provided by Kyoto Tycoon's original author(s). - - http://fallabs.com/kyototycoon/kyototycoon.idl - -There is currently no documentation for this software but for -the meantime please see kyototycoon.py for the interface. - -This fork is backwards compatible, and includes cursor support -and significant performance improvements over the original -library. - -INSTALLATION ------------- -Using pip:: - - pip install python-kyototycoon - -Or, from source:: - - python setup.py build - sudo python setup.py install - - -AUTHORS -------- -* Toru Maesaka -* Stephen Hamer diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..7dfc38f --- /dev/null +++ b/README.rst @@ -0,0 +1,83 @@ +ABOUT +----- +This is a native Python client library for the Kyoto Tycoon server, +supporting both Python 2 and 3. It includes significant performance +improvements and bug fixes over the (now seemingly unmaintained) +original library by Toru Maesaka and Stephen Hamer. + +For more information on Kyoto Tycoon server, please refer to: + + http://fallabs.com/kyototycoon/ + + +FEATURES +-------- +The more efficient binary protocol is also supported along with +the HTTP protocol. It provides a performance improvement of up +to 6x, but only the following operations are available: + + * ``get()`` and ``get_bulk()`` + * ``set()`` and ``set_bulk()`` + * ``remove()`` and ``remove_bulk()`` + * ``play_script()`` + +Atomic operations aren't supported with the binary protocol, +the use of "atomic=False" is mandatory when using it. Operations +besides these will raise a ``NotImplementedError`` exception. + +It's possible to have two KyotoTycoon objects open to the same +server in the same application, one using HTTP and the other +using the binary protocol, if necessary. + +The library does automatic packing and unpacking (marshalling) +of values coming from/to the database. The following data +storage formats are available by default: + + * ``KT_PACKER_PICKLE`` - Python "pickle" format. + * ``KT_PACKER_JSON`` - JSON format (compact representation). + * ``KT_PACKER_STRING`` - Strings (UTF-8). + * ``KT_PACKER_BYTES`` - Binary data. + +There is also a ``KT_PACKER_CUSTOM`` format available where you +can specify your own object to do the marshalling. This object +needs to provide the following two methods: + + * ``.pack(self, data)`` - convert "data" to ``bytes()`` + * ``.unpack(self, data)`` - convert "data" from ``bytes()`` + +Marshalling is done for all methods except ``play_script()``, +because the server can return data in more than one format at +once. The caller will most likely know the type of data that +the called script returns and must do the marshalling itself. + + +COMPATIBILITY +------------- +This library is still not at version 1.0, which means the API and +behavior are not guaranteed to remain consistent between versions. + +Support for using an error object has been removed. If you need +the old behavior for compatibility reasons, use a version up to +(and including) v0.5.9. Versions later than this raise exceptions +on all Kyoto Tycoon errors. + + + +INSTALLATION +------------ +You can install the latest version of this library from source:: + + python setup.py build + python setup.py install + +AUTHORS +------- + * Toru Maesaka + * Stephen Hamer + * Carlos Rodrigues + +Binary protocol support was added based on Ulrich Mierendorff's +code with only minimal changes to make it fit this library. +You can find the original library at the following URL: + + http://www.ulrichmierendorff.com/software/kyoto_tycoon/python_library.html diff --git a/kyototycoon/__init__.py b/kyototycoon/__init__.py index 85b1549..f18f936 100644 --- a/kyototycoon/__init__.py +++ b/kyototycoon/__init__.py @@ -1 +1,13 @@ -from kyototycoon import * +# -*- coding: utf-8 -*- + +from .kyototycoon import KyotoTycoon + +from .kt_common import KT_PACKER_CUSTOM, \ + KT_PACKER_PICKLE, \ + KT_PACKER_JSON, \ + KT_PACKER_STRING, \ + KT_PACKER_BYTES + +from .kt_error import KyotoTycoonException + +# EOF - __init__.py diff --git a/kyototycoon/kt_binary.py b/kyototycoon/kt_binary.py new file mode 100644 index 0000000..ed65563 --- /dev/null +++ b/kyototycoon/kt_binary.py @@ -0,0 +1,264 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013, Carlos Rodrigues +# +# Redistribution and use of this source code is licensed under +# the BSD license. See COPYING file for license description. +# +# This is based on Ulrich Mierendorff's code, originally at: +# - http://www.ulrichmierendorff.com/software/kyoto_tycoon/python_library.html +# + +import socket +import struct + +from .kt_error import KyotoTycoonException + +from .kt_common import KT_PACKER_CUSTOM, \ + KT_PACKER_PICKLE, \ + KT_PACKER_JSON, \ + KT_PACKER_STRING, \ + KT_PACKER_BYTES + +try: + import cPickle as pickle +except ImportError: + import pickle + +import json + +MB_SET_BULK = 0xb8 +MB_GET_BULK = 0xba +MB_REMOVE_BULK = 0xb9 +MB_PLAY_SCRIPT = 0xb4 + +# Maximum signed 64bit integer... +DEFAULT_EXPIRE = 0x7fffffffffffffff + +class ProtocolHandler(object): + def __init__(self, pack_type=KT_PACKER_PICKLE, custom_packer=None): + self.socket = None + + if pack_type != KT_PACKER_CUSTOM and custom_packer is not None: + raise KyotoTycoonException('custom packer object supported for "KT_PACKER_CUSTOM" only') + + if pack_type == KT_PACKER_PICKLE: + # Pickle protocol v2 is is used here instead of the default... + self.pack = lambda data: pickle.dumps(data, 2) + self.unpack = lambda data: pickle.loads(data) + + elif pack_type == KT_PACKER_JSON: + self.pack = lambda data: json.dumps(data, separators=(',',':')).encode('utf-8') + self.unpack = lambda data: json.loads(data.decode('utf-8')) + + elif pack_type == KT_PACKER_STRING: + self.pack = lambda data: data.encode('utf-8') + self.unpack = lambda data: data.decode('utf-8') + + elif pack_type == KT_PACKER_BYTES: + self.pack = lambda data: data + self.unpack = lambda data: data + + elif pack_type == KT_PACKER_CUSTOM: + if custom_packer is None: + raise KyotoTycoonException('"KT_PACKER_CUSTOM" requires a packer object') + + self.pack = custom_packer.pack + self.unpack = custom_packer.unpack + + else: + raise KyotoTycoonException('unsupported pack type specified') + + def cursor(self): + raise NotImplementedError('supported under the HTTP procotol only') + + def open(self, host, port, timeout): + self.socket = socket.create_connection((host, port), timeout) + return True + + def close(self): + self.socket.shutdown(socket.SHUT_RDWR) + self.socket.close() + return True + + def get(self, key, db=0): + values = self.get_bulk([key], False, db) + return values[key] if values else None + + def set_bulk(self, kv_dict, expire, atomic, db=0): + if atomic: + raise KyotoTycoonException('atomic supported under the HTTP procotol only') + + if isinstance(kv_dict, dict) and len(kv_dict) < 1: + return 0 # ...done + + if expire is None: + expire = DEFAULT_EXPIRE + + request = [struct.pack('!BII', MB_SET_BULK, 0, len(kv_dict))] + + for key, value in kv_dict.items(): + key = key.encode('utf-8') + value = self.pack(value) + request.extend([struct.pack('!HIIq', db, len(key), len(value), expire), key, value]) + + self._write(b''.join(request)) + + magic, = struct.unpack('!B', self._read(1)) + if magic != MB_SET_BULK: + raise KyotoTycoonException('bad response [%s]' % hex(magic)) + + # Number of items set... + return struct.unpack('!I', self._read(4))[0] + + def remove_bulk(self, keys, atomic, db=0): + if atomic: + raise KyotoTycoonException('atomic supported under the HTTP procotol only') + + if len(keys) < 1: + return 0 # ...done + + request = [struct.pack('!BII', MB_REMOVE_BULK, 0, len(keys))] + + for key in keys: + key = key.encode('utf-8') + request.extend([struct.pack('!HI', db, len(key)), key]) + + self._write(b''.join(request)) + + magic, = struct.unpack('!B', self._read(1)) + if magic != MB_REMOVE_BULK: + raise KyotoTycoonException('bad response [%s]' % hex(magic)) + + # Number of items removed... + return struct.unpack('!I', self._read(4))[0] + + def get_bulk(self, keys, atomic, db=0): + if atomic: + raise KyotoTycoonException('atomic supported under the HTTP procotol only') + + if len(keys) < 1: + return {} # ...done + + request = [struct.pack('!BII', MB_GET_BULK, 0, len(keys))] + + for key in keys: + key = key.encode('utf-8') + request.extend([struct.pack('!HI', db, len(key)), key]) + + self._write(b''.join(request)) + + magic, = struct.unpack('!B', self._read(1)) + if magic != MB_GET_BULK: + raise KyotoTycoonException('bad response [%s]' % hex(magic)) + + num_items, = struct.unpack('!I', self._read(4)) + items = {} + for i in range(num_items): + key_db, key_length, value_length, key_expire = struct.unpack('!HIIq', self._read(18)) + key = self._read(key_length) + value = self._read(value_length) + items[key.decode('utf-8')] = self.unpack(value) + + return items + + def get_int(self, key, db=0): + raise NotImplementedError('supported under the HTTP procotol only') + + def vacuum(self, db=0): + raise NotImplementedError('supported under the HTTP procotol only') + + def match_prefix(self, prefix, limit, db=0): + raise NotImplementedError('supported under the HTTP procotol only') + + def match_regex(self, regex, limit, db=0): + raise NotImplementedError('supported under the HTTP procotol only') + + def set(self, key, value, expire, db=0): + numitems = self.set_bulk({key: value}, expire, False, db) + return numitems > 0 + + def add(self, key, value, expire, db=0): + raise NotImplementedError('supported under the HTTP procotol only') + + def cas(self, key, old_val, new_val, expire, db=0): + raise NotImplementedError('supported under the HTTP procotol only') + + def remove(self, key, db=0): + numitems = self.remove_bulk([key], False, db) + return numitems > 0 + + def replace(self, key, value, expire, db=0): + raise NotImplementedError('supported under the HTTP procotol only') + + def append(self, key, value, expire, db=0): + raise NotImplementedError('supported under the HTTP procotol only') + + def increment(self, key, delta, expire, db=0): + raise NotImplementedError('supported under the HTTP procotol only') + + def increment_double(self, key, delta, expire, db=0): + raise NotImplementedError('supported under the HTTP procotol only') + + def report(self): + raise NotImplementedError('supported under the HTTP procotol only') + + def status(self, db=0): + raise NotImplementedError('supported under the HTTP procotol only') + + def clear(self, db=0): + raise NotImplementedError('supported under the HTTP procotol only') + + def count(self, db=0): + raise NotImplementedError('supported under the HTTP procotol only') + + def size(self, db=0): + raise NotImplementedError('supported under the HTTP procotol only') + + def play_script(self, name, kv_dict=None): + if kv_dict is None: + kv_dict = {} + + name = name.encode('utf-8') + request = [struct.pack('!BIII', MB_PLAY_SCRIPT, 0, len(name), len(kv_dict)), name] + + for key, value in kv_dict.items(): + if not isinstance(value, bytes): + raise ValueError('value must be a byte sequence') + + key = key.encode('utf-8') + request.extend([struct.pack('!II', len(key), len(value)), key, value]) + + self._write(b''.join(request)) + + magic, = struct.unpack('!B', self._read(1)) + if magic != MB_PLAY_SCRIPT: + raise KyotoTycoonException('bad response [%s]' % hex(magic)) + + num_items, = struct.unpack('!I', self._read(4)) + items = {} + for i in range(num_items): + key_length, value_length = struct.unpack('!II', self._read(8)) + key = self._read(key_length) + value = self._read(value_length) + items[key.decode('utf-8')] = value + + return items + + def _write(self, data): + self.socket.sendall(data) + + def _read(self, bytecnt): + buf = [] + read = 0 + while read < bytecnt: + recv = self.socket.recv(bytecnt - read) + if not recv: + raise IOError('no data while reading') + + buf.append(recv) + read += len(recv) + + return b''.join(buf) + +# EOF - kt_binary.py diff --git a/kyototycoon/kt_common.py b/kyototycoon/kt_common.py new file mode 100644 index 0000000..108fe06 --- /dev/null +++ b/kyototycoon/kt_common.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2011, Toru Maesaka +# +# Redistribution and use of this source code is licensed under +# the BSD license. See COPYING file for license description. + +KT_PACKER_CUSTOM = 0 +KT_PACKER_PICKLE = 1 +KT_PACKER_JSON = 2 +KT_PACKER_STRING = 3 +KT_PACKER_BYTES = 4 + +# EOF - kt_common.py diff --git a/kyototycoon/kt_error.py b/kyototycoon/kt_error.py index b9d102e..51c373f 100644 --- a/kyototycoon/kt_error.py +++ b/kyototycoon/kt_error.py @@ -1,60 +1,11 @@ -#!/usr/bin/env python +# -*- coding: utf-8 -*- # -# Copyright 2011, Toru Maesaka +# Copyright 2014, Carlos Rodrigues # # Redistribution and use of this source code is licensed under # the BSD license. See COPYING file for license description. -class KyotoTycoonError(object): - SUCCESS = 0 - NOIMPL = 1 - INVALID = 2 - LOGIC = 3 - INTERNAL = 4 - NETWORK = 5 - NOTFOUND = 6 - EMISC = 255 +class KyotoTycoonException(Exception): + pass - ErrorNameDict = { - SUCCESS: "SUCCESS", - NOIMPL: "UNIMPLEMENTED", - INVALID: "INVALID", - LOGIC: "LOGIC", - INTERNAL: "INTERNAL", - NETWORK: "NETWORK", - NOTFOUND: "NOTFOUND", - EMISC: "EMISC", - } - - ErrorMessageDict = { - SUCCESS: "Operation Successful", - NOIMPL: "Unimplemented Operation", - INVALID: "Invalid Operation", - LOGIC: "Logic Error", - INTERNAL: "Internal Error", - NETWORK: "Network Error", - NOTFOUND: "Record Not Found", - EMISC: "Miscellenious Error", - } - - def __init__(self): - self.set_success() - - def set_success(self): - self.error_code = self.SUCCESS - self.error_name = self.ErrorNameDict[self.SUCCESS] - self.error_message = self.ErrorMessageDict[self.SUCCESS] - - def set_error(self, code): - self.error_code = code - self.error_name = self.ErrorNameDict[code] - self.error_message = self.ErrorMessageDict[code] - - def code(self): - return self.error_code - - def name(self): - return self.error_name - - def message(self): - return self.error_message +# EOF - kt_error.py diff --git a/kyototycoon/kt_http.py b/kyototycoon/kt_http.py index 58cc8cb..c925b38 100644 --- a/kyototycoon/kt_http.py +++ b/kyototycoon/kt_http.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +# -*- coding: utf-8 -*- # # Copyright 2011, Toru Maesaka # @@ -6,64 +6,80 @@ # the BSD license. See COPYING file for license description. import base64 -import httplib import struct import time -import kt_error +import sys + +from .kt_error import KyotoTycoonException + +from .kt_common import KT_PACKER_CUSTOM, \ + KT_PACKER_PICKLE, \ + KT_PACKER_JSON, \ + KT_PACKER_STRING, \ + KT_PACKER_BYTES + try: - from percentcoding import quote, unquote + import httplib except ImportError: + import http.client as httplib + +try: from urllib import quote as _quote - from urllib import unquote - quote = lambda s: _quote(s, safe="") + from urllib import quote as _quote_from_bytes + from urllib import unquote as unquote_to_bytes +except ImportError: + from urllib.parse import quote as _quote + from urllib.parse import quote_from_bytes as _quote_from_bytes + from urllib.parse import unquote_to_bytes + +quote = lambda s: _quote(s, safe='') +quote_from_bytes = lambda s: _quote_from_bytes(s, safe='') try: import cPickle as pickle except ImportError: import pickle -# Stick with URL encoding for now. Eventually run a benchmark -# to evaluate what the most approariate encoding algorithm is. -KT_HTTP_HEADER = { - 'Content-Type' : 'text/tab-separated-values; colenc=U', -} +import json + +KT_HTTP_HEADER = {'Content-Type' : 'text/tab-separated-values; colenc=U'} -KT_PACKER_CUSTOM = 0 -KT_PACKER_PICKLE = 1 -KT_PACKER_JSON = 2 -KT_PACKER_STRING = 3 +def _dict_to_tsv(kv_dict): + lines = [] + for k, v in kv_dict.items(): + quoted = quote_from_bytes(v) if isinstance(v, bytes) else quote(str(v)) + lines.append('%s\t%s' % (quote(k.encode('utf-8')), quoted)) + return '\n'.join(lines) -def _dict_to_tsv(dict): - return '\n'.join(quote(k) + '\t' + quote(str(v)) for (k, v) in dict.items()) +def _content_type_decoder(content_type): + '''Select the appropriate decoding function to use based on the response headers.''' -def _content_type_decoder(content_type=''): - ''' Select the appropriate decoding function to use based on the response headers. ''' if content_type.endswith('colenc=B'): return base64.decodestring - elif content_type.endswith('colenc=U'): - return unquote - else: - return lambda x: x -def _tsv_to_dict(tsv_str, content_type=''): + if content_type.endswith('colenc=U'): + return unquote_to_bytes + + return lambda x: x + +def _tsv_to_dict(tsv_str, content_type): decode = _content_type_decoder(content_type) rv = {} - for row in tsv_str.split('\n'): - kv = row.split('\t') + for row in tsv_str.split(b'\n'): + kv = row.split(b'\t') if len(kv) == 2: rv[decode(kv[0])] = decode(kv[1]) return rv -def _tsv_to_list(tsv_str, content_type=''): +def _tsv_to_list(tsv_str, content_type): decode = _content_type_decoder(content_type) rv = [] - for row in tsv_str.split('\n'): - kv = row.split('\t') + for row in tsv_str.split(b'\n'): + kv = row.split(b'\t') if len(kv) == 2: - pair = (decode(kv[0]), decode(kv[1])) - rv.append(pair) + rv.append([decode(kv[0]), decode(kv[1])]) return rv @@ -75,290 +91,268 @@ def __init__(self, protocol_handler): self.cursor_id = Cursor.cursor_id_counter Cursor.cursor_id_counter += 1 - self.err = kt_error.KyotoTycoonError() self.pack = self.protocol_handler.pack self.unpack = self.protocol_handler.unpack - self.pack_type = self.protocol_handler.pack_type def __enter__(self): return self - def __exit__(self, type, value, tb): - # Cleanup the cursor when leaving "with" blocks + def __exit__(self, type, value, traceback): + # Cleanup the cursor when leaving "with" blocks... self.delete() - def jump(self, key=None, db=None): - path = '/rpc/cur_jump' - if db: - db = quote(db) - path += '?DB=' + db + def jump(self, key=None, db=0): + '''Jump the cursor to a record (first record if "None") for forward scan.''' - request_dict = {} - request_dict['CUR'] = self.cursor_id + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/rpc/cur_jump?DB=' + db + + request_dict = {'CUR': self.cursor_id} if key: - request_dict['key'] = key + request_dict['key'] = key.encode('utf-8') request_body = _dict_to_tsv(request_dict) - self.protocol_handler.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.protocol_handler.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.protocol_handler.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() return True - def jump_back(self, key=None, db=None): - path = '/rpc/cur_jump_back' - if db: - db = quote(db) - path += '?DB=' + db + def jump_back(self, key=None, db=0): + '''Jump the cursor to a record (last record if "None") for forward scan.''' + + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path += '/rpc/cur_jump_back?DB=' + db - request_dict = {} - request_dict['CUR'] = self.cursor_id + request_dict = {'CUR': self.cursor_id} if key: - request_dict['key'] = key + request_dict['key'] = key.encode('utf-8') request_body = _dict_to_tsv(request_dict) - self.protocol_handler.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.protocol_handler.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.protocol_handler.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() return True def step(self): - path = '/rpc/cur_step' - - request_dict = {} - request_dict['CUR'] = self.cursor_id + '''Step the cursor to the next record.''' + path = '/rpc/cur_step' + request_dict = {'CUR': self.cursor_id} request_body = _dict_to_tsv(request_dict) - self.protocol_handler.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.protocol_handler.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.protocol_handler.getresponse() - if res.status != 200: - self.err.set_error(self.err.EMISC) + if res.status == 450: + # Since this is normal while iterating, do not raise an exception... return False - self.err.set_success() + if res.status != 200: + raise KyotoTycoonException('protocol error [%d]' % res.status) + return True def step_back(self): - path = '/rpc/cur_step_back' - - request_dict = {} - request_dict['CUR'] = self.cursor_id + '''Step the cursor to the previous record.''' + path = '/rpc/cur_step_back' + request_dict = {'CUR': self.cursor_id} request_body = _dict_to_tsv(request_dict) - self.protocol_handler.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.protocol_handler.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.protocol_handler.getresponse() - if res.status != 200: - self.err.set_error(self.err.EMISC) + if res.status == 450: + # Since this is normal while iterating, do not raise an exception... return False - self.err.set_success() + if res.status != 200: + raise KyotoTycoonException('protocol error [%d]' % res.status) + return True - def set_value(self, value, step=False, xt=None): + def set_value(self, value, step=False, expire=None): + '''Set the value for the current record.''' + path = '/rpc/cur_set_value' + request_dict = {'CUR': self.cursor_id, 'value': self.pack(value)} - request_dict = {} - request_dict['CUR'] = self.cursor_id - request_dict['value'] = self.pack(value) if step: request_dict['step'] = True - if xt: - request_dict['xt'] = xt + + if expire: + request_dict['xt'] = expire request_body = _dict_to_tsv(request_dict) - self.protocol_handler.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.protocol_handler.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.protocol_handler.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() return True def remove(self): - path = '/rpc/cur_remove' - - request_dict = {} - request_dict['CUR'] = self.cursor_id + '''Remove the current record.''' + path = '/rpc/cur_remove' + request_dict = {'CUR': self.cursor_id} request_body = _dict_to_tsv(request_dict) - self.protocol_handler.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.protocol_handler.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.protocol_handler.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() return True def get_key(self, step=False): + '''Get the key for the current record.''' + path = '/rpc/cur_get_key' + request_dict = {'CUR': self.cursor_id} - request_dict = {} - request_dict['CUR'] = self.cursor_id if step: request_dict['step'] = True request_body = _dict_to_tsv(request_dict) - self.protocol_handler.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.protocol_handler.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.protocol_handler.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() - return _tsv_to_dict(body, res.getheader('Content-Type', ''))['key'] + return _tsv_to_dict(body, res.getheader('Content-Type', ''))[b'key'].decode('utf-8') def get_value(self, step=False): + '''Get the value for the current record.''' + path = '/rpc/cur_get_value' + request_dict = {'CUR': self.cursor_id} - request_dict = {} - request_dict['CUR'] = self.cursor_id if step: request_dict['step'] = True request_body = _dict_to_tsv(request_dict) - self.protocol_handler.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.protocol_handler.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.protocol_handler.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() - return self.unpack(_tsv_to_dict(body, res.getheader('Content-Type', ''))['value']) + return self.unpack(_tsv_to_dict(body, res.getheader('Content-Type', ''))[b'value']) def get(self, step=False): + '''Get a (key,value) pair for the current record.''' + path = '/rpc/cur_get' + request_dict = {'CUR': self.cursor_id} - request_dict = {} - request_dict['CUR'] = self.cursor_id if step: request_dict['step'] = True request_body = _dict_to_tsv(request_dict) - self.protocol_handler.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.protocol_handler.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.protocol_handler.getresponse() if res.status == 404: - self.err.set_error(self.err.NOTFOUND) return None, None if res.status != 200: - self.err.set_error(self.err.EMISC) - return False, False + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() res_dict = _tsv_to_dict(body, res.getheader('Content-Type', '')) - key = res_dict['key'] - value = self.unpack(res_dict['value']) + key = res_dict[b'key'].decode('utf-8') + value = self.unpack(res_dict[b'value']) + return key, value def seize(self): - path = '/rpc/cur_seize' - - request_dict = {} - request_dict['CUR'] = self.cursor_id + '''Get a (key,value) pair for the current record, and remove it atomically.''' + path = '/rpc/cur_seize' + request_dict = {'CUR': self.cursor_id} request_body = _dict_to_tsv(request_dict) - self.protocol_handler.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.protocol_handler.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.protocol_handler.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() res_dict = _tsv_to_dict(body, res.getheader('Content-Type', '')) - res_dict['key'] = res_dict['key'] - res_dict['value'] = self.unpack(res_dict['value']) - return res_dict + seize_dict = {'key': res_dict[b'key'].decode('utf-8'), + 'value': self.unpack(res_dict[b'value'])} - def delete(self): - path = '/rpc/cur_delete' + return seize_dict - request_dict = {} - request_dict['CUR'] = self.cursor_id + def delete(self): + '''Delete the cursor.''' + path = '/rpc/cur_delete' + request_dict = {'CUR': self.cursor_id} request_body = _dict_to_tsv(request_dict) - self.protocol_handler.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.protocol_handler.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.protocol_handler.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() return True - class ProtocolHandler(object): - def __init__(self, pack_type=KT_PACKER_PICKLE, pickle_protocol=2): - self.err = kt_error.KyotoTycoonError() - self.pickle_protocol = pickle_protocol + def __init__(self, pack_type=KT_PACKER_PICKLE, custom_packer=None): self.pack_type = pack_type - if self.pack_type == KT_PACKER_PICKLE: - self.pack = self._pickle_packer - self.unpack = self._pickle_unpacker + if pack_type != KT_PACKER_CUSTOM and custom_packer is not None: + raise KyotoTycoonException('custom packer object supported for "KT_PACKER_CUSTOM" only') + + if pack_type == KT_PACKER_PICKLE: + # Pickle protocol v2 is is used here instead of the default... + self.pack = lambda data: pickle.dumps(data, 2) + self.unpack = lambda data: pickle.loads(data) + + elif pack_type == KT_PACKER_JSON: + self.pack = lambda data: json.dumps(data, separators=(',',':')).encode('utf-8') + self.unpack = lambda data: json.loads(data.decode('utf-8')) - elif self.pack_type == KT_PACKER_STRING: + elif pack_type == KT_PACKER_STRING: + self.pack = lambda data: data.encode('utf-8') + self.unpack = lambda data: data.decode('utf-8') + + elif pack_type == KT_PACKER_BYTES: self.pack = lambda data: data self.unpack = lambda data: data - else: - raise Exception('unknown pack type specified') + elif pack_type == KT_PACKER_CUSTOM: + if custom_packer is None: + raise KyotoTycoonException('"KT_PACKER_CUSTOM" requires a packer object') - def error(self): - return self.err + self.pack = custom_packer.pack + self.unpack = custom_packer.unpack + + else: + raise KyotoTycoonException('unsupported pack type specified') def cursor(self): return Cursor(self) def open(self, host, port, timeout): - # Save connection parameters so the connection can be re-established - # on "Connection: close" response. + # Save connection parameters so the connection can be + # re-established on a "Connection: close" response... self.host = host self.port = port self.timeout = timeout - try: - self.conn = httplib.HTTPConnection(host, port, timeout=timeout) - except Exception, e: - raise e + self.conn = httplib.HTTPConnection(host, port, timeout=timeout) return True def close(self): - try: - self.conn.close() - except Exception, e: - raise e + self.conn.close() return True def getresponse(self): @@ -376,495 +370,409 @@ def echo(self): res, body = self.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() return True - def get(self, key, db=None): - if key is None: - return False - - path = key - if db: - path = '/%s/%s' % (db, key) - path = quote(path.encode('UTF-8')) + def get(self, key, db=0): + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/%s/%s' % (db, quote(key.encode('utf-8'))) self.conn.request('GET', path) res, body = self.getresponse() if res.status == 404: - self.err.set_error(self.err.NOTFOUND) return None + if res.status != 200: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() return self.unpack(body) - def set_bulk(self, kv_dict, expire, atomic, db): - if not isinstance(kv_dict, dict): - return False - - if len(kv_dict) < 1: - self.err.set_error(self.err.LOGIC) - return False - - path = '/rpc/set_bulk' - if db: - db = quote(db) - path += '?DB=' + db + def set_bulk(self, kv_dict, expire, atomic, db=0): + if isinstance(kv_dict, dict) and len(kv_dict) < 1: + return 0 # ...done - request_body = '' + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/rpc/set_bulk?DB=' + db - if atomic: - request_body = 'atomic\t\n' + request_body = ['atomic\t\n' if atomic else ''] - for k, v in kv_dict.items(): - k = quote(k) - v = quote(self.pack(v)) - request_body += '_' + k + '\t' + v + '\n' + for key, value in kv_dict.items(): + key = quote(key.encode('utf-8')) + value = quote(self.pack(value)) + request_body.append('_%s\t%s\n' % (key, value)) - self.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.conn.request('POST', path, body=''.join(request_body), headers=KT_HTTP_HEADER) res, body = self.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % res.status) + + # Number of items set... + return int(_tsv_to_dict(body, res.getheader('Content-Type', ''))[b'num']) - self.err.set_success() - return int(_tsv_to_dict(body, res.getheader('Content-Type', ''))['num']) + def remove_bulk(self, keys, atomic, db=0): + if len(keys) < 1: + return 0 # ...done - def remove_bulk(self, keys, atomic, db): - if not hasattr(keys, '__iter__'): - self.err.set_error(self.err.LOGIC) - return 0 + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/rpc/remove_bulk?DB=' + db - request_header = '' - if atomic: - request_header = 'atomic\t\n' + request_body = ['atomic\t\n' if atomic else ''] - request_body = '' for key in keys: - request_body += '_' + quote(key) + '\t\n' - if len(request_body) < 1: - self.err.set_error(self.err.LOGIC) - return 0 - - path = '/rpc/remove_bulk' - if db: - db = quote(db) - path += '?DB=' + db - self.conn.request('POST', path, body=request_header + request_body, - headers=KT_HTTP_HEADER) + request_body.append('_%s\t\n' % quote(key.encode('utf-8'))) + + self.conn.request('POST', path, body=''.join(request_body), headers=KT_HTTP_HEADER) res, body = self.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() - return int(_tsv_to_dict(body, res.getheader('Content-Type', ''))['num']) + # Number of items removed... + return int(_tsv_to_dict(body, res.getheader('Content-Type', ''))[b'num']) - def get_bulk(self, keys, atomic, db): - if not hasattr(keys, '__iter__'): - self.err.set_error(self.err.LOGIC) - return None + def get_bulk(self, keys, atomic, db=0): + if len(keys) < 1: + return {} # ...done - request_header = '' - if atomic: - request_header = 'atomic\t\n' + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/rpc/get_bulk?DB=' + db - request_body = '' - for key in keys: - request_body += '_' + quote(key) + '\t\n' + request_body = ['atomic\t\n' if atomic else ''] - if len(request_body) < 1: - self.err.set_error(self.err.LOGIC) - return {} + for key in keys: + request_body.append('_%s\t\n' % quote(key.encode('utf-8'))) - path = '/rpc/get_bulk' - if db: - db = quote(db) - path += '?DB=' + db - self.conn.request('POST', path, body=request_header + request_body, - headers=KT_HTTP_HEADER) + self.conn.request('POST', path, body=''.join(request_body), headers=KT_HTTP_HEADER) res, body = self.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return None + raise KyotoTycoonException('protocol error [%d]' % res.status) rv = {} res_dict = _tsv_to_dict(body, res.getheader('Content-Type', '')) - n = res_dict.pop('num') + n = res_dict.pop(b'num') if n == '0': - self.err.set_error(self.err.NOTFOUND) return {} for k, v in res_dict.items(): if v is not None: - rv[k[1:]] = self.unpack(v) + rv[k.decode('utf-8')[1:]] = self.unpack(v) - self.err.set_success() return rv - def get_int(self, key, db=None): - if key is None: - self.err.set_error(self.err.LOGIC) - return False - - path = key - if db: - path = '/%s/%s' % (db, key) - path = quote(path.encode('UTF-8')) + def get_int(self, key, db=0): + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/%s/%s' % (db, quote(key.encode('utf-8'))) self.conn.request('GET', path) res, body = self.getresponse() if res.status != 200: - self.err.set_error(self.err.NOTFOUND) - return None + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() return struct.unpack('>q', body)[0] - def vacuum(self, db): - path = '/rpc/vacuum' - - if db: - db = quote(db) - path += '?DB=' + db + def vacuum(self, db=0): + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/rpc/vacuum?DB=' + db self.conn.request('GET', path) res, body = self.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() - return res.status == 200 + return True - def match_prefix(self, prefix, max, db): + def match_prefix(self, prefix, limit, db=0): if prefix is None: - self.err.set_error(self.err.LOGIC) - return None + raise ValueError('no key prefix specified') - rv = [] - request_dict = {} - request_dict['prefix'] = prefix + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/rpc/match_prefix?DB=' + db - if max: - request_dict['max'] = max - if db: - request_dict['DB'] = db + request_dict = {'prefix': prefix.encode('utf-8')} + if limit: + request_dict['max'] = limit request_body = _dict_to_tsv(request_dict) - self.conn.request('POST', '/rpc/match_prefix', - body=request_body, headers=KT_HTTP_HEADER) + self.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % res.status) + rv = [] res_list = _tsv_to_list(body, res.getheader('Content-Type', '')) - if len(res_list) == 0 or res_list[-1][0] != 'num': - self.err.set_err(self.err.EMISC) - return False + if len(res_list) == 0 or res_list[-1][0] != b'num': + raise KyotoTycoonException('server returned no data') + num_key, num = res_list.pop() if num == '0': - self.err.set_error(self.err.NOTFOUND) return [] for k, v in res_list: - rv.append(k[1:]) + rv.append(k.decode('utf-8')[1:]) - self.err.set_success() return rv - def match_regex(self, regex, max, db): + def match_regex(self, regex, limit, db=0): if regex is None: - self.err.set_error(self.err.LOGIC) - return None + raise ValueError('no regular expression specified') - path = '/rpc/match_regex' - if db: - path += '?DB=' + db + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/rpc/match_regex?DB=' + db - request_dict = { 'regex': regex } - if max: - request_dict['max'] = max + request_dict = {'regex': regex.encode('utf-8')} + if limit: + request_dict['max'] = limit request_body = _dict_to_tsv(request_dict) - self.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return None + raise KyotoTycoonException('protocol error [%d]' % res.status) rv = [] res_list = _tsv_to_list(body, res.getheader('Content-Type', '')) - if len(res_list) == 0 or res_list[-1][0] != 'num': - self.err.set_err(self.err.EMISC) - return False + if len(res_list) == 0 or res_list[-1][0] != b'num': + raise KyotoTycoonException('server returned no data') + num_key, num = res_list.pop() if num == '0': - self.err.set_error(self.err.NOTFOUND) return [] for k, v in res_list: - rv.append(k[1:]) + rv.append(k.decode('utf-8')[1:]) - self.err.set_success() return rv - def set(self, key, value, expire, db): - if key is None: - self.err.set_error(self.err.LOGIC) - return False + def set(self, key, value, expire, db=0): + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/%s/%s' % (db, quote(key.encode('utf-8'))) - if db: - key = '/%s/%s' % (db, key) - key = quote(key.encode('UTF-8')) value = self.pack(value) - - self.err.set_success() - - status = self._rest_put('set', key, value, expire) + status = self._rest_put('set', path, value, expire) if status != 201: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % status) - self.err.set_success() return True - def add(self, key, value, expire, db): - if key is None: - self.err.set_error(self.err.LOGIC) - return False - - if db: - key = '/%s/%s' % (db, key) + def add(self, key, value, expire, db=0): + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/%s/%s' % (db, quote(key.encode('utf-8'))) - key = quote(key.encode('UTF-8')) value = self.pack(value) - status = self._rest_put('add', key, value, expire) - + status = self._rest_put('add', path, value, expire) if status != 201: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % status) - self.err.set_success() return True - def cas(self, key, old_val, new_val, expire, db): - if key is None: - self.err.set_error(self.err.LOGIC) - return False + def cas(self, key, old_val, new_val, expire, db=0): + if old_val is None and new_val is None: + raise ValueError('old value and/or new value must be specified') - path = '/rpc/cas' - if db: - path += '?DB=' + db + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/rpc/cas?DB=' + db - request_dict = { 'key': key } + request_dict = {'key': key.encode('utf-8')} - if old_val: + if old_val is not None: request_dict['oval'] = self.pack(old_val) - if new_val: + + if new_val is not None: request_dict['nval'] = self.pack(new_val) + if expire: request_dict['xt'] = expire request_body = _dict_to_tsv(request_dict) - self.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() return True - def remove(self, key, db): - if key is None: - self.err.set_error(self.err.LOGIC) - return False - - if db: - key = '/%s/%s' % (db, key) + def remove(self, key, db=0): + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/%s/%s' % (db, quote(key.encode('utf-8'))) - key = quote(key.encode('UTF-8')) - self.conn.request('DELETE', key) + self.conn.request('DELETE', path) res, body = self.getresponse() if res.status != 204: - self.err.set_error(self.err.NOTFOUND) return False - self.err.set_success() return True - def replace(self, key, value, expire, db): - if key is None: - self.err.set_error(self.err.LOGIC) - return False - - if db: - key = '/%s/%s' % (db, key) + def replace(self, key, value, expire, db=0): + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/%s/%s' % (db, quote(key.encode('utf-8'))) - key = quote(key.encode('UTF-8')) value = self.pack(value) - status = self._rest_put('replace', key, value, expire) - + status = self._rest_put('replace', path, value, expire) if status != 201: - self.err.set_error(self.err.NOTFOUND) return False - self.err.set_success() return True - def append(self, key, value, expire, db): - self.err.set_error(self.err.LOGIC) - if key is None: - return False - elif not isinstance(value, str): - return False + def append(self, key, value, expire, db=0): + # Simultaneous support for Python 2/3 makes this cumbersome... + if sys.version_info[0] >= 3: + bytes_type = bytes + unicode_type = str + else: + bytes_type = str + unicode_type = unicode + + if (not isinstance(value, bytes_type) and + not isinstance(value, unicode_type)): + raise ValueError('value is not a string or bytes type') + + old_data = self.get(key) + data = type(value)() if old_data is None else old_data - # Only handle Pickle for now. - if self.pack_type == KT_PACKER_PICKLE: - data = self.get(key) - if data is None: - data = value + if (not isinstance(data, bytes_type) and + not isinstance(data, unicode_type)): + raise KyotoTycoonException('stored value is not a string or bytes type') + + if type(data) != type(value): + if isinstance(data, bytes_type): + value = value.encode('utf-8') else: - data = data + value + value = value.decode('utf-8') - if self.set(key, data, expire, db) is True: - self.err.set_success() - return True + data += value - self.err.set_error(self.err.EMISC) - return False + # This makes the operation atomic... + if self.cas(key, old_data, data, expire, db) is not True: + raise KyotoTycoonException('error while storing modified value') - def increment(self, key, delta, expire, db): - if key is None: - self.err.set_error(self.err.LOGIC) - return False + return True - path = '/rpc/increment' - if db: - path += '?DB=' + db + def increment(self, key, delta, expire, db=0): + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/rpc/increment?DB=' + db - delta = int(delta) request_body = 'key\t%s\nnum\t%d\n' % (key, delta) - self.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return None + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() - return int(_tsv_to_dict(body, res.getheader('Content-Type', ''))['num']) + return int(_tsv_to_dict(body, res.getheader('Content-Type', ''))[b'num']) - def increment_double(self, key, delta, expire, db): + def increment_double(self, key, delta, expire, db=0): if key is None: - self.err.set_error(self.err.LOGIC) - return False + raise ValueError('no key specified') - path = '/rpc/increment_double' - if db: - path += '?DB=' + db + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/rpc/increment_double?DB=' + db - delta = float(delta) request_body = 'key\t%s\nnum\t%f\n' % (key, delta) - self.conn.request('POST', path, body=request_body, - headers=KT_HTTP_HEADER) + self.conn.request('POST', path, body=request_body, headers=KT_HTTP_HEADER) res, body = self.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return None + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() - return float(_tsv_to_dict(body, res.getheader('Content-Type', ''))['num']) + return float(_tsv_to_dict(body, res.getheader('Content-Type', ''))[b'num']) def report(self): self.conn.request('GET', '/rpc/report') res, body = self.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return None + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() - return _tsv_to_dict(body, res.getheader('Content-Type', '')) + res_dict = _tsv_to_dict(body, res.getheader('Content-Type', '')) + report_dict = {} + for k, v in res_dict.items(): + report_dict[k.decode('utf-8')] = v.decode('utf-8') - def status(self, db=None): - url = '/rpc/status' + return report_dict - if db: - db = quote(db) - url += '?DB=' + db + def status(self, db=0): + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/rpc/status?DB=' + db - self.conn.request('GET', url) + self.conn.request('GET', path) res, body = self.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return None + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() - return _tsv_to_dict(body, res.getheader('Content-Type', '')) + res_dict = _tsv_to_dict(body, res.getheader('Content-Type', '')) + status_dict = {} + for k, v in res_dict.items(): + status_dict[k.decode('utf-8')] = v.decode('utf-8') - def clear(self, db=None): - url = '/rpc/clear' + return status_dict - if db: - db = quote(db) - url += '?DB=' + db + def clear(self, db=0): + db = str(db) if isinstance(db, int) else quote(db.encode('utf-8')) + path = '/rpc/clear?DB=' + db - self.conn.request('GET', url) + self.conn.request('GET', path) res, body = self.getresponse() if res.status != 200: - self.err.set_error(self.err.EMISC) - return False + raise KyotoTycoonException('protocol error [%d]' % res.status) - self.err.set_success() return True - def count(self, db=None): + def count(self, db=0): st = self.status(db) - if st is None: - return None - return int(st['count']) + return None if st is None else int(st['count']) - def size(self, db=None): + def size(self, db=0): st = self.status(db) - if st is None: - return None - return int(st['size']) + return None if st is None else int(st['size']) + + def play_script(self, name, kv_dict=None): + if kv_dict is None: + kv_dict = {} + + path = '/rpc/play_script?name=' + quote(name.encode('utf-8')) + + request_body = [] + for k, v in kv_dict.items(): + if not isinstance(v, bytes): + raise ValueError('value must be a byte sequence') + + k = quote(k.encode('utf-8')) + v = quote(v) + request_body.append('_%s\t%s\n' % (k, v)) + + self.conn.request('POST', path, body=''.join(request_body), headers=KT_HTTP_HEADER) + + res, body = self.getresponse() + if res.status != 200: + raise KyotoTycoonException('protocol error [%d]' % res.status) + + rv = {} + res_dict = _tsv_to_dict(body, res.getheader('Content-Type', '')) + + for k, v in res_dict.items(): + if v is not None: + rv[k.decode('utf-8')[1:]] = v + + return rv def _rest_put(self, operation, key, value, expire): - headers = { 'X-Kt-Mode' : operation } - if expire != None: - expire = int(time.time()) + expire; - headers["X-Kt-Xt"] = str(expire) + headers = {'X-Kt-Mode' : operation} + if expire is not None: + headers["X-Kt-Xt"] = str(int(time.time()) + expire) self.conn.request('PUT', key, value, headers) res, body = self.getresponse() return res.status - def _pickle_packer(self, data): - return pickle.dumps(data, self.pickle_protocol) - - def _pickle_unpacker(self, data): - return pickle.loads(data) +# EOF - kt_http.py diff --git a/kyototycoon/kyototycoon.py b/kyototycoon/kyototycoon.py index baf348e..3a52ee3 100644 --- a/kyototycoon/kyototycoon.py +++ b/kyototycoon/kyototycoon.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +# -*- coding: utf-8 -*- # # Copyright 2011, Toru Maesaka # @@ -8,88 +8,182 @@ # Note that python-kyototycoon follows the following interface # standard: http://fallabs.com/kyototycoon/kyototycoon.idl -import kt_http +import warnings -KT_DEFAULT_HOST = '127.0.0.1' -KT_DEFAULT_PORT = 1978 -KT_DEFAULT_TIMEOUT = 30 +from . import kt_http +from . import kt_binary + +from .kt_common import KT_PACKER_PICKLE class KyotoTycoon(object): - def __init__(self, binary=False, *args, **kwargs): - self.core = kt_http.ProtocolHandler(*args, **kwargs) + def __init__(self, binary=False, pack_type=KT_PACKER_PICKLE, + custom_packer=None, exceptions=True): + ''' + Initialize a "Binary Protocol" or "HTTP Protocol" KyotoTycoon object. + + Note: The default packer uses pickle protocol v2, which is the highest + version that's still compatible with both Python 2 and 3. If you + require a different version, specify a custom packer object. + + ''' + + if not exceptions: + # Relying on separate error states is bad form and should be avoided... + raise DeprecationWarning('not raising exceptions on error has been removed') + + if binary: + self.atomic = False # The binary protocol does not support atomic operations. + self.core = kt_binary.ProtocolHandler(pack_type, custom_packer) + else: + self.atomic = True + self.core = kt_http.ProtocolHandler(pack_type, custom_packer) + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + + def open(self, host='127.0.0.1', port=1978, timeout=30): + '''Open a new connection to a KT server.''' + + return True if self.core.open(host, port, timeout) else False + + def connect(self, *args, **kwargs): + ''' + Open a new connection to a KT server. - def error(self): - return self.core.error() + The same as "open()" but returning "self" instead of a boolean, allowing + KyotoTycoon objects to be used as context managers in "with" statements. - def open(self, host=KT_DEFAULT_HOST, port=KT_DEFAULT_PORT, - timeout=KT_DEFAULT_TIMEOUT): - return self.core.open(host, port, timeout) + ''' + + return self if self.open(*args, **kwargs) else None def close(self): + '''Close an open connection to the KT server.''' + return self.core.close() def report(self): + '''Get a server information report.''' + return self.core.report() - def status(self, db=None): + def status(self, db=0): + '''Get status information for the database.''' + return self.core.status(db) - def clear(self, db=None): + def clear(self, db=0): + '''Remove all records in the database.''' + return self.core.clear(db) - def count(self, db=None): + def count(self, db=0): + '''Number of records in the database.''' + return self.core.count(db) - def size(self, db=None): + def size(self, db=0): + '''Current database size (in bytes).''' + return self.core.size(db) - def set(self, key, value, expire=None, db=None): + def set(self, key, value, expire=None, db=0): + '''Set the value for a record.''' + return self.core.set(key, value, expire, db) - def add(self, key, value, expire=None, db=None): + def add(self, key, value, expire=None, db=0): + '''Set the value for a record (does nothing if the record already exists).''' + return self.core.add(key, value, expire, db) - def replace(self, key, value, expire=None, db=None): + def replace(self, key, value, expire=None, db=0): + '''Replace the value of an existing record.''' + return self.core.replace(key, value, expire, db) - def append(self, key, value, expire=None, db=None): + def append(self, key, value, expire=None, db=0): + '''Append "value" to the string value of a record.''' + return self.core.append(key, value, expire, db) - def increment(self, key, delta, expire=None, db=None): + def increment(self, key, delta, expire=None, db=0): + '''Add "delta" to the numeric integer value of a record.''' + return self.core.increment(key, delta, expire, db) - def increment_double(self, key, delta, expire=None, db=None): + def increment_double(self, key, delta, expire=None, db=0): + '''Add "delta" to the numeric double value of a record.''' + return self.core.increment_double(key, delta, expire, db) - def cas(self, key, old_val=None, new_val=None, expire=None, db=None): + def cas(self, key, old_val=None, new_val=None, expire=None, db=0): + '''If the old value of a record is "old_val", replace it with "new_val".''' + return self.core.cas(key, old_val, new_val, expire, db) - def remove(self, key, db=None): + def remove(self, key, db=0): + '''Remove a record.''' + return self.core.remove(key, db) - def get(self, key, db=None): + def get(self, key, db=0): + '''Retrieve the value for a record.''' + return self.core.get(key, db) - def get_int(self, key, db=None): + def get_int(self, key, db=0): + '''Retrieve the numeric integer value for a record.''' + return self.core.get_int(key, db) - def set_bulk(self, kv_dict, expire=None, atomic=True, db=None): - return self.core.set_bulk(kv_dict, expire, atomic, db) + def set_bulk(self, kv_dict, expire=None, atomic=None, db=0): + '''Set the values for several records at once.''' + + return self.core.set_bulk(kv_dict, expire, self.atomic if atomic is None else atomic, db) + + def remove_bulk(self, keys, atomic=None, db=0): + '''Remove several records at once.''' + + return self.core.remove_bulk(keys, self.atomic if atomic is None else atomic, db) - def remove_bulk(self, keys, atomic=True, db=None): - return self.core.remove_bulk(keys, atomic, db) + def get_bulk(self, keys, atomic=None, db=0): + '''Retrieve the values for several records at once.''' - def get_bulk(self, keys, atomic=True, db=None): - return self.core.get_bulk(keys, atomic, db) + return self.core.get_bulk(keys, self.atomic if atomic is None else atomic, db) + + def vacuum(self, db=0): + '''Scan the database and eliminate regions of expired records.''' - def vacuum(self, db=None): return self.core.vacuum(db) - def match_prefix(self, prefix, max=None, db=None): - return self.core.match_prefix(prefix, max, db) + def match_prefix(self, prefix, limit=None, db=0): + '''Get keys matching a prefix string.''' + + return self.core.match_prefix(prefix, limit, db) + + def match_regex(self, regex, limit=None, db=0): + '''Get keys matching a ragular expression string.''' - def match_regex(self, regex, max=None, db=None): - return self.core.match_regex(regex, max, db) + return self.core.match_regex(regex, limit, db) def cursor(self): + '''Obtain a new (uninitialized) record cursor.''' + return self.core.cursor() + + def play_script(self, name, kv_dict=None): + ''' + Call a procedure of the scripting language extension. + + Because the input/output of server-side scripts may use a mix of formats, and unlike all + other methods, no implicit packing/unpacking is done to either input or output values. + + ''' + + return self.core.play_script(name, kv_dict) + +# EOF - kyototycoon.py diff --git a/setup.py b/setup.py index f05b7ee..4e54804 100644 --- a/setup.py +++ b/setup.py @@ -10,15 +10,13 @@ setup( author='Toru Maesaka', author_email='dev@torum.net', - maintainer='Stephen Hamer', - maintainer_email='stephen.hamer@upverter.com', + maintainer='Carlos Rodrigues', + maintainer_email='cefrodrigues@gmail.com', name='python-kyototycoon', description='Kyoto Tycoon Client Library', - version='0.4.8', + version='0.6.0', license='BSD', keywords='Kyoto Tycoon, Kyoto Cabinet', packages=['kyototycoon'], - requires=['percentcoding'], - url='https://github.com/upverter/python-kyototycoon', - zip_safe=False + url='https://github.com/carlosefr/python-kyototycoon', ) diff --git a/tests/kt_performance.py b/tests/kt_performance.py new file mode 100644 index 0000000..c762ed5 --- /dev/null +++ b/tests/kt_performance.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# kt_performance.py - measure performance of the python-kyototycoon library. +# +# Copyright (c) 2014 Carlos Rodrigues +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + + +from __future__ import print_function +from __future__ import division + +#from __future__ import unicode_literals + + +import sys +import os, os.path + +from time import time +from getopt import getopt, GetoptError + +from kyototycoon import KyotoTycoon, KT_PACKER_PICKLE, KT_PACKER_JSON, KT_PACKER_STRING + + +NUM_ITERATIONS = 2000 + + +def print_usage(): + """Output the proper usage syntax for this program.""" + + print("USAGE: %s [-s ]" % os.path.basename(sys.argv[0])) + + +def parse_args(): + """Parse and enforce command-line arguments.""" + + try: + options, _ = getopt(sys.argv[1:], "s:", ["server="]) + except GetoptError as e: + print("error: %s." % e, file=sys.stderr) + print_usage() + sys.exit(1) + + server = { "host": "127.0.0.1", "port": 1978 } + + for option, value in options: + if option in ("-s", "--server"): + fields = value.strip().split(":") + server["host"] = fields[0].strip() + + if len(fields) > 1: + server["port"] = int(fields[1]) + + return (server,) + + +def main(): + server, = parse_args() + + print("Running %d iterations for each parameter..." % NUM_ITERATIONS) + + header = "%-15s | %-16s | %-16s | %-7s | %-14s | %-6s" % \ + ("Binary Protocol", "Unicode Literals", "Packer Type", "Elapsed", + "Iteration Rate", "Passed") + print(header) + print("=" * len(header)) + + # Test for both Python 2.x and 3.x... + unicode_literal = str("") if sys.version_info[0] == 3 else unicode("") + is_unicode = isinstance("", type(unicode_literal)) + + for binary in (True, False): + for packer_type, packer_name in ((KT_PACKER_PICKLE, "KT_PACKER_PICKLE"), + (KT_PACKER_JSON, "KT_PACKER_JSON"), + (KT_PACKER_STRING, "KT_PACKER_STRING")): + kt = KyotoTycoon(binary=binary, pack_type=packer_type) + + with kt.connect(server["host"], server["port"], timeout=2) as db: + start = time() + bad = 0 + + for i in range(NUM_ITERATIONS): + if is_unicode: + key = "key-%d-u-%d-%d-café" % (binary, packer_type, i) + value = "value-%d-u-%d-%d-café" % (binary, packer_type, i) + else: + key = "key-%d-s-%d-%d-cafe" % (binary, packer_type, i) + value = "value-%d-s-%d-%d-cafe" % (binary, packer_type, i) + + db.set(key, value) + output = db.get(key) + + if output != value: + bad += 1 + + db.remove(key) + output = db.get(key) + + if output is not None: + bad += 1 + + duration = time() - start + rate = NUM_ITERATIONS / duration + + print("%-15s | %-16s | %-16s | %5.2f s | %10.2f ips | %-6s" % + (binary, is_unicode, packer_name, duration, rate, + "No" if bad > 0 else "Yes")) + + +if __name__ == "__main__": + main() + + +# EOF - kt_performance.py diff --git a/tests/t_all.py b/tests/t_all.py index 9087630..69bf85a 100644 --- a/tests/t_all.py +++ b/tests/t_all.py @@ -5,9 +5,11 @@ # Redistribution and use of this source code is licensed under # the BSD license. See COPYING file for license description. # +# Kyoto Tycoon should be started like this: +# $ ktserver -ls -scr t_script.lua '%' '%' +# # USAGE: # $ python t_all.py -# $ python t_all.py ExpireTestCase import os import re @@ -23,15 +25,13 @@ def _run_all_tests(): for filename in os.listdir(test_path): match = _TEST_MODULE_PATTERN.search(filename) if match: - case = match.group(1) - if case != 't_expire' and case != 't_multi': - module_names.append(case) + module_names.append(match.group(1)) return loader.loadTestsFromNames(module_names) -def ExpireTestCase(): +def ScriptTestCase(): loader = unittest.TestLoader() - return loader.loadTestsFromName('t_expire') + return loader.loadTestsFromName('t_script') if __name__ == '__main__': unittest.main(defaultTest='_run_all_tests') diff --git a/tests/t_arithmetic.py b/tests/t_arithmetic.py index 26d16e1..0b85c4c 100644 --- a/tests/t_arithmetic.py +++ b/tests/t_arithmetic.py @@ -7,7 +7,7 @@ import config import unittest -from kyototycoon import KyotoTycoon +from kyototycoon import KyotoTycoon, KyotoTycoonException class UnitTest(unittest.TestCase): def setUp(self): @@ -27,8 +27,7 @@ def test_increment(self): # Incrementing against a non numeric value. Must fail. self.assertTrue(self.kt_handle.set(key, 'foo')) - self.assertEqual(self.kt_handle.increment(key, 10), None) - self.assertEqual(self.kt_handle.increment(key, 10), None) + self.assertRaises(KyotoTycoonException, self.kt_handle.increment, key, 10) def test_increment_double(self): self.assertTrue(self.kt_handle.clear()) diff --git a/tests/t_expire.py b/tests/t_expire.py index e27e1ac..95f3c41 100644 --- a/tests/t_expire.py +++ b/tests/t_expire.py @@ -12,31 +12,38 @@ class UnitTest(unittest.TestCase): def setUp(self): - self.kt_handle = KyotoTycoon() - self.kt_handle.open() + self.kt_http_handle = KyotoTycoon(binary=False) + self.kt_http_handle.open() + + self.kt_bin_handle = KyotoTycoon(binary=True) + self.kt_bin_handle.open() + self.LARGE_KEY_LEN = 8000 def test_set_expire(self): - self.assertTrue(self.kt_handle.clear()) + self.assertTrue(self.kt_http_handle.clear()) # Set record to be expired in 2 seconds. - self.assertTrue(self.kt_handle.set('key', 'value', 2)) - self.assertEqual(self.kt_handle.get('key'), 'value') - self.assertEqual(self.kt_handle.count(), 1) + self.assertTrue(self.kt_http_handle.set('key1', 'value', 2)) + self.assertEqual(self.kt_http_handle.get('key1'), 'value') + self.assertEqual(self.kt_http_handle.count(), 1) + time.sleep(4) + self.assertEqual(self.kt_http_handle.count(), 0) - # Must be expired after 3 seconds. - time.sleep(3) - self.assertEqual(self.kt_handle.get('key'), None) - self.assertEqual(self.kt_handle.count(), 0) + # Set record to be expired in 2 seconds. + self.assertTrue(self.kt_bin_handle.set('key2', 'value', 2)) + self.assertEqual(self.kt_bin_handle.get('key2'), 'value') + self.assertEqual(self.kt_http_handle.count(), 1) + time.sleep(4) + self.assertEqual(self.kt_http_handle.count(), 0) def test_add_expire(self): - self.assertTrue(self.kt_handle.clear()) - - self.assertTrue(self.kt_handle.add('hello', 'world', 2)) - self.assertEqual(self.kt_handle.get('hello'), 'world') + self.assertTrue(self.kt_http_handle.clear()) - time.sleep(3) - self.assertEqual(self.kt_handle.get('hello'), None) + self.assertTrue(self.kt_http_handle.add('hello', 'world', 2)) + self.assertEqual(self.kt_http_handle.get('hello'), 'world') + time.sleep(4) + self.assertEqual(self.kt_http_handle.get('hello'), None) if __name__ == '__main__': unittest.main() diff --git a/tests/t_multi.py b/tests/t_multi.py index 5fb4286..bfda8ca 100644 --- a/tests/t_multi.py +++ b/tests/t_multi.py @@ -6,177 +6,238 @@ # the BSD license. See COPYING file for license description. # # Kyoto Tycoon should be started like this: -# $ ktserver one.kch two.kch three.kch +# $ ktserver one.kch two.kch import config import time import unittest -from kyototycoon import KyotoTycoon +from kyototycoon import KyotoTycoon, KyotoTycoonException -DB_1 = 'one.kch' -DB_2 = 'two.kch' -DB_INVALID = 'invalid.kch' +DB_1 = 0 +DB_2 = 1 +DB_INVALID = 3 class UnitTest(unittest.TestCase): def setUp(self): - self.kt_handle = KyotoTycoon() - self.kt_handle.open() + self.kt_handle_http = KyotoTycoon(binary=False) + self.kt_handle_http.open() + + self.kt_handle_bin = KyotoTycoon(binary=True) + self.kt_handle_bin.open() + self.LARGE_KEY_LEN = 8000 def clear_all(self): - self.assertTrue(self.kt_handle.clear(db=DB_1)) - self.assertTrue(self.kt_handle.clear(db=DB_2)) + self.assertTrue(self.kt_handle_http.clear(db=DB_1)) + self.assertTrue(self.kt_handle_http.clear(db=DB_2)) return True def test_status(self): - status = self.kt_handle.status(DB_1) + status = self.kt_handle_http.status(DB_1) assert status is not None - status = self.kt_handle.status(DB_2) + status = self.kt_handle_http.status(DB_2) assert status is not None - status = self.kt_handle.status(DB_INVALID) - assert status is None - - status = self.kt_handle.status('non_existent') - assert status is None + self.assertRaises(KyotoTycoonException, self.kt_handle_http.status, DB_INVALID) + self.assertRaises(KyotoTycoonException, self.kt_handle_http.status, 'non_existent') def test_set_get(self): self.assertTrue(self.clear_all()) - self.assertTrue(self.kt_handle.set('ice', 'cream', db=DB_2)) - self.assertFalse(self.kt_handle.set('palo', 'alto', db=DB_INVALID)) + self.assertTrue(self.kt_handle_http.set('ice', 'cream', db=DB_2)) + self.assertRaises(KyotoTycoonException, self.kt_handle_http.set, 'palo', 'alto', db=DB_INVALID) - assert self.kt_handle.get('ice') is None - assert self.kt_handle.get('ice', db=DB_1) is None - assert self.kt_handle.get('ice', db=DB_INVALID) is None + self.assertEqual(self.kt_handle_http.get('ice'), None) + self.assertEqual(self.kt_handle_http.get('ice', db=DB_1), None) + self.assertRaises(KyotoTycoonException, self.kt_handle_http.get, 'ice', db=DB_INVALID) - self.assertEqual(self.kt_handle.get('ice', db='two.kch'), 'cream') - self.assertEqual(self.kt_handle.count(db=DB_1), 0) - self.assertEqual(self.kt_handle.count(db=DB_2), 1) + self.assertEqual(self.kt_handle_http.get('ice', db=DB_2), 'cream') + self.assertEqual(self.kt_handle_http.count(db=DB_1), 0) + self.assertEqual(self.kt_handle_http.count(db=DB_2), 1) - self.assertTrue(self.kt_handle.set('frozen', 'yoghurt', db=DB_1)) - self.assertEqual(self.kt_handle.count(db=DB_1), 1) + self.assertTrue(self.kt_handle_http.set('frozen', 'yoghurt', db=DB_1)) + self.assertEqual(self.kt_handle_http.count(db=DB_1), 1) - self.assertEqual(self.kt_handle.get('frozen'), 'yoghurt') - self.assertEqual(self.kt_handle.get('frozen', db=DB_1), 'yoghurt') - assert self.kt_handle.get('frozen', db=DB_2) is None - assert self.kt_handle.get('frozen', db=DB_INVALID) is None + self.assertEqual(self.kt_handle_http.get('frozen'), 'yoghurt') + self.assertEqual(self.kt_handle_http.get('frozen', db=DB_1), 'yoghurt') + self.assertEqual(self.kt_handle_http.get('frozen', db=DB_2), None) + self.assertRaises(KyotoTycoonException, self.kt_handle_http.get, 'frozen', db=DB_INVALID) - self.assertTrue(self.kt_handle.clear(db=DB_1)) - self.assertEqual(self.kt_handle.count(db=DB_1), 0) - self.assertEqual(self.kt_handle.count(db=DB_2), 1) + self.assertTrue(self.kt_handle_http.clear(db=DB_1)) + self.assertEqual(self.kt_handle_http.count(db=DB_1), 0) + self.assertEqual(self.kt_handle_http.count(db=DB_2), 1) def test_get_multi(self): self.assertTrue(self.clear_all()) - self.assertTrue(self.kt_handle.set('a', 'xxxx', db=DB_1)) - self.assertTrue(self.kt_handle.set('b', 'yyyy', db=DB_1)) - self.assertTrue(self.kt_handle.set('c', 'zzzz', db=DB_1)) - self.assertTrue(self.kt_handle.set('a1', 'xxxx', db=DB_2)) - self.assertTrue(self.kt_handle.set('b1', 'yyyy', db=DB_2)) - self.assertTrue(self.kt_handle.set('c1', 'zzzz', db=DB_2)) + self.assertTrue(self.kt_handle_http.set('a', 'xxxx', db=DB_1)) + self.assertTrue(self.kt_handle_http.set('b', 'yyyy', db=DB_1)) + self.assertTrue(self.kt_handle_http.set('c', 'zzzz', db=DB_1)) + self.assertTrue(self.kt_handle_http.set('a1', 'xxxx', db=DB_2)) + self.assertTrue(self.kt_handle_http.set('b1', 'yyyy', db=DB_2)) + self.assertTrue(self.kt_handle_http.set('c1', 'zzzz', db=DB_2)) - d = self.kt_handle.get_bulk(['a', 'b', 'c'], db=DB_1) + d = self.kt_handle_http.get_bulk(['a', 'b', 'c'], db=DB_1) self.assertEqual(len(d), 3) self.assertEqual(d['a'], 'xxxx') self.assertEqual(d['b'], 'yyyy') self.assertEqual(d['c'], 'zzzz') - d = self.kt_handle.get_bulk(['a', 'b', 'c'], db=DB_2) + d = self.kt_handle_http.get_bulk(['a', 'b', 'c'], db=DB_2) self.assertEqual(len(d), 0) - d = self.kt_handle.get_bulk(['a1', 'b1', 'c1'], db=DB_2) + d = self.kt_handle_http.get_bulk(['a1', 'b1', 'c1'], db=DB_2) self.assertEqual(len(d), 3) self.assertEqual(d['a1'], 'xxxx') self.assertEqual(d['b1'], 'yyyy') self.assertEqual(d['c1'], 'zzzz') - d = self.kt_handle.get_bulk(['a1', 'b1', 'c1'], db=DB_1) + d = self.kt_handle_http.get_bulk(['a1', 'b1', 'c1'], db=DB_1) self.assertEqual(len(d), 0) def test_add(self): self.assertTrue(self.clear_all()) # Should not conflict due to different databases. - self.assertTrue(self.kt_handle.add('key1', 'val1', db=DB_1)) - self.assertTrue(self.kt_handle.add('key1', 'val1', db=DB_2)) + self.assertTrue(self.kt_handle_http.add('key1', 'val1', db=DB_1)) + self.assertTrue(self.kt_handle_http.add('key1', 'val1', db=DB_2)) # Now they should. - self.assertFalse(self.kt_handle.add('key1', 'val1', db=DB_1)) - self.assertFalse(self.kt_handle.add('key1', 'val1', db=DB_2)) - self.assertFalse(self.kt_handle.add('key1', 'val1', db=DB_INVALID)) + self.assertRaises(KyotoTycoonException, self.kt_handle_http.add, 'key1', 'val1', db=DB_1) + self.assertRaises(KyotoTycoonException, self.kt_handle_http.add, 'key1', 'val1', db=DB_2) + self.assertRaises(KyotoTycoonException, self.kt_handle_http.add, 'key1', 'val1', db=DB_INVALID) def test_replace(self): self.assertTrue(self.clear_all()) - self.assertTrue(self.kt_handle.add('key1', 'val1', db=DB_1)) - self.assertFalse(self.kt_handle.replace('key1', 'val2', db=DB_2)) - self.assertTrue(self.kt_handle.replace('key1', 'val2', db=DB_1)) - self.assertFalse(self.kt_handle.replace('key1', 'val2', db=DB_INVALID)) + self.assertTrue(self.kt_handle_http.add('key1', 'val1', db=DB_1)) + self.assertFalse(self.kt_handle_http.replace('key1', 'val2', db=DB_2)) + self.assertTrue(self.kt_handle_http.replace('key1', 'val2', db=DB_1)) + self.assertFalse(self.kt_handle_http.replace('key1', 'val2', db=DB_INVALID)) - self.assertTrue(self.kt_handle.add('key2', 'aaa')) - self.assertTrue(self.kt_handle.replace('key2', 'bbb')) - self.assertTrue(self.kt_handle.replace('key1', 'zzz')) - self.assertEqual(self.kt_handle.get('key2'), 'bbb') - self.assertEqual(self.kt_handle.get('key1'), 'zzz') + self.assertTrue(self.kt_handle_http.add('key2', 'aaa')) + self.assertTrue(self.kt_handle_http.replace('key2', 'bbb')) + self.assertTrue(self.kt_handle_http.replace('key1', 'zzz')) + self.assertEqual(self.kt_handle_http.get('key2'), 'bbb') + self.assertEqual(self.kt_handle_http.get('key1'), 'zzz') def test_cas(self): self.assertTrue(self.clear_all()) - self.assertTrue(self.kt_handle.set('key', 'xxx')) - self.assertFalse(self.kt_handle.cas('key', old_val='xxx', - new_val='yyy', db=DB_2)) - self.assertTrue(self.kt_handle.cas('key', old_val='xxx', - new_val='yyy', db=DB_1)) - self.assertTrue(self.kt_handle.cas('key', new_val='xxx', db=DB_2)) + self.assertTrue(self.kt_handle_http.set('key', 'xxx')) + self.assertEqual(self.kt_handle_http.get('key', db=DB_2), None) + self.assertRaises(KyotoTycoonException, self.kt_handle_http.cas, 'key', old_val='xxx', new_val='yyy', db=DB_2) + self.assertEqual(self.kt_handle_http.get('key', db=DB_1), 'xxx') + self.assertTrue(self.kt_handle_http.cas('key', old_val='xxx', new_val='yyy', db=DB_1)) + self.assertTrue(self.kt_handle_http.cas('key', new_val='xxx', db=DB_2)) def test_remove(self): self.assertTrue(self.clear_all()) - self.assertTrue(self.kt_handle.add('key', 'value', db=DB_1)) - self.assertTrue(self.kt_handle.add('key', 'value', db=DB_2)) + self.assertTrue(self.kt_handle_http.add('key', 'value', db=DB_1)) + self.assertTrue(self.kt_handle_http.add('key', 'value', db=DB_2)) - self.assertTrue(self.kt_handle.remove('key', db=DB_1)) - self.assertEqual(self.kt_handle.get('key', db=DB_2), 'value') - assert self.kt_handle.get('key', db=DB_1) is None + self.assertTrue(self.kt_handle_http.remove('key', db=DB_1)) + self.assertEqual(self.kt_handle_http.get('key', db=DB_2), 'value') + assert self.kt_handle_http.get('key', db=DB_1) is None def test_vacuum(self): - self.assertTrue(self.kt_handle.vacuum()) - self.assertTrue(self.kt_handle.vacuum(db=DB_1)) - self.assertTrue(self.kt_handle.vacuum(db=DB_2)) - self.assertFalse(self.kt_handle.vacuum(db=DB_INVALID)) + self.assertTrue(self.kt_handle_http.vacuum()) + self.assertTrue(self.kt_handle_http.vacuum(db=DB_1)) + self.assertTrue(self.kt_handle_http.vacuum(db=DB_2)) + self.assertRaises(KyotoTycoonException, self.kt_handle_http.vacuum, db=DB_INVALID) def test_append(self): self.assertTrue(self.clear_all()) - self.assertTrue(self.kt_handle.set('key', 'xxx', db=DB_1)) - self.assertTrue(self.kt_handle.set('key', 'xxx', db=DB_2)) - self.assertTrue(self.kt_handle.append('key', 'xxx', db=DB_1)) + self.assertTrue(self.kt_handle_http.set('key', 'xxx', db=DB_1)) + self.assertTrue(self.kt_handle_http.set('key', 'xxx', db=DB_2)) + self.assertTrue(self.kt_handle_http.append('key', 'xxx', db=DB_1)) - self.assertEqual(self.kt_handle.get('key', db=DB_1), 'xxxxxx') - self.assertEqual(self.kt_handle.get('key', db=DB_2), 'xxx') + self.assertEqual(self.kt_handle_http.get('key', db=DB_1), 'xxxxxx') + self.assertEqual(self.kt_handle_http.get('key', db=DB_2), 'xxx') def test_increment(self): self.assertTrue(self.clear_all()) - self.assertEqual(self.kt_handle.increment('key', 0, db=DB_1), 0) - self.assertEqual(self.kt_handle.increment('key', 0, db=DB_2), 0) + self.assertEqual(self.kt_handle_http.increment('key', 0, db=DB_1), 0) + self.assertEqual(self.kt_handle_http.increment('key', 0, db=DB_2), 0) - self.assertEqual(self.kt_handle.increment('key', 100, db=DB_1), 100) - self.assertEqual(self.kt_handle.increment('key', 200, db=DB_2), 200) - self.assertEqual(self.kt_handle.increment('key', 100, db=DB_1), 200) - self.assertEqual(self.kt_handle.increment('key', 200, db=DB_2), 400) - self.assertEqual(self.kt_handle.get_int('key', db=DB_1), 200) - self.assertEqual(self.kt_handle.get_int('key', db=DB_2), 400) + self.assertEqual(self.kt_handle_http.increment('key', 100, db=DB_1), 100) + self.assertEqual(self.kt_handle_http.increment('key', 200, db=DB_2), 200) + self.assertEqual(self.kt_handle_http.increment('key', 100, db=DB_1), 200) + self.assertEqual(self.kt_handle_http.increment('key', 200, db=DB_2), 400) + self.assertEqual(self.kt_handle_http.get_int('key', db=DB_1), 200) + self.assertEqual(self.kt_handle_http.get_int('key', db=DB_2), 400) def test_match_prefix(self): self.assertTrue(self.clear_all()) - self.assertTrue(self.kt_handle.set('abcdef', 'val', db=DB_1)) - self.assertTrue(self.kt_handle.set('fedcba', 'val', db=DB_2)) + self.assertTrue(self.kt_handle_http.set('abcdef', 'val', db=DB_1)) + self.assertTrue(self.kt_handle_http.set('fedcba', 'val', db=DB_2)) - list = self.kt_handle.match_prefix('abc', db=DB_1) + list = self.kt_handle_http.match_prefix('abc', db=DB_1) self.assertEqual(len(list), 1) self.assertEqual(list[0], 'abcdef') - list = self.kt_handle.match_prefix('abc', db=DB_2) + list = self.kt_handle_http.match_prefix('abc', db=DB_2) self.assertEqual(len(list), 0) - list = self.kt_handle.match_prefix('fed', db=DB_1) + list = self.kt_handle_http.match_prefix('fed', db=DB_1) self.assertEqual(len(list), 0) - list = self.kt_handle.match_prefix('fed', db=DB_2) + list = self.kt_handle_http.match_prefix('fed', db=DB_2) self.assertEqual(len(list), 1) self.assertEqual(list[0], 'fedcba') + def test_set_get_bin(self): + self.assertTrue(self.clear_all()) + self.assertTrue(self.kt_handle_bin.set('ice', 'cream', db=DB_2)) + self.assertFalse(self.kt_handle_bin.set('palo', 'alto', db=DB_INVALID)) + + self.assertEqual(self.kt_handle_bin.get('ice'), None) + self.assertEqual(self.kt_handle_bin.get('ice', db=DB_1), None) + self.assertFalse(self.kt_handle_bin.get('ice', db=DB_INVALID)) + + self.assertEqual(self.kt_handle_bin.get('ice', db=DB_2), 'cream') + self.assertEqual(self.kt_handle_http.count(db=DB_1), 0) + self.assertEqual(self.kt_handle_http.count(db=DB_2), 1) + + self.assertTrue(self.kt_handle_bin.set('frozen', 'yoghurt', db=DB_1)) + self.assertEqual(self.kt_handle_http.count(db=DB_1), 1) + + self.assertEqual(self.kt_handle_bin.get('frozen'), 'yoghurt') + self.assertEqual(self.kt_handle_bin.get('frozen', db=DB_1), 'yoghurt') + self.assertEqual(self.kt_handle_bin.get('frozen', db=DB_2), None) + self.assertFalse(self.kt_handle_bin.get('frozen', db=DB_INVALID), None) + + self.assertTrue(self.kt_handle_http.clear(db=DB_1)) + self.assertEqual(self.kt_handle_http.count(db=DB_1), 0) + self.assertEqual(self.kt_handle_http.count(db=DB_2), 1) + + def test_get_multi_bin(self): + self.assertTrue(self.clear_all()) + + self.assertTrue(self.kt_handle_bin.set('a', 'xxxx', db=DB_1)) + self.assertTrue(self.kt_handle_bin.set('b', 'yyyy', db=DB_1)) + self.assertTrue(self.kt_handle_bin.set('c', 'zzzz', db=DB_1)) + self.assertTrue(self.kt_handle_bin.set('a1', 'xxxx', db=DB_2)) + self.assertTrue(self.kt_handle_bin.set('b1', 'yyyy', db=DB_2)) + self.assertTrue(self.kt_handle_bin.set('c1', 'zzzz', db=DB_2)) + + d = self.kt_handle_bin.get_bulk(['a', 'b', 'c'], db=DB_1, atomic=False) + self.assertEqual(len(d), 3) + self.assertEqual(d['a'], 'xxxx') + self.assertEqual(d['b'], 'yyyy') + self.assertEqual(d['c'], 'zzzz') + d = self.kt_handle_bin.get_bulk(['a', 'b', 'c'], db=DB_2, atomic=False) + self.assertEqual(len(d), 0) + + d = self.kt_handle_bin.get_bulk(['a1', 'b1', 'c1'], db=DB_2, atomic=False) + self.assertEqual(len(d), 3) + self.assertEqual(d['a1'], 'xxxx') + self.assertEqual(d['b1'], 'yyyy') + self.assertEqual(d['c1'], 'zzzz') + d = self.kt_handle_bin.get_bulk(['a1', 'b1', 'c1'], db=DB_1, atomic=False) + self.assertEqual(len(d), 0) + + def test_remove_bin(self): + self.assertTrue(self.clear_all()) + self.assertTrue(self.kt_handle_bin.set('key', 'value', db=DB_1)) + self.assertTrue(self.kt_handle_bin.set('key', 'value', db=DB_2)) + + self.assertTrue(self.kt_handle_bin.remove('key', db=DB_1)) + self.assertEqual(self.kt_handle_bin.get('key', db=DB_2), 'value') + assert self.kt_handle_bin.get('key', db=DB_1) is None + if __name__ == '__main__': unittest.main() diff --git a/tests/t_packer_bytes.py b/tests/t_packer_bytes.py new file mode 100644 index 0000000..1211b89 --- /dev/null +++ b/tests/t_packer_bytes.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011, Toru Maesaka +# +# Redistribution and use of this source code is licensed under +# the BSD license. See COPYING file for license description. + +import config +import unittest +from kyototycoon import KyotoTycoon + +import kyototycoon.kt_http as kt_http +import kyototycoon.kt_binary as kt_binary + +class UnitTest(unittest.TestCase): + def setUp(self): + self.kt_bin_handle = KyotoTycoon(binary=True, pack_type=kt_binary.KT_PACKER_BYTES) + self.kt_bin_handle.open() + + self.kt_http_handle = KyotoTycoon(binary=False, pack_type=kt_http.KT_PACKER_BYTES) + self.kt_http_handle.open() + + def test_packer_bytes(self): + self.assertTrue(self.kt_http_handle.clear()) + self.assertEqual(self.kt_http_handle.count(), 0) + + value1 = bytes([0x00, 0x01, 0x02, 0xff]) + self.assertTrue(self.kt_bin_handle.set('key1', value1)) + value2 = self.kt_bin_handle.get('key1') + self.assertEqual(value1, value2) + self.assertEqual(type(value1), type(value2)) + + value3 = bytes([0x00, 0x01, 0x03, 0xff]) + self.assertTrue(self.kt_http_handle.set('key2', value3)) + value4 = self.kt_http_handle.get('key2') + self.assertEqual(value3, value4) + self.assertEqual(type(value3), type(value4)) + + self.assertTrue(self.kt_http_handle.append('key2', 'xyz')) + self.assertEqual(self.kt_http_handle.get('key2'), value3 + b'xyz') + + self.assertEqual(self.kt_http_handle.count(), 2) + self.assertTrue(self.kt_http_handle.clear()) + self.assertEqual(self.kt_http_handle.count(), 0) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/t_packer_custom.py b/tests/t_packer_custom.py new file mode 100644 index 0000000..9705571 --- /dev/null +++ b/tests/t_packer_custom.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011, Toru Maesaka +# +# Redistribution and use of this source code is licensed under +# the BSD license. See COPYING file for license description. + +import config +import unittest +from kyototycoon import KyotoTycoon + +import kyototycoon.kt_http as kt_http +import kyototycoon.kt_binary as kt_binary + +class CustomPacker(object): + prefix = '' + suffix = '' + + def pack(self, data): + data = self.prefix + data + self.suffix + return data.encode('utf-8') + + def unpack(self, data): + data = data[len(self.prefix):-len(self.suffix)] + return data.decode('utf-8') + +class UnitTest(unittest.TestCase): + def setUp(self): + self.kt_bin_handle = KyotoTycoon(binary=True, pack_type=kt_binary.KT_PACKER_CUSTOM, + custom_packer=CustomPacker()) + self.kt_bin_handle.open() + + self.kt_http_handle = KyotoTycoon(binary=False, pack_type=kt_http.KT_PACKER_CUSTOM, + custom_packer=CustomPacker()) + self.kt_http_handle.open() + + def test_packer_bytes(self): + self.assertTrue(self.kt_http_handle.clear()) + self.assertEqual(self.kt_http_handle.count(), 0) + + self.assertTrue(self.kt_bin_handle.set('key1', 'abc')) + self.assertEqual(self.kt_bin_handle.get('key1'), 'abc') + + self.assertTrue(self.kt_http_handle.set('key2', 'abcd')) + self.assertEqual(self.kt_http_handle.get('key2'), 'abcd') + + self.assertTrue(self.kt_http_handle.append('key2', 'xyz')) + self.assertEqual(self.kt_http_handle.get('key2'), 'abcdxyz') + + self.assertEqual(self.kt_http_handle.count(), 2) + self.assertTrue(self.kt_http_handle.clear()) + self.assertEqual(self.kt_http_handle.count(), 0) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/t_script.lua b/tests/t_script.lua new file mode 100644 index 0000000..0b1c22c --- /dev/null +++ b/tests/t_script.lua @@ -0,0 +1,51 @@ +-- This is a subset of "example/ktscrex.lua" from the Kyoto Tycoon 0.9.56 source distribution. + +kt = __kyototycoon__ +db = kt.db + +-- log the start-up message +if kt.thid == 0 then + kt.log("system", "loaded Lua script for 'play_script' unit tests") +end + +-- echo back the input data as the output data +function echo(inmap, outmap) + for key, value in pairs(inmap) do + outmap[key] = value + end + return kt.RVSUCCESS +end + +-- store a record +function set(inmap, outmap) + local key = inmap.key + local value = inmap.value + if not key or not value then + return kt.RVEINVALID + end + local xt = inmap.xt + if not db:set(key, value, xt) then + return kt.RVEINTERNAL + end + return kt.RVSUCCESS +end + +-- retrieve the value of a record +function get(inmap, outmap) + local key = inmap.key + if not key then + return kt.RVEINVALID + end + local value, xt = db:get(key) + if value then + outmap.value = value + outmap.xt = xt + else + local err = db:error() + if err:code() == kt.Error.NOREC then + return kt.RVELOGIC + end + return kt.RVEINTERNAL + end + return kt.RVSUCCESS +end diff --git a/tests/t_script.py b/tests/t_script.py new file mode 100644 index 0000000..9498a2e --- /dev/null +++ b/tests/t_script.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# +# Copyright 2011, Toru Maesaka +# +# Redistribution and use of this source code is licensed under +# the BSD license. See COPYING file for license description. +# +# Kyoto Tycoon should be started like this: +# $ ktserver -scr example/ktscrex.lua '%' + +import config +import unittest +from kyototycoon import KyotoTycoon + +class UnitTest(unittest.TestCase): + def setUp(self): + self.kt_http_handle = KyotoTycoon(binary=False) + self.kt_http_handle.open() + + self.kt_bin_handle = KyotoTycoon(binary=True) + self.kt_bin_handle.open() + + def test_play_script(self): + self.assertTrue(self.kt_http_handle.clear()) + self.assertEqual(self.kt_http_handle.count(), 0) + + self.assertEqual(self.kt_bin_handle.play_script('echo', {'key1': b'abc'}), {'key1': b'abc'}) + + self.assertEqual(self.kt_bin_handle.play_script('set', {'key': b'key2', 'value': b'abcd'}), {}) + out = self.kt_bin_handle.play_script('get', {'key': b'key2'}) + self.assertTrue('value' in out) + self.assertEqual(out['value'], b'abcd') + + self.assertEqual(self.kt_http_handle.play_script('echo', {'key3': b'abc'}), {'key3': b'abc'}) + + self.assertEqual(self.kt_http_handle.play_script('set', {'key': b'key4', 'value': b'abcd'}), {}) + out = self.kt_http_handle.play_script('get', {'key': b'key4'}) + self.assertTrue('value' in out) + self.assertEqual(out['value'], b'abcd') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/t_simple.py b/tests/t_simple.py index 7f75efe..ad3140c 100644 --- a/tests/t_simple.py +++ b/tests/t_simple.py @@ -7,20 +7,18 @@ import config import unittest -from kyototycoon import KyotoTycoon +from kyototycoon import KyotoTycoon, KyotoTycoonException, KT_PACKER_PICKLE class UnitTest(unittest.TestCase): def setUp(self): - self.kt_handle = KyotoTycoon() + self.kt_handle = KyotoTycoon(binary=False, pack_type=KT_PACKER_PICKLE) self.kt_handle.open() self.LARGE_KEY_LEN = 8000 def test_set(self): self.assertTrue(self.kt_handle.clear()) - error = self.kt_handle.error() self.assertTrue(self.kt_handle.set('key', 'value')) - self.assertEqual(error.code(), error.SUCCESS) self.assertTrue(self.kt_handle.set('k e y', 'v a l u e')) self.assertTrue(self.kt_handle.set('k\te\ty', 'tabbed')) @@ -44,85 +42,51 @@ def test_set(self): self.assertTrue(self.kt_handle.set('https://github.com', 'url2')) self.assertTrue(self.kt_handle.set('https://github.com/blog/', 'url3')) - self.assertFalse(self.kt_handle.set(None, 'value')) - self.assertEqual(error.code(), error.LOGIC) - - self.assertFalse(self.kt_handle.get(None)) - self.assertEqual(error.code(), error.LOGIC) - self.assertEqual(self.kt_handle.get('non_existent'), None) - self.assertEqual(error.code(), error.NOTFOUND) self.assertTrue(self.kt_handle.set('cb', 1791)) - self.assertEqual(error.code(), error.SUCCESS) self.assertEqual(self.kt_handle.get('cb'), 1791) - self.assertEqual(error.code(), error.SUCCESS) self.assertTrue(self.kt_handle.set('cb', 1791.1226)) self.assertEqual(self.kt_handle.get('cb'), 1791.1226) def test_cas(self): self.assertTrue(self.kt_handle.clear()) - error = self.kt_handle.error() self.assertTrue(self.kt_handle.set('key', 'xxx')) - self.assertEqual(error.code(), error.SUCCESS) self.assertTrue(self.kt_handle.cas('key', old_val='xxx', new_val='yyy')) - self.assertEqual(error.code(), error.SUCCESS) self.assertEqual(self.kt_handle.get('key'), 'yyy') - self.assertEqual(error.code(), error.SUCCESS) self.assertTrue(self.kt_handle.cas('key', old_val='yyy')) assert self.kt_handle.get('key') is None - self.assertEqual(error.code(), error.NOTFOUND) self.assertTrue(self.kt_handle.cas('key', new_val='zzz')) self.assertEqual(self.kt_handle.get('key'), 'zzz') - self.assertEqual(error.code(), error.SUCCESS) - self.assertFalse(self.kt_handle.cas('key', old_val='foo', new_val='zz')) - self.assertEqual(error.code(), error.EMISC) + self.assertRaises(KyotoTycoonException, self.kt_handle.cas, 'key', old_val='foo', new_val='zz') self.assertEqual(self.kt_handle.get('key'), 'zzz') - self.assertEqual(error.code(), error.SUCCESS) def test_remove(self): self.assertTrue(self.kt_handle.clear()) - error = self.kt_handle.error() self.assertFalse(self.kt_handle.remove('must fail key')) - self.assertEqual(error.code(), error.NOTFOUND) self.assertTrue(self.kt_handle.set('deleteable key', 'xxx')) - self.assertEqual(error.code(), error.SUCCESS) self.assertTrue(self.kt_handle.remove('deleteable key')) - self.assertEqual(error.code(), error.SUCCESS) - self.assertFalse(self.kt_handle.remove(None)) - self.assertEqual(error.code(), error.LOGIC) def test_replace(self): self.assertTrue(self.kt_handle.clear()) - error = self.kt_handle.error() # Must Fail - Can't replace something that doesn't exist. self.assertFalse(self.kt_handle.replace('xxxxxx', 'some value')) - self.assertEqual(error.code(), error.NOTFOUND) # Popuate then Replace. self.assertTrue(self.kt_handle.set('apple', 'ringo')) - self.assertEqual(error.code(), error.SUCCESS) self.assertTrue(self.kt_handle.replace('apple', 'apfel')) - self.assertEqual(error.code(), error.SUCCESS) self.assertEqual(self.kt_handle.get('apple'), 'apfel') - self.assertEqual(error.code(), error.SUCCESS) - self.assertFalse(self.kt_handle.replace(None, 'value')) - self.assertEqual(error.code(), error.LOGIC) self.assertTrue(self.kt_handle.replace('apple', 212)) - self.assertEqual(error.code(), error.SUCCESS) self.assertEqual(self.kt_handle.get('apple'), 212) - self.assertEqual(error.code(), error.SUCCESS) self.assertTrue(self.kt_handle.replace('apple', 121)) - self.assertEqual(error.code(), error.SUCCESS) self.assertEqual(self.kt_handle.get('apple'), 121) - self.assertEqual(error.code(), error.SUCCESS) def test_append(self): self.assertTrue(self.kt_handle.clear()) @@ -134,21 +98,27 @@ def test_append(self): self.assertEqual(self.kt_handle.get('k2'), 'new val') self.assertTrue(self.kt_handle.set('k3', 777)) - self.assertFalse(self.kt_handle.append('k3', 111)) + + self.assertTrue(self.kt_handle.set('k4', b'abc')) + self.assertTrue(self.kt_handle.append('k4', 'abc')) + self.assertEqual(self.kt_handle.get('k4'), b'abcabc') + + self.assertTrue(self.kt_handle.set('k5', 'abc')) + self.assertTrue(self.kt_handle.append('k5', b'abc')) + self.assertEqual(self.kt_handle.get('k5'), 'abcabc') def test_add(self): self.assertTrue(self.kt_handle.clear()) self.assertTrue(self.kt_handle.set('stewie', 'griffin')) # Must Fail - Stewie exists - self.assertFalse(self.kt_handle.add('stewie', 'hopkin')) + self.assertRaises(KyotoTycoonException, self.kt_handle.add, 'stewie', 'hopkin') # New records self.assertTrue(self.kt_handle.add('peter', 'griffin')) self.assertTrue(self.kt_handle.add('lois', 'griffin')) self.assertTrue(self.kt_handle.add('seth', 'green')) self.assertTrue(self.kt_handle.add('nyc', 'new york city')) - self.assertFalse(self.kt_handle.add(None, 'value')) self.assertTrue(self.kt_handle.add('number', 111)) self.assertEqual(self.kt_handle.get('number'), 111) @@ -221,6 +191,8 @@ def test_report(self): report = self.kt_handle.report() assert report is not None + self.assertTrue(int(report['serv_conn_count']) > 0) + def test_status(self): self.assertTrue(self.kt_handle.clear()) status = None @@ -233,12 +205,6 @@ def test_status(self): self.kt_handle.set('pink', 'peach') self.assertTrue(status['count'], 3) - def test_error(self): - self.assertTrue(self.kt_handle.clear()) - kt_error = self.kt_handle.error() - assert kt_error is not None - self.assertEqual(kt_error.code(), kt_error.SUCCESS) - def test_vacuum(self): self.assertTrue(self.kt_handle.vacuum()) @@ -256,7 +222,7 @@ def test_match_prefix(self): list = self.kt_handle.match_prefix('abcd') self.assertEqual(len(list), 5) list = self.kt_handle.match_prefix('abc', 1) - self.assertEqual(list[0], 'abc') + self.assertEqual(list[0][:3], 'abc') def test_match_regex(self): self.assertTrue(self.kt_handle.clear()) @@ -271,5 +237,60 @@ def test_match_regex(self): list = self.kt_handle.match_regex('e$') self.assertEqual(len(list), 1) + def test_cursor(self): + self.assertTrue(self.kt_handle.clear()) + self.assertEqual(self.kt_handle.count(), 0) + self.assertTrue(self.kt_handle.set('abc', 'val')) + self.assertTrue(self.kt_handle.set('abcd', 'val')) + self.assertTrue(self.kt_handle.set('abcde', 'val')) + self.assertEqual(self.kt_handle.count(), 3) + + cur = self.kt_handle.cursor() + self.assertTrue(cur.jump()) + while True: + self.assertEqual(cur.get_key()[:3], 'abc') + self.assertEqual(cur.get_value(), 'val') + + pair = cur.get() + self.assertEqual(len(pair), 2) + self.assertEqual(pair[0][:3], 'abc') + self.assertEqual(pair[1], 'val') + + self.assertTrue(cur.set_value('foo')) + self.assertEqual(cur.get_value(), 'foo') + + if pair[0] == 'abcd': + self.assertTrue(cur.remove()) + break + + if not cur.step(): + break + + self.assertTrue(cur.delete()) + self.assertEqual(self.kt_handle.count(), 2) + + self.assertTrue(self.kt_handle.set('zabc', 'val')) + self.assertTrue(self.kt_handle.set('zabcd', 'val')) + self.assertTrue(self.kt_handle.set('zabcde', 'val')) + self.assertEqual(self.kt_handle.count(), 5) + + cur = self.kt_handle.cursor() + self.assertTrue(cur.jump(key='zabc')) + while True: + pair = cur.get() + + if pair[0] == 'zabc': + dict = cur.seize() + self.assertEqual(len(dict), 2) + self.assertEqual(dict['key'][:4], 'zabc') + self.assertEqual(dict['value'], 'val') + + if not cur.step(): + break + + self.assertTrue(cur.delete()) + self.assertEqual(self.kt_handle.count(), 4) + + if __name__ == '__main__': unittest.main() diff --git a/tests/t_simple_binary.py b/tests/t_simple_binary.py new file mode 100644 index 0000000..2c5b367 --- /dev/null +++ b/tests/t_simple_binary.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# +# Copyright 2011, Toru Maesaka +# +# Redistribution and use of this source code is licensed under +# the BSD license. See COPYING file for license description. + +import config +import unittest +from kyototycoon import KyotoTycoon +from kyototycoon.kt_binary import KT_PACKER_PICKLE + +class UnitTest(unittest.TestCase): + def setUp(self): + # For operations not supported by the binary protocol, but useful for testing it... + self.kt_handle_http = KyotoTycoon(binary=False) + self.kt_handle_http.open() + + self.kt_handle = KyotoTycoon(binary=True, pack_type=KT_PACKER_PICKLE) + self.kt_handle.open() + self.LARGE_KEY_LEN = 8000 + + def test_set(self): + self.assertTrue(self.kt_handle_http.clear()) + + self.assertTrue(self.kt_handle.set('key', 'value')) + + self.assertTrue(self.kt_handle.set('k e y', 'v a l u e')) + self.assertTrue(self.kt_handle.set('k\te\ty', 'tabbed')) + + self.assertEqual(self.kt_handle.get('key'), 'value') + self.assertEqual(self.kt_handle.get('k e y'), 'v a l u e') + self.assertEqual(self.kt_handle.get('k\te\ty'), 'tabbed') + + self.assertTrue(self.kt_handle.set('\\key', '\\xxx')) + self.assertEqual(self.kt_handle.get('\\key'), '\\xxx') + self.assertEqual(self.kt_handle_http.count(), 4) + + self.assertTrue(self.kt_handle.set('tabbed\tkey', 'tabbled\tvalue')) + self.assertTrue(self.kt_handle.get('tabbed\tkey')) + + self.assertTrue(self.kt_handle.set('url1', 'http://github.com')) + self.assertTrue(self.kt_handle.set('url2', 'https://github.com/')) + self.assertTrue(self.kt_handle.set('url3', 'https://github.com/blog/')) + + self.assertTrue(self.kt_handle.set('http://github.com', 'url1')) + self.assertTrue(self.kt_handle.set('https://github.com', 'url2')) + self.assertTrue(self.kt_handle.set('https://github.com/blog/', 'url3')) + + self.assertEqual(self.kt_handle.get('non_existent'), None) + + self.assertTrue(self.kt_handle.set('cb', 1791)) + self.assertEqual(self.kt_handle.get('cb'), 1791) + + self.assertTrue(self.kt_handle.set('cb', 1791.1226)) + self.assertEqual(self.kt_handle.get('cb'), 1791.1226) + + def test_remove(self): + self.assertTrue(self.kt_handle_http.clear()) + + self.assertFalse(self.kt_handle.remove('must fail key')) + self.assertTrue(self.kt_handle.set('deleteable key', 'xxx')) + self.assertTrue(self.kt_handle.remove('deleteable key')) + + def test_set_bulk(self): + self.assertTrue(self.kt_handle_http.clear()) + + dict = { + 'k1': 'one', + 'k2': 'two', + 'k3': 'three', + 'k4': 'four', + 'k\n5': 'five', + 'k\t6': 'six', + 'k7': 111 + } + + n = self.kt_handle.set_bulk(dict, atomic=False) + self.assertEqual(len(dict), n) + self.assertEqual(self.kt_handle.get('k1'), 'one') + self.assertEqual(self.kt_handle.get('k2'), 'two') + self.assertEqual(self.kt_handle.get('k3'), 'three') + self.assertEqual(self.kt_handle.get('k4'), 'four') + self.assertEqual(self.kt_handle.get('k\n5'), 'five') + self.assertEqual(self.kt_handle.get('k\t6'), 'six') + self.assertEqual(self.kt_handle.get('k7'), 111) + + d = self.kt_handle.get_bulk(['k1', 'k2', 'k3', 'k4', + 'k\n5', 'k\t6', 'k7'], atomic=False) + + self.assertEqual(len(d), len(dict)) + self.assertEqual(d, dict) + + self.assertEqual(self.kt_handle_http.count(), 7) + n = self.kt_handle.remove_bulk(['k1', 'k2', 'k\t6'], atomic=False) + self.assertEqual(self.kt_handle_http.count(), 4) + n = self.kt_handle.remove_bulk(['k3'], atomic=False) + self.assertEqual(self.kt_handle_http.count(), 3) + + def test_get_bulk(self): + self.assertTrue(self.kt_handle_http.clear()) + self.assertTrue(self.kt_handle.set('a', 'one')) + self.assertTrue(self.kt_handle.set('b', 'two')) + self.assertTrue(self.kt_handle.set('c', 'three')) + self.assertTrue(self.kt_handle.set('d', 'four')) + + d = self.kt_handle.get_bulk(['a','b','c','d'], atomic=False) + assert d is not None + + self.assertEqual(d['a'], 'one') + self.assertEqual(d['b'], 'two') + self.assertEqual(d['c'], 'three') + self.assertEqual(d['d'], 'four') + self.assertEqual(len(d), 4) + + d = self.kt_handle.get_bulk(['a','x','y','d'], atomic=False) + self.assertEqual(len(d), 2) + d = self.kt_handle.get_bulk(['w','x','y','z'], atomic=False) + self.assertEqual(len(d), 0) + d = self.kt_handle.get_bulk([], atomic=False) + self.assertEqual(d, {}) + + def test_large_key(self): + large_key = 'x' * self.LARGE_KEY_LEN + self.assertTrue(self.kt_handle.set(large_key, 'value')) + self.assertEqual(self.kt_handle.get(large_key), 'value') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/t_utils.py b/tests/t_utils.py index e8a640f..fd5d203 100644 --- a/tests/t_utils.py +++ b/tests/t_utils.py @@ -14,16 +14,16 @@ def setUp(self): self.kt_core = kyototycoon.kt_http.ProtocolHandler() def test_packer(self): - str = 'hello world sir' - buf = self.kt_core._pickle_packer(str) - assert buf != str - ret = self.kt_core._pickle_unpacker(buf) - self.assertEqual(str, ret) + stri = 'hello world sir' + buf = self.kt_core.pack(stri) + assert buf != stri + ret = self.kt_core.unpack(buf) + self.assertEqual(stri, ret) num = 777 - buf = self.kt_core._pickle_packer(num) + buf = self.kt_core.pack(num) assert buf != num - ret = self.kt_core._pickle_unpacker(buf) + ret = self.kt_core.unpack(buf) self.assertEqual(type(num), type(ret)) self.assertEqual(num, ret)