diff --git a/.travis.yml b/.travis.yml index ec33da3..19f174c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,13 @@ language: python -python: 2.7 +python: 3.4 jdk: oraclejdk7 sudo: false branches: except: - gh-pages env: - - TOX_ENV=py26-cdh - - TOX_ENV=py27-cdh - - TOX_ENV=py26-hdp - - TOX_ENV=py27-hdp + - TOX_ENV=py34-cdh + - TOX_ENV=py34-hdp install: - pip install tox script: diff --git a/requirements.txt b/requirements.txt index 544f3b8..e842bc8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ argparse -protobuf>2.4.1 \ No newline at end of file +protobuf==3.0.0b2.post2 diff --git a/snakebite/channel.py b/snakebite/channel.py index 31c1420..1a97a15 100644 --- a/snakebite/channel.py +++ b/snakebite/channel.py @@ -38,6 +38,9 @@ ''' +# python 3 support +from __future__ import absolute_import, print_function, division + # Standard library imports import socket import os @@ -56,13 +59,13 @@ from snakebite.formatter import format_bytes from snakebite.errors import RequestError, TransientException, FatalException from snakebite.crc32c import crc +from snakebite.compat import range, py_2 +from snakebite import logger import google.protobuf.internal.encoder as encoder import google.protobuf.internal.decoder as decoder # Module imports - -import logger import logging import struct import uuid @@ -131,7 +134,7 @@ def read(self, n): def _buffer_bytes(self, n): to_read = n - for _ in xrange(self.MAX_READ_ATTEMPTS): + for _ in range(self.MAX_READ_ATTEMPTS): bytes_read = self.socket.recv(to_read) self.buffer += bytes_read to_read -= len(bytes_read) @@ -152,7 +155,7 @@ def rewind(self, places): log.debug("Reset buffer to pos %d" % self.pos) def reset(self): - self.buffer = "" + self.buffer = b"" self.pos = -1 # position of last byte read @property @@ -162,8 +165,8 @@ def buffer_length(self): class SocketRpcChannel(RpcChannel): - ERROR_BYTES = 18446744073709551615L - RPC_HEADER = "hrpc" + ERROR_BYTES = 18446744073709551615 + RPC_HEADER = b"hrpc" RPC_SERVICE_CLASS = 0x00 AUTH_PROTOCOL_NONE = 0x00 AUTH_PROTOCOL_SASL = 0xDF @@ -181,7 +184,7 @@ def __init__(self, host, port, version, effective_user=None, use_sasl=False, hdf self.sock = None self.call_id = -3 # First time (when the connection context is sent, the call_id should be -3, otherwise start with 0 and increment) self.version = version - self.client_id = str(uuid.uuid4()) + self.client_id = str(uuid.uuid4()).encode("utf-8") self.use_sasl = use_sasl self.hdfs_namenode_principal = hdfs_namenode_principal if self.use_sasl: @@ -490,12 +493,12 @@ def _read_bytes(self, n, depth=0): if depth > self.MAX_READ_ATTEMPTS: raise TransientException("Tried to read %d more bytes, but failed after %d attempts" % (n, self.MAX_READ_ATTEMPTS)) - bytes = self.sock.recv(n) - if len(bytes) < n: - left = n - len(bytes) + _bytes = self.sock.recv(n) + if len(_bytes) < n: + left = n - len(_bytes) depth += 1 - bytes += self._read_bytes(left, depth) - return bytes + _bytes += self._read_bytes(left, depth) + return _bytes def write(self, data): if log.getEffectiveLevel() == logging.DEBUG: @@ -613,7 +616,7 @@ def readBlock(self, length, pool_id, block_id, generation_stamp, offset, block_t # Collect checksums if check_crc and checksum_type != self.CHECKSUM_NULL: checksums = [] - for _ in xrange(0, chunks_per_packet): + for _ in range(0, chunks_per_packet): checksum = self._read_bytes(checksum_len) checksum = struct.unpack("!I", checksum)[0] checksums.append(checksum) @@ -627,7 +630,7 @@ def readBlock(self, length, pool_id, block_id, generation_stamp, offset, block_t read_on_packet = 0 for i in range(loads_per_packet): - load = '' + load = b'' for j in range(chunks_per_load): log.debug("Reading chunk %s in load %s:", j, i) bytes_to_read = min(bytes_per_chunk, data_len - read_on_packet) diff --git a/snakebite/client.py b/snakebite/client.py index e5e44c4..5c6c9c8 100644 --- a/snakebite/client.py +++ b/snakebite/client.py @@ -13,6 +13,9 @@ # License for the specific language governing permissions and limitations under # the License. +# python 3 support +from __future__ import absolute_import, print_function, division + import snakebite.protobuf.ClientNamenodeProtocol_pb2 as client_proto import snakebite.glob as glob from snakebite.platformutils import get_current_username @@ -30,8 +33,13 @@ FatalException, TransientException) from snakebite.namenode import Namenode from snakebite.service import RpcService +from snakebite.compat import range, unicode, long + +try: + from Queue import PriorityQueue +except ImportError: + from queue import PriorityQueue -import Queue import zlib import bz2 import logging @@ -46,8 +54,6 @@ import re import sys -if sys.version_info[0] == 3: - long = int log = logging.getLogger(__name__) @@ -60,7 +66,7 @@ class Client(object): >>> from snakebite.client import Client >>> client = Client("localhost", 8020, use_trash=False) >>> for x in client.ls(['/']): - ... print x + ... print(x) .. warning:: @@ -787,7 +793,7 @@ def getmerge(self, path, dst, newline=False, check_crc=False): raise InvalidInputException("getmerge: no destination given") temporary_target = "%s._COPYING_" % dst - f = open(temporary_target, 'w') + f = open(temporary_target, 'wb') processor = lambda path, node, dst=dst, check_crc=check_crc: self._handle_getmerge(path, node, dst, check_crc) try: @@ -800,7 +806,7 @@ def getmerge(self, path, dst, newline=False, check_crc=False): os.remove(temporary_target) raise FatalException(load['error']) if newline and load['response']: - f.write("\n") + f.write(b"\n") yield {"path": dst, "response": '', "result": True, "error": load['error'], "source_path": path} finally: @@ -897,7 +903,7 @@ def tail(self, path, tail_length=1024, append=False): yield item def _handle_tail(self, path, node, tail_length, append): - data = '' + data = b'' for load in self._read_file(path, node, tail_only=True, check_crc=False, tail_length=tail_length): data += load # We read only the necessary packets but still @@ -964,7 +970,7 @@ def _handle_text(self, path, node, check_crc): if self._is_dir(node): raise DirectoryException("text: `%s': Is a directory" % path) - text = '' + text = b'' for load in self._read_file(path, node, False, check_crc): text += load @@ -1053,7 +1059,7 @@ def _is_zero_length(self, should_check, node): def _get_full_path(self, path, node): if node.path: - return posixpath.join(path, node.path) + return posixpath.join(path, node.path.decode("utf-8")) else: return path @@ -1123,7 +1129,7 @@ def _read_file(self, path, node, tail_only, check_crc, tail_length=1024): offset_in_block = max(0, lastblock.b.numBytes - tail_length) # Prioritize locations to read from - locations_queue = Queue.PriorityQueue() # Primitive queuing based on a node's past failure + locations_queue = PriorityQueue() # Primitive queuing based on a node's past failure for location in block.locs: if location.id.storageID in failed_nodes: locations_queue.put((1, location)) # Priority num, data @@ -1225,7 +1231,7 @@ def _find_items(self, paths, processor, include_toplevel=False, include_children # Recurse into directories if recurse and self._is_dir(node): # Construct the full path before processing - full_path = posixpath.join(path, node.path) + full_path = posixpath.join(path, node.path.decode("utf-8")) for item in self._find_items([full_path], processor, include_toplevel=False, @@ -1233,7 +1239,7 @@ def _find_items(self, paths, processor, include_toplevel=False, include_children recurse=recurse): yield item - def _get_dir_listing(self, path, start_after=''): + def _get_dir_listing(self, path, start_after=b''): request = client_proto.GetListingRequestProto() request.src = path request.startAfter = start_after @@ -1352,7 +1358,7 @@ class HAClient(Client): >>> n2 = Namenode("namenode2.mydomain", 8020) >>> client = HAClient([n1, n2], use_trash=True) >>> for x in client.ls(['/']): - ... print x + ... print(x) .. note:: Different Hadoop distributions use different protocol versions. Snakebite defaults to 9, but this can be set by passing @@ -1392,7 +1398,7 @@ def __init__(self, namenodes, use_trash=False, effective_user=None, use_sasl=Fal # is not. raise InvalidInputException("List of namenodes is empty - couldn't create the client") self.namenode = self._switch_namenode(namenodes) - self.namenode.next() + next(self.namenode) def _switch_namenode(self, namenodes): for namenode in namenodes: @@ -1419,7 +1425,7 @@ def __handle_request_error(self, exception): else: # There's a valid NN in active state, but there's still request error - raise raise - self.namenode.next() + next(self.namenode) def __handle_socket_error(self, exception): log.debug("Request failed with %s" % exception) @@ -1431,7 +1437,7 @@ def __handle_socket_error(self, exception): pass else: raise - self.namenode.next() + next(self.namenode) @staticmethod def _ha_return_method(func): @@ -1454,7 +1460,7 @@ def wrapped(self, *args, **kw): try: results = func(self, *args, **kw) while(True): # yield all results - yield results.next() + yield next(results) except RequestError as e: self.__handle_request_error(e) except socket.error as e: @@ -1477,7 +1483,7 @@ class AutoConfigClient(HAClient): >>> from snakebite.client import AutoConfigClient >>> client = AutoConfigClient() >>> for x in client.ls(['/']): - ... print x + ... print(x) .. note:: Different Hadoop distributions use different protocol versions. Snakebite defaults to 9, but this can be set by passing diff --git a/snakebite/commandlineparser.py b/snakebite/commandlineparser.py index de8285e..8763d64 100644 --- a/snakebite/commandlineparser.py +++ b/snakebite/commandlineparser.py @@ -13,12 +13,18 @@ # License for the specific language governing permissions and limitations under # the License. +# python 3 support +from __future__ import absolute_import, print_function, division + import argparse import errno import sys import os import json -from urlparse import urlparse +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse from snakebite.client import HAClient from snakebite.errors import FileNotFoundException @@ -35,35 +41,42 @@ from snakebite.version import version from snakebite.namenode import Namenode from snakebite.platformutils import get_current_username +from snakebite.compat import py_2 + + +try: + stdout = sys.stdout.buffer +except AttributeError: + stdout = sys.stdout def print_error_exit(msg, fd=sys.stderr): - print >> fd, "Error: %s" % msg + print("Error: %s" % msg, file=fd) sys.exit(-1) def print_info(msg, fd=sys.stderr): - print >> fd, "Info: %s" % msg + print("Info: %s" % msg, file=fd) def exitError(exc_info): exc_type, exc_value, exc_traceback = exc_info if isinstance( exc_value, (FileNotFoundException, DirectoryException, FileException), ): - print str(exc_value) + print(str(exc_value)) elif isinstance(exc_value, RequestError): - print "Request error: %s" % str(exc_value) + print("Request error: %s" % str(exc_value)) else: - raise exc_type, exc_value, exc_traceback + raise exc_type(exc_value).with_traceback(exc_traceback) sys.exit(-1) def command(args="", descr="", allowed_opts="", visible=True): def wrap(f): - Commands.methods[f.func_name] = {"method": f, - "args": args, - "descr": descr, - "allowed_opts": allowed_opts, - "visible": visible} + Commands.methods[f.__name__] = {"method": f, + "args": args, + "descr": descr, + "allowed_opts": allowed_opts, + "visible": visible} return wrap @@ -82,7 +95,7 @@ def __init__(self, message, error_message, prog, stdout=None, stderr=None, error class Parser(argparse.ArgumentParser): def print_help(self): - print ''.join([self.usage, self.epilog]) + print(''.join([self.usage, self.epilog])) def error(self, message): # Override error message to show custom help. raise ArgumentParserError("SystemExit", message, self.prog) @@ -171,9 +184,9 @@ class CommandLineParser(object): def __init__(self): usage = "snakebite [general options] cmd [arguments]" epilog = "\ngeneral options:\n" - epilog += "\n".join(sorted([" %-30s %s" % ("%s %s" % (v['short'], v['long']), v['help']) for k, v in self.GENERIC_OPTS.iteritems()])) + epilog += "\n".join(sorted([" %-30s %s" % ("%s %s" % (v['short'], v['long']), v['help']) for k, v in self.GENERIC_OPTS.items()])) epilog += "\n\ncommands:\n" - epilog += "\n".join(sorted([" %-30s %s" % ("%s %s" % (k, v['args']), v['descr']) for k, v in Commands.methods.iteritems() if v['visible']])) + epilog += "\n".join(sorted([" %-30s %s" % ("%s %s" % (k, v['args']), v['descr']) for k, v in Commands.methods.items() if v['visible']])) epilog += "\n\nto see command-specific options use: snakebite [cmd] --help" self.parser = Parser(usage=usage, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter, add_help=False) @@ -185,7 +198,7 @@ def __init__(self): def _build_parent_parser(self): #general options - for opt_name, opt_data in self.GENERIC_OPTS.iteritems(): + for opt_name, opt_data in self.GENERIC_OPTS.items(): if 'action' in opt_data: self.parser.add_argument(opt_data['short'], opt_data['long'], help=opt_data['help'], action=opt_data['action']) else: @@ -199,7 +212,7 @@ def _add_subparsers(self): #sub-options arg_parsers = {} - for opt_name, opt_data in self.SUB_OPTS.iteritems(): + for opt_name, opt_data in self.SUB_OPTS.items(): arg_parsers[opt_name] = argparse.ArgumentParser(add_help=False) arg_parsers[opt_name].add_argument(opt_data['short'], opt_data['long'], help=opt_data['help'], action=opt_data['action']) @@ -237,7 +250,7 @@ def _add_subparsers(self): type=int) subparsers = self.parser.add_subparsers() - for cmd_name, cmd_info in Commands.methods.iteritems(): + for cmd_name, cmd_info in Commands.methods.items(): parents = [arg_parsers[opt] for opt in cmd_info['allowed_opts'] if opt in arg_parsers] parents += [subcommand_help_parser] if 'req_args' in cmd_info and not cmd_info['req_args'] is None: @@ -295,22 +308,22 @@ def read_config(self): if len(self.namenodes): return else: - print "No ~/.snakebiterc found, no HADOOP_HOME set and no -n and -p provided" - print "Tried to find core-site.xml in:" + print("No ~/.snakebiterc found, no HADOOP_HOME set and no -n and -p provided") + print("Tried to find core-site.xml in:") for core_conf_path in HDFSConfig.core_try_paths: - print " - %s" % core_conf_path - print "Tried to find hdfs-site.xml in:" + print(" - %s" % core_conf_path) + print("Tried to find hdfs-site.xml in:") for hdfs_conf_path in HDFSConfig.hdfs_try_paths: - print " - %s" % hdfs_conf_path - print "\nYou can manually create ~/.snakebiterc with the following content:" - print '{' - print ' "config_version": 2,' - print ' "use_trash": true,' - print ' "namenodes": [' - print ' {"host": "namenode-ha1", "port": %d, "version": %d},' % (Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION) - print ' {"host": "namenode-ha2", "port": %d, "version": %d}' % (Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION) - print ' ]' - print '}' + print(" - %s" % hdfs_conf_path) + print("\nYou can manually create ~/.snakebiterc with the following content:") + print('{') + print(' "config_version": 2,') + print(' "use_trash": true,') + print(' "namenodes": [') + print(' {"host": "namenode-ha1", "port": %d, "version": %d},' % (Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION)) + print(' {"host": "namenode-ha2", "port": %d, "version": %d}' % (Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION)) + print(' ]') + print('}') sys.exit(1) @@ -387,11 +400,11 @@ def _read_config_cl(self): ports.append(parse_result.port) # remove duplicates and None from (hosts + self.args.namenode) - hosts = filter(lambda x: x != None, set(hosts + [self.args.namenode])) + hosts = list(filter(lambda x: x != None, set(hosts + [self.args.namenode]))) if len(hosts) > 1: print_error_exit('Conficiting namenode hosts in commandline arguments, hosts: %s' % str(hosts)) - ports = filter(lambda x: x != None, set(ports + [self.args.port])) + ports = list(filter(lambda x: x != None, set(ports + [self.args.port]))) if len(ports) > 1: print_error_exit('Conflicting namenode ports in commandline arguments, ports: %s' % str(ports)) @@ -418,9 +431,9 @@ def parse(self, non_cli_input=None): # Allow input for testing purposes try: args = self.parser.parse_args(non_cli_input) - except ArgumentParserError, error: + except ArgumentParserError as error: if "-h" in sys.argv or "--help" in sys.argv: # non cli input? - commands = [cmd for (cmd, description) in Commands.methods.iteritems() if description['visible'] is True] + commands = [cmd for (cmd, description) in Commands.methods.items() if description['visible'] is True] command = error.prog.split()[-1] if command in commands: self.usage_helper(command) @@ -428,7 +441,7 @@ def parse(self, non_cli_input=None): # Allow input for testing purposes self.parser.print_help() self.parser.exit(2) elif "-v" in sys.argv or "--ver" in sys.argv: - print version() + print(version()) self.parser.exit(0) else: self.parser.print_usage(sys.stderr) @@ -463,17 +476,17 @@ def execute(self): def command(args="", descr="", allowed_opts="", visible=True, req_args=None): def wrap(f): - Commands.methods[f.func_name] = {"method": f, - "args": args, - "descr": descr, - "allowed_opts": allowed_opts, - "visible": visible, - "req_args": req_args} + Commands.methods[f.__name__] = {"method": f, + "args": args, + "descr": descr, + "allowed_opts": allowed_opts, + "visible": visible, + "req_args": req_args} return wrap @command(visible=False) def commands(self): - print "\n".join(sorted([k for k, v in Commands.methods.iteritems() if v['visible']])) + print("\n".join(sorted([k for k, v in Commands.methods.items() if v['visible']]))) @command(args="[path]", descr="Used for command line completion", visible=False, req_args=['[dirs]']) def complete(self): @@ -483,14 +496,14 @@ def complete(self): self.args.human = False try: for line in self._listing(): - print line.replace(" ", "\\\\ ") + print(line.replace(" ", "\\\\ ")) except FileNotFoundException: pass @command(args="[paths]", descr="list a path", allowed_opts=["d", "R", "s", "h"], req_args=['[dirs]']) def ls(self): for line in self._listing(): - print line + print(line) def _listing(self): # Mimicking hadoop client behaviour @@ -517,13 +530,13 @@ def _listing(self): def mkdir(self): creations = self.client.mkdir(self.args.dir) for line in format_results(creations, json_output=self.args.json): - print line + print(line) @command(args="[paths]", descr="create directories and their parents", req_args=['dir [dirs]']) def mkdirp(self): creations = self.client.mkdir(self.args.dir, create_parent=True) for line in format_results(creations, json_output=self.args.json): - print line + print(line) @command(args=" [paths]", descr="change owner", allowed_opts=["R"], req_args=['arg', 'dir [dirs]']) def chown(self): @@ -531,7 +544,7 @@ def chown(self): try: mods = self.client.chown(self.args.dir, owner, recurse=self.args.recurse) for line in format_results(mods, json_output=self.args.json): - print line + print(line) except FileNotFoundException: exitError(sys.exc_info()) @@ -540,28 +553,28 @@ def chmod(self): mode = int(str(self.args.single_int_arg), 8) mods = self.client.chmod(self.args.dir, mode, recurse=self.args.recurse) for line in format_results(mods, json_output=self.args.json): - print line + print(line) @command(args=" [paths]", descr="change group", allowed_opts=["R"], req_args=['arg', 'dir [dirs]']) def chgrp(self): grp = self.args.single_arg mods = self.client.chgrp(self.args.dir, grp, recurse=self.args.recurse) for line in format_results(mods, json_output=self.args.json): - print line + print(line) @command(args="[paths]", descr="display stats for paths", allowed_opts=['h'], req_args=['[dirs]']) def count(self): counts = self.client.count(self.args.dir) for line in format_counts(counts, json_output=self.args.json, human_readable=self.args.human): - print line + print(line) @command(args="", descr="display fs stats", allowed_opts=['h']) def df(self): result = self.client.df() for line in format_fs_stats(result, json_output=self.args.json, human_readable=self.args.human): - print line + print(line) @command(args="[paths]", descr="display disk usage statistics", allowed_opts=["s", "h"], req_args=['[dirs]']) def du(self): @@ -573,7 +586,7 @@ def du(self): include_toplevel = False result = self.client.du(self.args.dir, include_toplevel=include_toplevel, include_children=include_children) for line in format_du(result, json_output=self.args.json, human_readable=self.args.human): - print line + print(line) @command(args="[paths] dst", descr="move paths to destination", req_args=['dir [dirs]', 'arg']) def mv(self): @@ -581,36 +594,36 @@ def mv(self): dst = self.args.single_arg result = self.client.rename(paths, dst) for line in format_results(result, json_output=self.args.json): - print line + print(line) @command(args="[paths]", descr="remove paths", allowed_opts=["R", "S", "T"], req_args=['dir [dirs]']) def rm(self): result = self.client.delete(self.args.dir, recurse=self.args.recurse) for line in format_results(result, json_output=self.args.json): - print line + print(line) @command(args="[paths]", descr="creates a file of zero length", req_args=['dir [dirs]']) def touchz(self): result = self.client.touchz(self.args.dir) for line in format_results(result, json_output=self.args.json): - print line + print(line) @command(args="", descr="show server information") def serverdefaults(self): - print self.client.serverdefaults() + print(self.client.serverdefaults()) @command(args="[dirs]", descr="delete a directory", req_args=['dir [dirs]']) def rmdir(self): result = self.client.rmdir(self.args.dir) for line in format_results(result, json_output=self.args.json): - print line + print(line) @command(args=" [paths]", descr="set replication factor", allowed_opts=['R'], req_args=['(int) arg', 'dir [dirs]']) def setrep(self): rep_factor = int(self.args.single_int_arg) result = self.client.setrep(self.args.dir, rep_factor, recurse=self.args.recurse) for line in format_results(result, json_output=self.args.json): - print line + print(line) @command(args="", descr="show cmd usage", req_args=['[args]']) def usage(self): @@ -636,18 +649,18 @@ def usage_helper(self, command): if args: cmd_args.append(args) - print "usage: snakebite [general options] %s %s" % (command, " ".join(cmd_args)) + print("usage: snakebite [general options] %s %s" % (command, " ".join(cmd_args))) general_opts = "\ngeneral options:\n" - general_opts += "\n".join(sorted([" %-30s %s" % ("%s %s" % (v['short'], v['long']), v['help']) for k, v in self.GENERIC_OPTS.iteritems()])) - print general_opts + general_opts += "\n".join(sorted([" %-30s %s" % ("%s %s" % (v['short'], v['long']), v['help']) for k, v in self.GENERIC_OPTS.items()])) + print(general_opts) if allowed_opts: - print cmd_descriptions + print(cmd_descriptions) @command(args="[paths]", descr="stat information", req_args=['dir [dirs]']) def stat(self): - print format_stat(self.client.stat(self.args.dir), json_output=self.args.json) + print(format_stat(self.client.stat(self.args.dir), json_output=self.args.json)) @command(args="path", descr="test a path", allowed_opts=['d', 'z', 'e'], req_args=['arg']) def test(self): @@ -667,7 +680,7 @@ def test(self): def cat(self): for file_to_read in self.client.cat(self.args.dir, check_crc=self.args.checkcrc): for load in file_to_read: - sys.stdout.write(load) + stdout.write(load) @command(args="path dst", descr="copy local file reference to destination", req_args=['dir [dirs]', 'arg'], visible=False) def copyFromLocal(self): @@ -675,7 +688,7 @@ def copyFromLocal(self): dst = self.args.single_arg result = self.client.copyFromLocal(src, dst) for line in format_results(result, json_output=self.args.json): - print line + print(line) @command(args="[paths] dst", descr="copy paths to local file system destination", allowed_opts=['checkcrc'], req_args=['dir [dirs]', 'arg']) def copyToLocal(self): @@ -683,7 +696,7 @@ def copyToLocal(self): dst = self.args.single_arg result = self.client.copyToLocal(paths, dst, check_crc=self.args.checkcrc) for line in format_results(result, json_output=self.args.json): - print line + print(line) @command(args="[paths] dst", descr="copy files from source to destination", allowed_opts=['checkcrc'], req_args=['dir [dirs]', 'arg'], visible=False) def cp(self): @@ -691,7 +704,7 @@ def cp(self): dst = self.args.single_arg result = self.client.cp(paths, dst, checkcrc=self.args.checkcrc) for line in format_results(result, json_output=self.args.json): - print line + print(line) @command(args="file dst", descr="copy files to local file system destination", allowed_opts=['checkcrc'], req_args=['dir [dirs]', 'arg']) def get(self): @@ -699,7 +712,7 @@ def get(self): dst = self.args.single_arg result = self.client.copyToLocal(paths, dst, check_crc=self.args.checkcrc) for line in format_results(result, json_output=self.args.json): - print line + print(line) @command(args="dir dst", descr="concatenates files in source dir into destination local file", allowed_opts=['nl'], req_args=['src dst']) def getmerge(self): @@ -707,7 +720,7 @@ def getmerge(self): dst = self.args.src_dst[1] result = self.client.getmerge(source, dst, newline=self.args.newline) for line in format_results(result, json_output=self.args.json): - print line + print(line) # @command(args="[paths] dst", descr="copy sources from local file system to destination", req_args=['dir [dirs]', 'arg']) # def put(self): @@ -722,11 +735,11 @@ def tail(self): path = self.args.single_arg result = self.client.tail(path, append=self.args.append) for line in result: - print line + print(line) @command(args="path [paths]", descr="output file in text format", allowed_opts=['checkcrc'], req_args=['dir [dirs]']) def text(self): paths = self.args.dir result = self.client.text(paths) for line in result: - print line + print(line) diff --git a/snakebite/compat.py b/snakebite/compat.py new file mode 100644 index 0000000..07b324c --- /dev/null +++ b/snakebite/compat.py @@ -0,0 +1,9 @@ +# python 3 support +from __future__ import absolute_import, print_function, division + +from sys import version_info +py_2 = version_info[0] < 3 + +long = py_2 and (long) or (int) +range = py_2 and (xrange) or (range) +unicode = py_2 and (unicode) or (str) diff --git a/snakebite/config.py b/snakebite/config.py index 9f4f498..0e09b7b 100644 --- a/snakebite/config.py +++ b/snakebite/config.py @@ -1,9 +1,15 @@ +# python 3 support +from __future__ import absolute_import, print_function, division + import os import logging import xml.etree.ElementTree as ET -from urlparse import urlparse +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse -from namenode import Namenode +from snakebite.namenode import Namenode log = logging.getLogger(__name__) diff --git a/snakebite/crc32c.py b/snakebite/crc32c.py index 3030f47..c968474 100644 --- a/snakebite/crc32c.py +++ b/snakebite/crc32c.py @@ -24,80 +24,83 @@ './pycrc.py --model=crc-32c --generate c --algorithm=table-driven' """ +# python 3 support +from __future__ import absolute_import, print_function, division + import array CRC_TABLE = ( - 0x00000000L, 0xf26b8303L, 0xe13b70f7L, 0x1350f3f4L, - 0xc79a971fL, 0x35f1141cL, 0x26a1e7e8L, 0xd4ca64ebL, - 0x8ad958cfL, 0x78b2dbccL, 0x6be22838L, 0x9989ab3bL, - 0x4d43cfd0L, 0xbf284cd3L, 0xac78bf27L, 0x5e133c24L, - 0x105ec76fL, 0xe235446cL, 0xf165b798L, 0x030e349bL, - 0xd7c45070L, 0x25afd373L, 0x36ff2087L, 0xc494a384L, - 0x9a879fa0L, 0x68ec1ca3L, 0x7bbcef57L, 0x89d76c54L, - 0x5d1d08bfL, 0xaf768bbcL, 0xbc267848L, 0x4e4dfb4bL, - 0x20bd8edeL, 0xd2d60dddL, 0xc186fe29L, 0x33ed7d2aL, - 0xe72719c1L, 0x154c9ac2L, 0x061c6936L, 0xf477ea35L, - 0xaa64d611L, 0x580f5512L, 0x4b5fa6e6L, 0xb93425e5L, - 0x6dfe410eL, 0x9f95c20dL, 0x8cc531f9L, 0x7eaeb2faL, - 0x30e349b1L, 0xc288cab2L, 0xd1d83946L, 0x23b3ba45L, - 0xf779deaeL, 0x05125dadL, 0x1642ae59L, 0xe4292d5aL, - 0xba3a117eL, 0x4851927dL, 0x5b016189L, 0xa96ae28aL, - 0x7da08661L, 0x8fcb0562L, 0x9c9bf696L, 0x6ef07595L, - 0x417b1dbcL, 0xb3109ebfL, 0xa0406d4bL, 0x522bee48L, - 0x86e18aa3L, 0x748a09a0L, 0x67dafa54L, 0x95b17957L, - 0xcba24573L, 0x39c9c670L, 0x2a993584L, 0xd8f2b687L, - 0x0c38d26cL, 0xfe53516fL, 0xed03a29bL, 0x1f682198L, - 0x5125dad3L, 0xa34e59d0L, 0xb01eaa24L, 0x42752927L, - 0x96bf4dccL, 0x64d4cecfL, 0x77843d3bL, 0x85efbe38L, - 0xdbfc821cL, 0x2997011fL, 0x3ac7f2ebL, 0xc8ac71e8L, - 0x1c661503L, 0xee0d9600L, 0xfd5d65f4L, 0x0f36e6f7L, - 0x61c69362L, 0x93ad1061L, 0x80fde395L, 0x72966096L, - 0xa65c047dL, 0x5437877eL, 0x4767748aL, 0xb50cf789L, - 0xeb1fcbadL, 0x197448aeL, 0x0a24bb5aL, 0xf84f3859L, - 0x2c855cb2L, 0xdeeedfb1L, 0xcdbe2c45L, 0x3fd5af46L, - 0x7198540dL, 0x83f3d70eL, 0x90a324faL, 0x62c8a7f9L, - 0xb602c312L, 0x44694011L, 0x5739b3e5L, 0xa55230e6L, - 0xfb410cc2L, 0x092a8fc1L, 0x1a7a7c35L, 0xe811ff36L, - 0x3cdb9bddL, 0xceb018deL, 0xdde0eb2aL, 0x2f8b6829L, - 0x82f63b78L, 0x709db87bL, 0x63cd4b8fL, 0x91a6c88cL, - 0x456cac67L, 0xb7072f64L, 0xa457dc90L, 0x563c5f93L, - 0x082f63b7L, 0xfa44e0b4L, 0xe9141340L, 0x1b7f9043L, - 0xcfb5f4a8L, 0x3dde77abL, 0x2e8e845fL, 0xdce5075cL, - 0x92a8fc17L, 0x60c37f14L, 0x73938ce0L, 0x81f80fe3L, - 0x55326b08L, 0xa759e80bL, 0xb4091bffL, 0x466298fcL, - 0x1871a4d8L, 0xea1a27dbL, 0xf94ad42fL, 0x0b21572cL, - 0xdfeb33c7L, 0x2d80b0c4L, 0x3ed04330L, 0xccbbc033L, - 0xa24bb5a6L, 0x502036a5L, 0x4370c551L, 0xb11b4652L, - 0x65d122b9L, 0x97baa1baL, 0x84ea524eL, 0x7681d14dL, - 0x2892ed69L, 0xdaf96e6aL, 0xc9a99d9eL, 0x3bc21e9dL, - 0xef087a76L, 0x1d63f975L, 0x0e330a81L, 0xfc588982L, - 0xb21572c9L, 0x407ef1caL, 0x532e023eL, 0xa145813dL, - 0x758fe5d6L, 0x87e466d5L, 0x94b49521L, 0x66df1622L, - 0x38cc2a06L, 0xcaa7a905L, 0xd9f75af1L, 0x2b9cd9f2L, - 0xff56bd19L, 0x0d3d3e1aL, 0x1e6dcdeeL, 0xec064eedL, - 0xc38d26c4L, 0x31e6a5c7L, 0x22b65633L, 0xd0ddd530L, - 0x0417b1dbL, 0xf67c32d8L, 0xe52cc12cL, 0x1747422fL, - 0x49547e0bL, 0xbb3ffd08L, 0xa86f0efcL, 0x5a048dffL, - 0x8ecee914L, 0x7ca56a17L, 0x6ff599e3L, 0x9d9e1ae0L, - 0xd3d3e1abL, 0x21b862a8L, 0x32e8915cL, 0xc083125fL, - 0x144976b4L, 0xe622f5b7L, 0xf5720643L, 0x07198540L, - 0x590ab964L, 0xab613a67L, 0xb831c993L, 0x4a5a4a90L, - 0x9e902e7bL, 0x6cfbad78L, 0x7fab5e8cL, 0x8dc0dd8fL, - 0xe330a81aL, 0x115b2b19L, 0x020bd8edL, 0xf0605beeL, - 0x24aa3f05L, 0xd6c1bc06L, 0xc5914ff2L, 0x37faccf1L, - 0x69e9f0d5L, 0x9b8273d6L, 0x88d28022L, 0x7ab90321L, - 0xae7367caL, 0x5c18e4c9L, 0x4f48173dL, 0xbd23943eL, - 0xf36e6f75L, 0x0105ec76L, 0x12551f82L, 0xe03e9c81L, - 0x34f4f86aL, 0xc69f7b69L, 0xd5cf889dL, 0x27a40b9eL, - 0x79b737baL, 0x8bdcb4b9L, 0x988c474dL, 0x6ae7c44eL, - 0xbe2da0a5L, 0x4c4623a6L, 0x5f16d052L, 0xad7d5351L, + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, + 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, + 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, + 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, + 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, + 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, + 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, + 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, + 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, + 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, + 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, + 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, + 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, + 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, + 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, + 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, + 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, + 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, + 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, + 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, + 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, + 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, + 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, + 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, + 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, + 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, + 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, + 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, + 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, + 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, + 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, + 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, + 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, + 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, + 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, + 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, + 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, + 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, + 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, + 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351, ) CRC_INIT = 0 -_MASK = 0xFFFFFFFFL +_MASK = 0xFFFFFFFF def crc_update(crc, data): diff --git a/snakebite/errors.py b/snakebite/errors.py index 4925694..f69688e 100644 --- a/snakebite/errors.py +++ b/snakebite/errors.py @@ -13,6 +13,9 @@ # License for the specific language governing permissions and limitations under # the License. +# python 3 support +from __future__ import absolute_import, print_function, division + class SnakebiteException(Exception): """ diff --git a/snakebite/formatter.py b/snakebite/formatter.py index af9573f..df5345d 100644 --- a/snakebite/formatter.py +++ b/snakebite/formatter.py @@ -12,7 +12,9 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. -from __future__ import division + +# python 3 support +from __future__ import absolute_import, print_function, division import datetime import stat @@ -90,7 +92,7 @@ def format_listing(listing, json_output=False, human_readable=False, recursive=F last_dir = None try: while True: - node = listing.next() + node = next(listing) dir_name = os.path.dirname(node['path']) if dir_name != last_dir: if last_dir: @@ -204,7 +206,7 @@ def format_du(listing, json_output=False, human_readable=False): last_dir = None try: while True: - node = listing.next() + node = next(listing) dir_name = os.path.dirname(node['path']) if dir_name != last_dir: if last_dir: @@ -236,11 +238,11 @@ def _format_permission(decimal_permissions): def format_stat(results, json_output=False): ret = results # By default snakebite returns permissions in decimal. - if ret.has_key('permission'): + if 'permissions' in ret: ret['permission'] = _format_permission(ret['permission']) if json_output: return json.dumps(ret) - return "\n".join(["%-20s\t%s" % (k, v) for k, v in sorted(ret.iteritems())]) + return "\n".join(["%-20s\t%s" % (k, v) for k, v in sorted(ret.items())]) def format_bytes(bytes): diff --git a/snakebite/glob.py b/snakebite/glob.py index 2d7acfb..242bf9d 100644 --- a/snakebite/glob.py +++ b/snakebite/glob.py @@ -13,6 +13,9 @@ # License for the specific language governing permissions and limitations under # the License. +# python 3 support +from __future__ import absolute_import, print_function, division + import re import itertools diff --git a/snakebite/kerberos.py b/snakebite/kerberos.py index 81f8771..8d9071d 100644 --- a/snakebite/kerberos.py +++ b/snakebite/kerberos.py @@ -29,6 +29,9 @@ ''' +# python 3 support +from __future__ import absolute_import, print_function, division + import krbV class Kerberos: diff --git a/snakebite/logger.py b/snakebite/logger.py index 5cde16c..1fc44e9 100644 --- a/snakebite/logger.py +++ b/snakebite/logger.py @@ -32,6 +32,9 @@ May 2009 ''' +# python 3 support +from __future__ import absolute_import, print_function, division + # Standard library imports import logging diff --git a/snakebite/minicluster.py b/snakebite/minicluster.py index d2f23af..ceba313 100644 --- a/snakebite/minicluster.py +++ b/snakebite/minicluster.py @@ -12,11 +12,16 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. + +# python 3 support +from __future__ import absolute_import, print_function, division + import os import subprocess import select import re import datetime +from snakebite.compat import long class MiniCluster(object): diff --git a/snakebite/namenode.py b/snakebite/namenode.py index cf73bb0..4a9e5a7 100644 --- a/snakebite/namenode.py +++ b/snakebite/namenode.py @@ -13,6 +13,9 @@ # License for the specific language governing permissions and limitations under # the License. +# python 3 support +from __future__ import absolute_import, print_function, division + class Namenode(object): '''Namenode class - represents HDFS namenode''' DEFAULT_PORT = 8020 diff --git a/snakebite/platformutils.py b/snakebite/platformutils.py index fb9f3aa..6ae30a6 100644 --- a/snakebite/platformutils.py +++ b/snakebite/platformutils.py @@ -1,3 +1,6 @@ +# python 3 support +from __future__ import absolute_import, print_function, division + import os import platform @@ -10,4 +13,4 @@ def get_current_username(): if platform.system() != "Windows": return pwd.getpwuid(os.getuid())[0] else: - return getpass.getuser() \ No newline at end of file + return getpass.getuser() diff --git a/snakebite/protobuf/__init__.py b/snakebite/protobuf/__init__.py index 5c9a5ab..56dbcfa 100644 --- a/snakebite/protobuf/__init__.py +++ b/snakebite/protobuf/__init__.py @@ -12,3 +12,6 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. +import os +import sys +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) diff --git a/snakebite/rpc_sasl.py b/snakebite/rpc_sasl.py index dc22737..1f261b2 100644 --- a/snakebite/rpc_sasl.py +++ b/snakebite/rpc_sasl.py @@ -30,17 +30,19 @@ ''' +# python 3 support +from __future__ import absolute_import, print_function, division + import struct import sasl import re from snakebite.protobuf.RpcHeader_pb2 import RpcRequestHeaderProto, RpcResponseHeaderProto, RpcSaslProto from snakebite.config import HDFSConfig +from snakebite import logger import google.protobuf.internal.encoder as encoder -import logger - # Configure package logging log = logger.getLogger(__name__) diff --git a/snakebite/service.py b/snakebite/service.py index e0fc521..3307418 100644 --- a/snakebite/service.py +++ b/snakebite/service.py @@ -13,6 +13,9 @@ # License for the specific language governing permissions and limitations under # the License. +# python 3 support +from __future__ import absolute_import, print_function, division + from snakebite.channel import SocketRpcChannel import google.protobuf.service as service diff --git a/snakebite/version.py b/snakebite/version.py index 543efbd..3a92957 100644 --- a/snakebite/version.py +++ b/snakebite/version.py @@ -1,3 +1,7 @@ +# python 3 support +from __future__ import absolute_import, print_function, division + + VERSION = "2.8.2" diff --git a/test/cat_test.py b/test/cat_test.py index f88b35d..97e9689 100644 --- a/test/cat_test.py +++ b/test/cat_test.py @@ -13,14 +13,19 @@ # License for the specific language governing permissions and limitations under # the License. -from minicluster_testbase import MiniClusterTestBase +# python 3 support +from __future__ import absolute_import, print_function, division + import os +from snakebite.compat import range +from test.minicluster_testbase import MiniClusterTestBase + class CatTest(MiniClusterTestBase): def test_cat_file_on_1_block(self): # Size < 1 block - client_output = '' + client_output = b'' for file_to_read in self.client.cat(['/test3']): for data in file_to_read: client_output += data @@ -28,7 +33,7 @@ def test_cat_file_on_1_block(self): # Size < 1 block self.assertEqual(expected_output, client_output) def test_cat_file_on_1_block_checkcrc(self): # Size < 1 block - client_output = '' + client_output = b'' for file_to_read in self.client.cat(['/test3'], check_crc=True): for data in file_to_read: client_output += data @@ -39,7 +44,7 @@ def test_cat_file_on_2_blocks(self): # 1 < size < 2 blocks self._write_to_test_cluster('/test1', 10, '/temp_test', 4 * 1024 * 1024) # 6.77972 MB of test data, with 4MB block size gives 2 blocks - client_output = '' + client_output = b'' for file_to_read in self.client.cat(['/temp_test']): for data in file_to_read: client_output += data @@ -52,7 +57,7 @@ def test_cat_file_on_3_blocks(self): # 2 < size < 3 blocks # size of the test data will be 677,972 * 10 = 6.77972 MB, and with #block size 3 MB, it gives as 3 blocks - client_output = '' + client_output = b'' for file_to_read in self.client.cat(['/temp_test2']): for data in file_to_read: client_output += data @@ -64,7 +69,7 @@ def test_cat_file_on_exactly_1_block(self): # Size == 1 block # test3 is 1024 bytes, write it 1024 times to get 1MB of test data # set block size to 1MB to get exactly one block - client_output = '' + client_output = b'' for file_to_read in self.client.cat(['/temp_test3']): for data in file_to_read: client_output += data @@ -75,8 +80,8 @@ def _write_to_test_cluster(self, testfile, times, dst, block_size=134217728): testfiles_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "testfiles") f = open(''.join([testfiles_path, testfile])) p = self.cluster.put_subprocess('-', dst, block_size) - for _ in xrange(times): + for _ in range(times): f.seek(0) for line in f.readlines(): - print >> p.stdin, line + print(line, file=p.stdin) p.communicate() diff --git a/test/chgrp_test.py b/test/chgrp_test.py index b01daea..9af44b1 100644 --- a/test/chgrp_test.py +++ b/test/chgrp_test.py @@ -12,7 +12,12 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. -from minicluster_testbase import MiniClusterTestBase + +# python 3 support +from __future__ import absolute_import, print_function, division + +from test.minicluster_testbase import MiniClusterTestBase +from snakebite.compat import py_2 from snakebite.errors import FileNotFoundException from snakebite.errors import InvalidInputException @@ -38,8 +43,8 @@ def test_recursive(self): def test_unknown_file(self): result = self.client.chgrp(['/nonexistent'], 'myOnwer', recurse=True) - self.assertRaises(FileNotFoundException, result.next) + self.assertRaises(FileNotFoundException, result.next if py_2 else result.__next__) def test_invalid_input(self): result = self.client.chgrp('/doesnotexist', 'myOnwer') - self.assertRaises(InvalidInputException, result.next) + self.assertRaises(InvalidInputException, result.next if py_2 else result.__next__) diff --git a/test/chmod_test.py b/test/chmod_test.py index f9f1e60..65d99b3 100644 --- a/test/chmod_test.py +++ b/test/chmod_test.py @@ -12,7 +12,12 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. -from minicluster_testbase import MiniClusterTestBase + +# python 3 support +from __future__ import absolute_import, print_function, division + +from test.minicluster_testbase import MiniClusterTestBase +from snakebite.compat import py_2 from snakebite.errors import FileNotFoundException from snakebite.errors import InvalidInputException @@ -20,26 +25,26 @@ class ChmodTest(MiniClusterTestBase): def test_onepath(self): - list(self.client.chmod(['/dir1'], 0777)) + list(self.client.chmod(['/dir1'], 0o777)) client_output = list(self.client.ls(['/dir1'], include_toplevel=True, include_children=False)) self.assertEqual(client_output[0]["permission"], 511) def test_multipath(self): - list(self.client.chmod(['/dir1', '/zerofile'], 0700)) + list(self.client.chmod(['/dir1', '/zerofile'], 0o700)) client_output = self.client.ls(['/dir1', '/zerofile'], include_toplevel=True, include_children=False) for node in client_output: self.assertEqual(node["permission"], 448) def test_recursive(self): - list(self.client.chmod(['/'], 0770, recurse=True)) + list(self.client.chmod(['/'], 0o770, recurse=True)) expected_output = self.cluster.ls(["/"], ["-R"]) for node in expected_output: self.assertEqual(node["permission"], 504) def test_unknown_file(self): - result = self.client.chmod(['/nonexistent'], 0777, recurse=True) - self.assertRaises(FileNotFoundException, result.next) + result = self.client.chmod(['/nonexistent'], 0o777, recurse=True) + self.assertRaises(FileNotFoundException, result.next if py_2 else result.__next__) def test_invalid_input(self): - result = self.client.chmod('/stringpath', 777) - self.assertRaises(InvalidInputException, result.next) + result = self.client.chmod('/stringpath', 0o777) + self.assertRaises(InvalidInputException, result.next if py_2 else result.__next__) diff --git a/test/chown_test.py b/test/chown_test.py index 2a8f9a4..ec616f1 100644 --- a/test/chown_test.py +++ b/test/chown_test.py @@ -12,7 +12,12 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. -from minicluster_testbase import MiniClusterTestBase + +# python 3 support +from __future__ import absolute_import, print_function, division + +from test.minicluster_testbase import MiniClusterTestBase +from snakebite.compat import py_2 from snakebite.errors import FileNotFoundException from snakebite.errors import InvalidInputException @@ -38,7 +43,7 @@ def test_recursive(self): def test_unknown_file(self): result = self.client.chown(['/nonexistent'], 'myGroup', recurse=True) - self.assertRaises(FileNotFoundException, result.next) + self.assertRaises(FileNotFoundException, result.next if py_2 else result.__next__) def test_user_group(self): list(self.client.chown(['/dir1'], "myUser:myGroup")) @@ -56,4 +61,4 @@ def test_group(self): def test_invalid_input(self): result = self.client.chown('/stringpath', 'myGroup') - self.assertRaises(InvalidInputException, result.next) + self.assertRaises(InvalidInputException, result.next if py_2 else result.__next__) diff --git a/test/client_test.py b/test/client_test.py index 6c95d85..83f77d4 100644 --- a/test/client_test.py +++ b/test/client_test.py @@ -1,3 +1,6 @@ +# python 3 support +from __future__ import absolute_import, print_function, division + import unittest2 import inspect import errno @@ -10,6 +13,11 @@ from snakebite.errors import OutOfNNException, RequestError, InvalidInputException +class SocketError(socket.error): + """Provide socket.error encapsulation for Mock side effects""" + pass + + class ClientTest(unittest2.TestCase): original_hdfs_try_path = HDFSConfig.hdfs_try_paths original_core_try_path = HDFSConfig.core_try_paths @@ -21,7 +29,7 @@ def setUp(self): HDFSConfig.core_try_paths = self.original_core_try_path def test_ha_client_econnrefused_socket_error(self): - e = socket.error + e = SocketError e.errno = errno.ECONNREFUSED mocked_client_cat = Mock(side_effect=e) ha_client = HAClient([Namenode("foo"), Namenode("bar")]) @@ -30,7 +38,7 @@ def test_ha_client_econnrefused_socket_error(self): self.assertRaises(OutOfNNException, all, cat_result_gen) def test_ha_client_ehostunreach_socket_error(self): - e = socket.error + e = SocketError e.errno = errno.EHOSTUNREACH mocked_client_cat = Mock(side_effect=e) ha_client = HAClient([Namenode("foo"), Namenode("bar")]) diff --git a/test/commandlineparser_test.py b/test/commandlineparser_test.py index deac538..8533aa8 100644 --- a/test/commandlineparser_test.py +++ b/test/commandlineparser_test.py @@ -12,6 +12,10 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. + +# python 3 support +from __future__ import absolute_import, print_function, division + import unittest2 import os import json @@ -25,7 +29,8 @@ from snakebite.namenode import Namenode from snakebite.platformutils import get_current_username -from config_test import ConfigTest +from test.config_test import ConfigTest + class CommandLineParserTest(unittest2.TestCase): @@ -818,7 +823,7 @@ def test_read_config_snakebiterc_one_valid(self, exists_mock): self.parser.args = MockParseArgs() self.parser.read_config() self.assert_namenodes_spec("foobar", 54310, 9) - self.assertEquals(self.parser.args.usetrash, self.parser.configs['use_trash']) + self.assertEqual(self.parser.args.usetrash, self.parser.configs['use_trash']) @patch("os.path.exists") def test_read_config_snakebiterc_ha_valid(self, exists_mock): @@ -829,7 +834,7 @@ def test_read_config_snakebiterc_ha_valid(self, exists_mock): self.parser.read_config() self.assert_namenodes_spec("foobar", 54310, 9) self.assert_namenodes_spec("foobar2", 54310, 9) - self.assertEquals(self.parser.args.usetrash, self.parser.configs['use_trash']) + self.assertEqual(self.parser.args.usetrash, self.parser.configs['use_trash']) @patch("os.path.exists") def test_read_config_snakebiterc_invalid(self, exists_mock): @@ -852,7 +857,7 @@ def test_read_config_snakebiterc_noport_one_valid(self, exists_mock): self.parser.args = MockParseArgs() self.parser.read_config() self.assert_namenodes_spec("foobar", Namenode.DEFAULT_PORT, 11) - self.assertEquals(self.parser.args.usetrash, self.parser.configs['use_trash']) + self.assertEqual(self.parser.args.usetrash, self.parser.configs['use_trash']) @patch("os.path.exists") def test_read_config_snakebiterc_noport_ha_valid(self, exists_mock): @@ -863,7 +868,7 @@ def test_read_config_snakebiterc_noport_ha_valid(self, exists_mock): self.parser.read_config() self.assert_namenodes_spec("foobar", Namenode.DEFAULT_PORT, 100) self.assert_namenodes_spec("foobar2", Namenode.DEFAULT_PORT, 100) - self.assertEquals(self.parser.args.usetrash, self.parser.configs['use_trash']) + self.assertEqual(self.parser.args.usetrash, self.parser.configs['use_trash']) valid_snake_noport_nov_one_rc = {"namenode": "foobar"} @@ -878,7 +883,7 @@ def test_read_config_snakebiterc_noport_nov_one_valid(self, exists_mock): self.parser.args = MockParseArgs() self.parser.read_config() self.assert_namenodes_spec("foobar", Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION) - self.assertEquals(self.parser.args.usetrash, self.parser.configs['use_trash']) + self.assertEqual(self.parser.args.usetrash, self.parser.configs['use_trash']) @patch("os.path.exists") def test_read_config_snakebiterc_noport_nov_ha_valid(self, exists_mock): @@ -889,7 +894,7 @@ def test_read_config_snakebiterc_noport_nov_ha_valid(self, exists_mock): self.parser.read_config() self.assert_namenodes_spec("foobar", Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION) self.assert_namenodes_spec("foobar2", Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION) - self.assertEquals(self.parser.args.usetrash, self.parser.configs['use_trash']) + self.assertEqual(self.parser.args.usetrash, self.parser.configs['use_trash']) valid_snake_noport_mix_rc = [{"namenode": "foobar", "version": 100}, {"namenode": "foobar2", "port": 66}] @@ -903,7 +908,7 @@ def test_read_config_snakebiterc_noport_mix_valid(self, exists_mock): self.parser.read_config() self.assert_namenodes_spec("foobar", Namenode.DEFAULT_PORT, 100) self.assert_namenodes_spec("foobar2", 66, Namenode.DEFAULT_VERSION) - self.assertEquals(self.parser.args.usetrash, self.parser.configs['use_trash']) + self.assertEqual(self.parser.args.usetrash, self.parser.configs['use_trash']) valid_snake_one_rc_v2 = { "config_version": 2, @@ -1014,7 +1019,7 @@ def test_read_config_snakebiterc_user_valid_v2(self, exists_mock): self.parser.read_config() self.parser.setup_client() self.assertTrue(self.parser.args.usetrash) - self.assertEquals(self.parser.client.effective_user, "hdfs_user") + self.assertEqual(self.parser.client.effective_user, "hdfs_user") self.assert_namenodes_spec("foobar4", Namenode.DEFAULT_PORT, 100) self.assert_namenodes_spec("foobar5", 54310, Namenode.DEFAULT_VERSION) @@ -1035,7 +1040,7 @@ def test_cl_trash_setting_preserved_after_cl_config(self): self.parser.read_config() self.assert_namenode_spec("foobar", 50070) self.assert_namenodes_spec("foobar", 50070) - self.assertEquals(self.parser.args.skiptrash, True) + self.assertEqual(self.parser.args.skiptrash, True) def _revert_hdfs_try_paths(self): # Make sure HDFSConfig is in vanilla state diff --git a/test/config_test.py b/test/config_test.py index 8838bfe..debeb6e 100644 --- a/test/config_test.py +++ b/test/config_test.py @@ -24,7 +24,7 @@ def get_config_path(config_name): def _verify_hdfs_settings(self, config): namenodes = config['namenodes'] - self.assertEquals(len(namenodes), 2) + self.assertEqual(len(namenodes), 2) # assert first NN self.assertEqual('namenode1.mydomain', namenodes[0]['namenode']) self.assertEqual(8888, namenodes[0]['port']) @@ -34,7 +34,7 @@ def _verify_hdfs_settings(self, config): def _verify_hdfs_noport_settings(self, config): namenodes = config['namenodes'] - self.assertEquals(len(namenodes), 2) + self.assertEqual(len(namenodes), 2) # assert first NN self.assertEqual('namenode1.mydomain', namenodes[0]['namenode']) self.assertEqual(8020, namenodes[0]['port']) @@ -51,17 +51,17 @@ def test_read_core_config_ha(self): core_site_path = self.get_config_path('ha-core-site.xml') config = HDFSConfig.read_core_config(core_site_path) namenodes = config['namenodes'] - self.assertEquals(len(namenodes), 1) - self.assertEquals('testha', namenodes[0]['namenode']) - self.assertEquals(8020, namenodes[0]['port']) + self.assertEqual(len(namenodes), 1) + self.assertEqual('testha', namenodes[0]['namenode']) + self.assertEqual(8020, namenodes[0]['port']) def test_read_core_config_emr(self): core_site_path = self.get_config_path('emr-core-site.xml') config = HDFSConfig.read_core_config(core_site_path) namenodes = config['namenodes'] - self.assertEquals(len(namenodes), 1) - self.assertEquals('testha', namenodes[0]['namenode']) - self.assertEquals(8020, namenodes[0]['port']) + self.assertEqual(len(namenodes), 1) + self.assertEqual('testha', namenodes[0]['namenode']) + self.assertEqual(8020, namenodes[0]['port']) @patch('os.environ.get') def test_read_config_ha_with_ports(self, environ_get): @@ -80,9 +80,9 @@ def test_read_config_non_ha_with_ports(self, environ_get): config = HDFSConfig.get_external_config() namenodes = config['namenodes'] - self.assertEquals(len(namenodes), 1) - self.assertEquals(namenodes[0]['namenode'], 'testhost.net') - self.assertEquals(namenodes[0]['port'], 8888) + self.assertEqual(len(namenodes), 1) + self.assertEqual(namenodes[0]['namenode'], 'testhost.net') + self.assertEqual(namenodes[0]['port'], 8888) self.assertFalse(config['use_trash']) @patch('os.environ.get') diff --git a/test/copytolocal_test.py b/test/copytolocal_test.py index 62e7dca..37bd91f 100644 --- a/test/copytolocal_test.py +++ b/test/copytolocal_test.py @@ -12,10 +12,14 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. + +# python 3 support +from __future__ import absolute_import, print_function, division + import os import shutil -from minicluster_testbase import MiniClusterTestBase +from test.minicluster_testbase import MiniClusterTestBase TESTFILES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "testfiles/temp_testfiles") @@ -32,7 +36,7 @@ def test_copyToLocal_file_content(self): for result in self.client.copyToLocal(['/test3'], target_dir): self.assertEqual(result['path'], target_dir) - client_content = self._read_file(target_dir) + client_content = self._read_file(target_dir).encode("utf-8") self.assertEqual(client_content, expected_content) self.assertEqual(result['result'], True) diff --git a/test/count_test.py b/test/count_test.py index 2c343a0..6520192 100644 --- a/test/count_test.py +++ b/test/count_test.py @@ -12,9 +12,14 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. + +# python 3 support +from __future__ import absolute_import, print_function, division + +from snakebite.compat import py_2 from snakebite.errors import FileNotFoundException from snakebite.errors import InvalidInputException -from minicluster_testbase import MiniClusterTestBase +from test.minicluster_testbase import MiniClusterTestBase class CountTest(MiniClusterTestBase): @@ -38,8 +43,8 @@ def test_count_multi(self): def test_unknown_file(self): result = self.client.count(['/doesnotexist']) - self.assertRaises(FileNotFoundException, result.next) + self.assertRaises(FileNotFoundException, result.next if py_2 else result.__next__) def test_invalid_input(self): result = self.client.count('/stringpath') - self.assertRaises(InvalidInputException, result.next) + self.assertRaises(InvalidInputException, result.next if py_2 else result.__next__) diff --git a/test/delete_test.py b/test/delete_test.py index 237fcf3..5190455 100644 --- a/test/delete_test.py +++ b/test/delete_test.py @@ -12,10 +12,15 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. + +# python 3 support +from __future__ import absolute_import, print_function, division + +from snakebite.compat import py_2 from snakebite.errors import FileNotFoundException from snakebite.errors import InvalidInputException from snakebite.platformutils import get_current_username -from minicluster_testbase import MiniClusterTestBase +from test.minicluster_testbase import MiniClusterTestBase import os import re @@ -37,11 +42,11 @@ def test_delete_multi(self): def test_unknown_file(self): result = self.client.delete(['/doesnotexist']) - self.assertRaises(FileNotFoundException, result.next) + self.assertRaises(FileNotFoundException, result.next if py_2 else result.__next__) def test_invalid_input(self): result = self.client.delete('/stringpath') - self.assertRaises(InvalidInputException, result.next) + self.assertRaises(InvalidInputException, result.next if py_2 else result.__next__) def test_recurse(self): list(self.client.delete(['/foo'], recurse=True)) @@ -104,11 +109,11 @@ def test_delete_multi(self): def test_unknown_file(self): result = self.client.delete(['/doesnotexist']) - self.assertRaises(FileNotFoundException, result.next) + self.assertRaises(FileNotFoundException, result.next if py_2 else result.__next__) def test_invalid_input(self): result = self.client.delete('/stringpath') - self.assertRaises(InvalidInputException, result.next) + self.assertRaises(InvalidInputException, result.next if py_2 else result.__next__) def test_recurse(self): location_under_test = '/foo' diff --git a/test/df_test.py b/test/df_test.py index 627c225..ff5b480 100644 --- a/test/df_test.py +++ b/test/df_test.py @@ -12,11 +12,15 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. + +# python 3 support +from __future__ import absolute_import, print_function, division + import re import unittest2 -from minicluster_testbase import MiniClusterTestBase - +from test.minicluster_testbase import MiniClusterTestBase +from snakebite.compat import long, py_2 from snakebite.formatter import format_fs_stats class DfTest(MiniClusterTestBase): @@ -51,32 +55,37 @@ class DfFormatTest(unittest2.TestCase): def test_middle(self): fake = StatsMock(100, 50, 50, 0, 0, 0, "foobar.com") - output = format_fs_stats(fake).next().split('\n') + fs_stats = format_fs_stats(fake) + output = (fs_stats.next() if py_2 else next(fs_stats)).split('\n') stats = output[1].split() self.assertEqual(stats[4], "50.00%") self.assertEqual(stats[0], "foobar.com") def test_frag(self): fake = StatsMock(312432, 23423, 289009, 0, 0, 0, "foobar.com") - output = format_fs_stats(fake).next().split('\n') + fs_stats = format_fs_stats(fake) + output = (fs_stats.next() if py_2 else next(fs_stats)).split('\n') stats = output[1].split() self.assertEqual(stats[4], "7.50%") def test_zero_size(self): fake = StatsMock(0, 0, 0, 0, 0, 0, "foobar.com") - output = format_fs_stats(fake).next().split('\n') + fs_stats = format_fs_stats(fake) + output = (fs_stats.next() if py_2 else next(fs_stats)).split('\n') stats = output[1].split() self.assertEqual(stats[4], "0.00%") def test_corrupted_zero_size(self): fake = StatsMock(0, 50, 50, 0, 0, 0, "foobar.com") - output = format_fs_stats(fake).next().split('\n') + fs_stats = format_fs_stats(fake) + output = (fs_stats.next() if py_2 else next(fs_stats)).split('\n') stats = output[1].split() self.assertEqual(stats[4], "0.00%") def test_full_size(self): fake = StatsMock(50, 50, 0, 0, 0, 0, "foobar.com") - output = format_fs_stats(fake).next().split('\n') + fs_stats = format_fs_stats(fake) + output = (fs_stats.next() if py_2 else next(fs_stats)).split('\n') stats = output[1].split() self.assertEqual(stats[4], "100.00%") diff --git a/test/du_test.py b/test/du_test.py index 9794f7e..705f70f 100644 --- a/test/du_test.py +++ b/test/du_test.py @@ -12,10 +12,15 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. + +# python 3 support +from __future__ import absolute_import, print_function, division + +from snakebite.compat import py_2 from snakebite.errors import FileNotFoundException from snakebite.errors import InvalidInputException -from minicluster_testbase import MiniClusterTestBase -from util import assertDu +from test.minicluster_testbase import MiniClusterTestBase +from test.util import assertDu class DfTest(MiniClusterTestBase): @@ -38,8 +43,8 @@ def test_toplevel(self): def test_unknown_file(self): result = self.client.du(['/nonexistent']) - self.assertRaises(FileNotFoundException, result.next) + self.assertRaises(FileNotFoundException, result.next if py_2 else result.__next__) def test_invalid_input(self): result = self.client.du('/stringpath') - self.assertRaises(InvalidInputException, result.next) + self.assertRaises(InvalidInputException, result.next if py_2 else result.__next__) diff --git a/test/effective_user_test.py b/test/effective_user_test.py index a2de59e..f678502 100644 --- a/test/effective_user_test.py +++ b/test/effective_user_test.py @@ -1,4 +1,7 @@ -from minicluster_testbase import MiniClusterTestBase +# python 3 support +from __future__ import absolute_import, print_function, division + +from test.minicluster_testbase import MiniClusterTestBase from snakebite.client import Client import os @@ -16,14 +19,14 @@ def setUp(self): effective_user='__foobar') def test_touch(self): - print tuple(self.custom_client.touchz([self.VALID_FILE])) + print(tuple(self.custom_client.touchz([self.VALID_FILE]))) try: tuple(self.custom_foobar_client.touchz([self.INVALID_FILE])) - except Exception, e: - self.assertTrue(e.message.startswith(self.ERR_MSG_TOUCH)) + except Exception as e: + self.assertTrue(e.args[0].startswith(self.ERR_MSG_TOUCH)) self.custom_client.stat([self.VALID_FILE]) try: self.custom_client.stat([self.INVALID_FILE]) - except Exception, e: - self.assertEquals(e.message, self.ERR_MSG_STAT) + except Exception as e: + self.assertEquals(e.args[0], self.ERR_MSG_STAT) diff --git a/test/getmerge_test.py b/test/getmerge_test.py index b7c3fb2..77f47aa 100644 --- a/test/getmerge_test.py +++ b/test/getmerge_test.py @@ -12,7 +12,11 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. -from minicluster_testbase import MiniClusterTestBase + +# python 3 support +from __future__ import absolute_import, print_function, division + +from test.minicluster_testbase import MiniClusterTestBase import os import shutil @@ -30,7 +34,7 @@ def test_getmerge_file(self): os.mkdir(TESTFILES_PATH) self.cluster.getmerge('/test3', '%s/expected' % TESTFILES_PATH) expected_output = self._read_file('%s/expected' % TESTFILES_PATH) - self.client.getmerge('/test3', '%s/client' % TESTFILES_PATH).next() + next(self.client.getmerge('/test3', '%s/client' % TESTFILES_PATH)) client_output = self._read_file('%s/client' % TESTFILES_PATH) self.assertEqual(expected_output, client_output) @@ -38,7 +42,7 @@ def test_getmerge_directory(self): os.mkdir(TESTFILES_PATH) self.cluster.getmerge('/dir2/dir3', '%s/expected' % TESTFILES_PATH) expected_output = self._read_file('%s/expected' % TESTFILES_PATH) - self.client.getmerge('/dir2/dir3', '%s/client' % TESTFILES_PATH).next() + next(self.client.getmerge('/dir2/dir3', '%s/client' % TESTFILES_PATH)) client_output = self._read_file('%s/client' % TESTFILES_PATH) self.assertEqual(expected_output, client_output) @@ -46,7 +50,7 @@ def test_getmerge_directory_nl(self): os.mkdir(TESTFILES_PATH) self.cluster.getmerge('/dir2/dir3', '%s/expected' % TESTFILES_PATH, extra_args=['-nl']) expected_output = self._read_file('%s/expected' % TESTFILES_PATH) - self.client.getmerge('/dir2/dir3', '%s/client' % TESTFILES_PATH, newline=True).next() + next(self.client.getmerge('/dir2/dir3', '%s/client' % TESTFILES_PATH, newline=True)) client_output = self._read_file('%s/client' % TESTFILES_PATH) self.assertEqual(expected_output, client_output) @@ -55,7 +59,7 @@ def test_getmerge_directory_tree(self): os.mkdir(TESTFILES_PATH) self.cluster.getmerge('/dir2', '%s/expected' % TESTFILES_PATH) expected_output = self._read_file('%s/expected' % TESTFILES_PATH) - self.client.getmerge('/dir2', '%s/client' % TESTFILES_PATH).next() + next(self.client.getmerge('/dir2', '%s/client' % TESTFILES_PATH)) client_output = self._read_file('%s/client' % TESTFILES_PATH) self.assertEqual(expected_output, client_output) diff --git a/test/glob_test.py b/test/glob_test.py index 8e710c7..437a38a 100644 --- a/test/glob_test.py +++ b/test/glob_test.py @@ -12,6 +12,10 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. + +# python 3 support +from __future__ import absolute_import, print_function, division + import unittest2 import snakebite.glob as glob diff --git a/test/input.txt b/test/input.txt new file mode 100644 index 0000000..c2a2bdd --- /dev/null +++ b/test/input.txt @@ -0,0 +1,4 @@ +This +is +a +test diff --git a/test/list_test.py b/test/list_test.py index 519d9a7..787f883 100644 --- a/test/list_test.py +++ b/test/list_test.py @@ -12,8 +12,13 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. -from util import assertListings -from minicluster_testbase import MiniClusterTestBase + +# python 3 support +from __future__ import absolute_import, print_function, division + +from test.util import assertListings +from test.minicluster_testbase import MiniClusterTestBase +from snakebite.compat import py_2 from snakebite.errors import FileNotFoundException from snakebite.errors import InvalidInputException @@ -112,8 +117,8 @@ def test_path_cleanup(self): def test_unknown_file(self): result = self.client.ls(['/doesnotexist']) - self.assertRaises(FileNotFoundException, result.next) + self.assertRaises(FileNotFoundException, result.next if py_2 else result.__next__) def test_invalid_input(self): result = self.client.ls('/stringpath') - self.assertRaises(InvalidInputException, result.next) + self.assertRaises(InvalidInputException, result.next if py_2 else result.__next__) diff --git a/test/minicluster_testbase.py b/test/minicluster_testbase.py index 08c3957..425123c 100644 --- a/test/minicluster_testbase.py +++ b/test/minicluster_testbase.py @@ -1,3 +1,6 @@ +# python 3 support +from __future__ import absolute_import, print_function, division + import unittest2 import os import time diff --git a/test/rename2_test.py b/test/rename2_test.py index fe8d412..91c928e 100644 --- a/test/rename2_test.py +++ b/test/rename2_test.py @@ -12,8 +12,12 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. + +# python 3 support +from __future__ import absolute_import, print_function, division + from snakebite.errors import FileAlreadyExistsException -from minicluster_testbase import MiniClusterTestBase +from test.minicluster_testbase import MiniClusterTestBase class Rename2Test(MiniClusterTestBase): diff --git a/test/rename_test.py b/test/rename_test.py index 6629e56..a2a25ff 100644 --- a/test/rename_test.py +++ b/test/rename_test.py @@ -12,19 +12,24 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. + +# python 3 support +from __future__ import absolute_import, print_function, division + +from snakebite.compat import py_2 from snakebite.errors import FileNotFoundException from snakebite.errors import InvalidInputException -from minicluster_testbase import MiniClusterTestBase +from test.minicluster_testbase import MiniClusterTestBase class RenameTest(MiniClusterTestBase): def test_rename_file(self): - print list(self.client.rename(['/zerofile'], '/zerofile2')) + print(list(self.client.rename(['/zerofile'], '/zerofile2'))) expected_output = list(self.client.ls(['/zerofile2'], include_toplevel=True)) self.assertEqual(len(expected_output), 1) self.assertEqual(expected_output[0]['path'], '/zerofile2') result = self.client.ls(['/zerofile']) - self.assertRaises(FileNotFoundException, result.next) + self.assertRaises(FileNotFoundException, result.next if py_2 else result.__next__) def test_rename_multi(self): list(self.client.rename(['/test1', '/test2'], '/dir1')) @@ -35,11 +40,11 @@ def test_rename_multi(self): def test_unknown_file(self): result = self.client.rename(['/doesnotexist'], '/somewhereelse') - self.assertRaises(FileNotFoundException, result.next) + self.assertRaises(FileNotFoundException, result.next if py_2 else result.__next__) def test_invalid_input(self): result = self.client.rename('/stringpath', '777') - self.assertRaises(InvalidInputException, result.next) + self.assertRaises(InvalidInputException, result.next if py_2 else result.__next__) def test_rename_multi_with_trailing_slash(self): list(self.client.rename(['/test3', '/test4'], '/dir1/')) diff --git a/test/stat_test.py b/test/stat_test.py index 13eb99b..239da0a 100644 --- a/test/stat_test.py +++ b/test/stat_test.py @@ -12,7 +12,11 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. -from minicluster_testbase import MiniClusterTestBase + +# python 3 support +from __future__ import absolute_import, print_function, division + +from test.minicluster_testbase import MiniClusterTestBase from snakebite.errors import InvalidInputException from snakebite.errors import FileNotFoundException from snakebite.formatter import _format_permission @@ -36,9 +40,9 @@ def test_invalid_path(self): ['/does/not/exist']) def test_format_permission(self): - self.assertEquals(_format_permission(int(0o1777)), '1777') - self.assertEquals(_format_permission(int(0o777)), '0777') - self.assertEquals(_format_permission(int(0o000777)), '0777') + self.assertEqual(_format_permission(int(0o1777)), '1777') + self.assertEqual(_format_permission(int(0o777)), '0777') + self.assertEqual(_format_permission(int(0o000777)), '0777') def test_sticky_mode(self): list(self.client.chmod(['/sticky_dir'], 0o1777)) diff --git a/test/tail_test.py b/test/tail_test.py index 770fbd3..199fa07 100644 --- a/test/tail_test.py +++ b/test/tail_test.py @@ -13,7 +13,11 @@ # License for the specific language governing permissions and limitations under # the License. -from minicluster_testbase import MiniClusterTestBase +# python 3 support +from __future__ import absolute_import, print_function, division + +from snakebite.compat import range +from test.minicluster_testbase import MiniClusterTestBase import os import random @@ -28,7 +32,7 @@ def test_tail_on_one_block(self): def test_tail_on_file_smaller_than_1KB(self): path = '/temp_test' p = self.cluster.put_subprocess('-', path) - print >> p.stdin, "just a couple of bytes" + print("just a couple of bytes", file=p.stdin) p.communicate() self._compare_files(path) @@ -63,10 +67,10 @@ def _generate_file_over_two_blocks(self, path): f = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'testfiles', 'test3')) p = self.cluster.put_subprocess('-', path) - for _ in xrange(131072): # 1024 * 131072 = 134,217,728 (default block size) + for _ in range(131072): # 1024 * 131072 = 134,217,728 (default block size) f.seek(0) for line in f.readlines(): - print >> p.stdin, line - print >> p.stdin, 'some extra bytes to exceed one blocksize' # +40 + print(line, file=p.stdin) + print('some extra bytes to exceed one blocksize', file=p.stdin) # +40 p.communicate() diff --git a/test/test.py b/test/test.py index abb370a..6310220 100755 --- a/test/test.py +++ b/test/test.py @@ -12,6 +12,10 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. + +# python 3 support +from __future__ import absolute_import, print_function, division + import sys import os import glob @@ -20,7 +24,7 @@ def suite(): suite = unittest2.TestSuite() for filename in glob.glob('test/*_test.py'): - print filename + print(filename) f = os.path.splitext(os.path.basename(filename))[0] module = __import__(f) suite.addTest(unittest2.defaultTestLoader.loadTestsFromModule(module)) @@ -35,8 +39,8 @@ def __init__(self): def usageExit(self, msg=None): if msg: - print msg - print self.USAGE % self.__dict__ + print(msg) + print(self.USAGE % self.__dict__) sys.exit(-2) def runTests(self): diff --git a/test/test_test.py b/test/test_test.py index 883d8bd..bf11d92 100644 --- a/test/test_test.py +++ b/test/test_test.py @@ -12,7 +12,11 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. -from minicluster_testbase import MiniClusterTestBase + +# python 3 support +from __future__ import absolute_import, print_function, division + +from test.minicluster_testbase import MiniClusterTestBase class TestTest(MiniClusterTestBase): diff --git a/test/text_test.py b/test/text_test.py index 3d37825..776cf4e 100644 --- a/test/text_test.py +++ b/test/text_test.py @@ -13,16 +13,19 @@ # License for the specific language governing permissions and limitations under # the License. -from minicluster_testbase import MiniClusterTestBase +# python 3 support +from __future__ import absolute_import, print_function, division + +from test.minicluster_testbase import MiniClusterTestBase class TextTest(MiniClusterTestBase): def test_text_gzip(self): expected_output = self.cluster.text('/zipped/test1.gz') client_output = list(self.client.text(['/zipped/test1.gz']))[0] - self.assertEqual(expected_output, client_output) + self.assertEqual(expected_output, client_output.decode("utf-8")) def test_text_bzip2(self): expected_output = self.cluster.text('/zipped/test1.bz2') client_output = list(self.client.text(['/zipped/test1.bz2']))[0] - self.assertEqual(expected_output, client_output) + self.assertEqual(expected_output, client_output.decode("utf-8")) diff --git a/test/touchz_test.py b/test/touchz_test.py index 700a5b8..13a9df0 100644 --- a/test/touchz_test.py +++ b/test/touchz_test.py @@ -13,8 +13,11 @@ # License for the specific language governing permissions and limitations under # the License. +# python 3 support +from __future__ import absolute_import, print_function, division + from snakebite.errors import DirectoryException, FileException -from minicluster_testbase import MiniClusterTestBase +from test.minicluster_testbase import MiniClusterTestBase class TouchZTest(MiniClusterTestBase): counter = 0 diff --git a/test/util.py b/test/util.py index e4f7f8c..b38a540 100644 --- a/test/util.py +++ b/test/util.py @@ -12,6 +12,10 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. + +# python 3 support +from __future__ import absolute_import, print_function, division + import datetime diff --git a/tox.ini b/tox.ini index 5ca3cf1..5509246 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py26,py27}-{cdh,hdp} +envlist = {py26,py27,py34,py35}-{cdh,hdp} [testenv] usedevelop = True @@ -7,6 +7,8 @@ deps = -rrequirements-dev.txt basepython = py26: python2.6 py27: python2.7 + py34: python3.4 + py35: python3.5 setenv = cdh: HADOOP_DISTRO=cdh cdh: HADOOP_HOME=/tmp/hadoop-cdh