Merge pull request #41 from gisce/py3_migration

PY3 compat

cchloader/adapters/a5d.py

@@ -12,7 +12,7 @@ class A5dBaseAdapter(Schema):
 
     @pre_load
     def fix_numbers(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.Integer):
                 if not data.get(attr):
                     data[attr] = None

cchloader/adapters/b5d.py

@@ -12,7 +12,7 @@ class B5dBaseAdapter(Schema):
 
     @pre_load
     def fix_numbers(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.Integer):
                 if not data.get(attr):
                     data[attr] = None

cchloader/adapters/corbagen.py

@@ -12,7 +12,7 @@ class CorbaGenBaseAdapter(Schema):
 
     @pre_load
     def fix_numbers(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.Integer):
                 if not data.get(attr):
                     data[attr] = None

cchloader/adapters/f1.py

@@ -12,7 +12,7 @@ class F1BaseAdapter(Schema):
 
     @pre_load
     def fix_numbers(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, (fields.Integer, fields.Float)):
                 if not data.get(attr):
                     data[attr] = None

cchloader/adapters/f5d.py

@@ -12,7 +12,7 @@ class F5dBaseAdapter(Schema):
 
     @pre_load
     def fix_numbers(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.Integer):
                 if not data.get(attr):
                     data[attr] = None

cchloader/adapters/infpa.py

@@ -12,7 +12,7 @@ class InfpaBaseAdapter(Schema):
 
     @pre_load
     def fix_numbers(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, (fields.Integer, fields.Float)):
                 if not data.get(attr):
                     data[attr] = None

cchloader/adapters/medidas.py

@@ -12,7 +12,7 @@ class MedidasBaseAdapter(Schema):
 
     @pre_load
     def fix_numbers(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, (fields.Integer, fields.Float)):
                 if not data.get(attr):
                     data[attr] = None

cchloader/adapters/mhcil.py

@@ -12,15 +12,15 @@ class MhcilBaseAdapter(Schema):
 
     @pre_load
     def fix_numbers(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, (fields.Integer, fields.Float)):
                 if not data.get(attr):
                     data[attr] = None
         return data
 
     @pre_load
     def fix_ae(self, data):
-        ae = data.get('ae', 0)
+        ae = int(data.get('ae', 0))
         if ae < 0:
             data['ae'] = 0
 

cchloader/adapters/p1.py

@@ -12,7 +12,7 @@ class P1BaseAdapter(Schema):
 
     @pre_load
     def fix_numbers(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, (fields.Integer, fields.Float)):
                 if not data.get(attr):
                     data[attr] = None
@@ -50,13 +50,13 @@ def fix_type(self, data):
 
     @pre_load
     def valid_measure(self, data):
-        aoquality = data.get('aoquality')
-        r1quality = data.get('r1quality')
-        r2quality = data.get('r2quality')
-        r3quality = data.get('r3quality')
-        r4quality = data.get('r4quality')
-        reserve1quality = data.get('reserve1quality')
-        reserve2quality = data.get('reserve2quality')
+        aoquality = int(data.get('aoquality'))
+        r1quality = int(data.get('r1quality'))
+        r2quality = int(data.get('r2quality'))
+        r3quality = int(data.get('r3quality'))
+        r4quality = int(data.get('r4quality'))
+        reserve1quality = int(data.get('reserve1quality'))
+        reserve2quality = int(data.get('reserve2quality'))
 
         aovalid = 0
         if aoquality < 128:

cchloader/adapters/p5d.py

@@ -12,7 +12,7 @@ class P5dBaseAdapter(Schema):
 
     @pre_load
     def fix_numbers(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.Integer):
                 if not data.get(attr):
                     data[attr] = None

cchloader/adapters/reganecu.py

@@ -12,7 +12,7 @@ class ReganecuBaseAdapter(Schema):
 
     @pre_load
     def fix_numbers(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, (fields.Integer, fields.Float)):
                 if not data.get(attr):
                     data[attr] = None

cchloader/compress.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
-
+from __future__ import absolute_import, unicode_literals
+import six
 import struct
 import zipfile
 import gzip
@@ -18,8 +18,8 @@ def __init__(self, filename):
         self.filename = filename
 
     @classmethod
-    def is_magic(self, data):
-        return data.startswith(self.magic)
+    def is_magic(cls, data):
+        return data.decode().startswith(cls.magic)
 
     def open(self):
         return None
@@ -104,7 +104,10 @@ def __init__(self, filename):
         self.files = self.list()
 
     def open(self):
-        return bz2.BZ2File(self.filename)
+        _file = bz2.BZ2File(self.filename)
+        if six.PY3:
+            setattr(_file, 'name', self.filename)
+        return _file
 
     def list(self):
         # BZ2 file does not include original filename
@@ -131,7 +134,7 @@ def is_compressed_data(fd):
     return False
 
 def is_compressed_file(filename):
-    with file(filename, 'rb') as f:
+    with open(filename, 'rb') as f:
         start_of_file = f.read(5)
         f.seek(0)
         for cls in (ZIPFile, GZFile, BZFile):
@@ -140,7 +143,7 @@ def is_compressed_file(filename):
         return False
 
 def get_compressed_file(filename):
-    with file(filename, 'rb') as f:
+    with open(filename, 'rb') as f:
         start_of_file = f.read(5)
         f.seek(0)
         for cls in (ZIPFile, GZFile, BZFile):

cchloader/file.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 import bz2
-from StringIO import StringIO
+from io import BytesIO
 
 from cchloader.parsers.parser import get_parser
 from cchloader import logger
@@ -42,6 +42,9 @@ def next(self):
             return sf
         raise StopIteration()
 
+    def __next__(self):
+        return self.next()
+
     def __enter__(self):
         return self
 
@@ -79,16 +82,16 @@ def __init__(self, path, fd=None, parser=None, strict=False):
         else:
             self.parser = parser
         if fd is None:
-            self.fd = open(path, 'r')
+            self.fd = open(path, 'rb')
         else:
             try:
                 file_content = fd.read()
                 # ZipExtFile has no seek method
-                tmp_fd = StringIO(file_content)
+                tmp_fd = BytesIO(file_content)
                 if is_compressed_data(tmp_fd):
                     # bz compressed files ina zip file
                     data = bz2.decompress(file_content)
-                    bzfd = StringIO(data)
+                    bzfd = BytesIO(data)
                     self.fd = bzfd
                 else:
                     self.fd = tmp_fd
@@ -98,10 +101,19 @@ def __init__(self, path, fd=None, parser=None, strict=False):
     def __iter__(self):
         return self
 
+    def __next__(self):
+        return self.next()
+
+    @staticmethod
+    def _bytes_to_unicode(data):
+        return {x: y.decode() if isinstance(y, bytes) else y for x, y in data.items()}
+
     def next(self):
         for line in self.fd:
             try:
+                # data, errors = self.parser.parse_line(line)
                 data, errors = self.parser.parse_line(line)
+                data = self._bytes_to_unicode(data)
 #                if errors:
 #                    self.stats.errors.append(
 #                        (self.stats.line_number, errors)

cchloader/parsers/a5d.py

@@ -6,6 +6,9 @@
 from cchloader.adapters.a5d import A5dAdapter
 from cchloader.models.cch_autocons import CchAutoconsSchema
 from cchloader.parsers.parser import Parser, register
+import six
+if six.PY3:
+    unicode = str
 
 
 class A5d(Parser):
@@ -26,7 +29,7 @@ def __init__(self, strict=False):
 
     def parse_line(self, line):
         slinia = tuple(unicode(line.decode(self.encoding)).split(self.delimiter))
-        slinia = map(lambda s: s.strip(), slinia)
+        slinia = list(map(lambda s: s.strip(), slinia))
         parsed = {'cch_autocons': {}, 'orig': line}
         data = build_dict(self.headers, slinia)
         result, errors = self.adapter.load(data)

cchloader/parsers/b5d.py

@@ -6,6 +6,9 @@
 from cchloader.adapters.b5d import B5dAdapter
 from cchloader.models.cch_gennetabeta import CchGenNetaBetaSchema
 from cchloader.parsers.parser import Parser, register
+import six
+if six.PY3:
+    unicode = str
 
 
 class B5d(Parser):
@@ -26,7 +29,7 @@ def __init__(self, strict=False):
 
     def parse_line(self, line):
         slinia = tuple(unicode(line.decode(self.encoding)).split(self.delimiter))
-        slinia = map(lambda s: s.strip(), slinia)
+        slinia = list(map(lambda s: s.strip(), slinia))
         parsed = {'cch_gennetabeta': {}, 'orig': line}
         data = build_dict(self.headers, slinia)
         result, errors = self.adapter.load(data)

cchloader/parsers/corbagen.py

@@ -6,6 +6,9 @@
 from cchloader.adapters.corbagen import CorbaGenAdapter
 from cchloader.models.corbagen import CorbaGenSchema
 from cchloader.parsers.parser import Parser, register
+import six
+if six.PY3:
+    unicode = str
 
 
 class CorbaGen(Parser):
@@ -27,7 +30,7 @@ def __init__(self, strict=False):
 
     def parse_line(self, line):
         slinia = tuple(unicode(line.decode(self.encoding)).split(self.delimiter))
-        slinia = map(lambda s: s.strip(), slinia)
+        slinia = list(map(lambda s: s.strip(), slinia))
         parsed = {'corbagen': {}, 'orig': line}
         data = build_dict(self.headers, slinia)
         result, errors = self.adapter.load(data)

cchloader/parsers/f1.py

@@ -6,6 +6,9 @@
 from cchloader.adapters.f1 import F1Adapter
 from cchloader.models.f1 import F1Schema
 from cchloader.parsers.parser import Parser, register
+import six
+if six.PY3:
+    unicode = str
 
 
 class F1(Parser):
@@ -36,7 +39,7 @@ def __init__(self, strict=False):
 
     def parse_line(self, line):
         slinia = tuple(unicode(line.decode(self.encoding)).split(self.delimiter))
-        slinia = map(lambda s: s.strip(), slinia)
+        slinia = list(map(lambda s: s.strip(), slinia))
         parsed = {'f1': {}, 'orig': line}
         data = build_dict(self.headers, slinia)
         result, errors = self.adapter.load(data)

cchloader/parsers/f5d.py

@@ -6,6 +6,9 @@
 from cchloader.adapters.f5d import F5dAdapter
 from cchloader.models.cchfact import CchFactSchema
 from cchloader.parsers.parser import Parser, register
+import six
+if six.PY3:
+    unicode = str
 
 
 class F5d(Parser):
@@ -27,7 +30,7 @@ def __init__(self, strict=False):
 
     def parse_line(self, line):
         slinia = tuple(unicode(line.decode(self.encoding)).split(self.delimiter))
-        slinia = map(lambda s: s.strip(), slinia)
+        slinia = list(map(lambda s: s.strip(), slinia))
         parsed = {'cchfact': {}, 'orig': line}
         data = build_dict(self.headers, slinia)
         result, errors = self.adapter.load(data)

cchloader/parsers/infpa.py

@@ -6,6 +6,10 @@
 from cchloader.adapters.infpa import InfpaAdapter
 from cchloader.models.infpa import InfpaSchema
 from cchloader.parsers.parser import Parser, register
+import six
+if six.PY3:
+    unicode = str
+
 
 
 class Infpa(Parser):
@@ -27,7 +31,7 @@ def __init__(self, strict=False):
 
     def parse_line(self, line):
         slinia = tuple(unicode(line.decode(self.encoding)).split(self.delimiter))
-        slinia = map(lambda s: s.strip(), slinia)
+        slinia = list(map(lambda s: s.strip(), slinia))
         parsed = {'infpa': {}, 'orig': line}
         data = build_dict(self.headers, slinia)
         result, errors = self.adapter.load(data)

cchloader/parsers/medidas.py

@@ -6,6 +6,10 @@
 from cchloader.adapters.medidas import MedidasAdapter
 from cchloader.models.medidas import MedidasSchema
 from cchloader.parsers.parser import Parser, register
+import six
+if six.PY3:
+    unicode = str
+
 
 
 class Medidas(Parser):
@@ -27,7 +31,7 @@ def __init__(self, strict=False):
 
     def parse_line(self, line):
         slinia = tuple(unicode(line.decode(self.encoding)).split(self.delimiter))
-        slinia = map(lambda s: s.strip(), slinia)
+        slinia = list(map(lambda s: s.strip(), slinia))
         parsed = {'medidas': {}, 'orig': line}
         data = build_dict(self.headers, slinia)
         result, errors = self.adapter.load(data)

cchloader/parsers/mhcil.py

@@ -6,6 +6,9 @@
 from cchloader.adapters.mhcil import MhcilAdapter
 from cchloader.models.mhcil import MhcilSchema
 from cchloader.parsers.parser import Parser, register
+import six
+if six.PY3:
+    unicode = str
 
 
 class Mhcil(Parser):
@@ -27,7 +30,7 @@ def __init__(self, strict=False):
 
     def parse_line(self, line):
         slinia = tuple(unicode(line.decode(self.encoding)).split(self.delimiter))
-        slinia = map(lambda s: s.strip(), slinia)
+        slinia = list(map(lambda s: s.strip(), slinia))
         parsed = {'mhcil': {}, 'orig': line}
         data = build_dict(self.headers, slinia)
         result, errors = self.adapter.load(data)