Merge pull request #35 from gisce/60185_migrate_gas_sips

Añadir modelos para sips de gas

sippers/adapters/cnmc_gas.py

@@ -0,0 +1,61 @@
+from sippers.adapters import SipsAdapter, MeasuresAdapter
+from sippers.models.cnmc_gas import CnmcGasSipsSchema, CnmcGasMeasuresSchema
+from marshmallow import pre_load, fields
+
+
+class CnmcGasSipsAdapter(SipsAdapter, CnmcGasSipsSchema):
+    '''A self.fields tenim els camps per defecte, els de SipsSchema base'''
+
+    @pre_load
+    def parse_selections(self, data):
+        selection_fields = [
+            'propiedadEquipoMedida',
+            'codigoResultadoInspeccion',
+            'tipoPerfilConsumo',
+            'codigoPeajeEnVigor'
+         ]
+        for select_field in selection_fields:
+            if data.get(select_field) == '':
+                data[select_field] = None
+        return data
+
+    @pre_load
+    def parse_propiedad_equipo_medida(self, data):
+        if data.get('propiedadEquipoMedida') and len(data.get('propiedadEquipoMedida'))==1:
+            data['propiedadEquipoMedida'] = '0' + data['propiedadEquipoMedida']
+        return data
+
+    @pre_load
+    def fix_dates(self, data):
+        for attr, field in self.fields.iteritems():
+            if isinstance(field, fields.DateTime):
+                # si ve alguna data, assumim que ve correcta
+                if data[attr] == u'':
+                    data[attr] = None
+                else:
+                    data[attr] += 'T00:00:00'
+        return data
+
+class CnmcGasMeasuresAdapter(MeasuresAdapter, CnmcGasMeasuresSchema):
+
+    @pre_load
+    def parse_selections(self, data):
+        selection_fields = [
+            'codigoTarifaPeaje',
+            'codigoTipoLectura'
+        ]
+        for select_field in selection_fields:
+            if data.get(select_field) == '':
+                data[select_field] = None
+        return data
+
+    @pre_load
+    def fix_dates(self, data):
+        for attr, field in self.fields.iteritems():
+            if isinstance(field, fields.DateTime):
+                # si ve alguna data, assumim que ve correcta
+                if data[attr] == u'':
+                    data[attr] = None
+                else:
+                    data[attr] += 'T00:00:00'
+        return data

sippers/backends/mongodb.py

@@ -1,4 +1,7 @@
+from __future__ import (absolute_import)
 from sippers.backends import BaseBackend, register, urlparse
+from sippers.parsers.cnmc_v2 import CnmcV2, CnmcV2Cons
+from sippers.parsers.cnmc_gas import CnmcGas, CnmcGasCons
 import pymongo
 
 
@@ -15,21 +18,19 @@ def __init__(self, uri=None):
         self.db = self.connection[self.config['db']]
         self.ps_collection = 'giscedata_sips_ps'
         self.measures_collection = 'giscedata_sips_consums'
-        self.db[self.ps_collection].ensure_index(
-            "name", unique=True, background=True
-        )
-        self.db[self.measures_collection].ensure_index(
-           "name", background=True,
-        )
-        self.db['cnmc_sips'].ensure_index(
-            "cups", unique=True, background=True
-        )
-        self.db['cnmc_sips_consums'].ensure_index(
-            [
-                ("cups", pymongo.ASCENDING),
-                ("fechaFinMesConsumo", pymongo.DESCENDING)
-            ], background=True,
-        )
+        self.index_tables()
+
+    def index_tables(self):
+        models = [CnmcV2, CnmcV2Cons, CnmcGas, CnmcGasCons]
+        sips_table =[
+            [self.ps_collection, 'name', True], [self.measures_collection, 'name', False]]
+        for model in models:
+            sips_table.append([model.collection, model.collection_index, model.index_unic])
+
+        for table in sips_table:
+            self.db[table[0]].ensure_index(
+                table[1], unique=table[2], background=True
+            )
 
     def insert(self, document):
         batch_insert = False

sippers/models/cnmc_gas.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+from marshmallow import Schema, fields
+from marshmallow.validate import OneOf
+
+from sippers.utils import TAULA_PROPIEDAD_CONTADOR_CORRECTOR, TAULA_RESULTADO_INSPECCION, TAULA_PERFIL_CONSUMO, TAULA_TIPO_PEAJE, TAULA_TIPO_LECTURA
+
+
+class CnmcGasSipsSchema(Schema):
+    """Base model for SIPS
+    """
+    apellido1Titular = fields.String(allow_none=True)
+    apellido2Titular = fields.String(allow_none=True)
+    calibreContador = fields.String(allow_none=True)
+    caudalHorarioEnWh = fields.String(allow_none=True)
+    caudalMaximoDiarioEnWh = fields.String(allow_none=True)
+    cnae = fields.String(allow_none=True)
+    codigoAccesibilidadContador = fields.String(allow_none=True)
+    codigoContador = fields.String(allow_none=True)
+    codigoEmpresaDistribuidora = fields.String(allow_none=True)
+    codigoPeajeEnVigor = fields.String(validate=OneOf(TAULA_TIPO_PEAJE),allow_none=True)
+    codigoPostalPS = fields.String(allow_none=True)
+    codigoPostalTitular = fields.String(allow_none=True)
+    codigoPresion = fields.String(allow_none=True)
+    codigoProvinciaPS = fields.String(allow_none=True)
+    codigoProvinciaTitular = fields.String(allow_none=True)
+    codigoResultadoInspeccion = fields.String(validate=OneOf(TAULA_RESULTADO_INSPECCION), allow_none=True)
+    codigoTelemedida = fields.String(allow_none=True)
+    conectadoPlantaSatelite = fields.String(allow_none=True)
+    cups = fields.String(allow_none=True)
+    derechoTUR = fields.String(allow_none=True)
+    desMunicipioPS = fields.String(allow_none=True)
+    desMunicipioTitular = fields.String(allow_none=True)
+    desProvinciaPS = fields.String(allow_none=True)
+    desProvinciaTitular = fields.String(allow_none=True)
+    esViviendaHabitual = fields.String(allow_none=True)
+    escaleraPS = fields.String(allow_none=False)
+    escaleraTitular = fields.String(allow_none=True)
+    fechaUltimaInspeccion = fields.DateTime(allow_none=True)
+    fechaUltimoCambioComercializador = fields.DateTime(allow_none=True)
+    fechaUltimoMovimientoContrato = fields.DateTime(allow_none=True)
+    idTipoTitular = fields.String(allow_none=True)
+    idTitular = fields.String(allow_none=True)
+    informacionImpagos = fields.String(allow_none=True)
+    municipioPS = fields.String(allow_none=True)
+    municipioTitular = fields.String(allow_none=True)
+    nombreEmpresaDistribuidora = fields.String(allow_none=True)
+    nombreTitular = fields.String(allow_none=True)
+    numFincaPS = fields.String(allow_none=True)
+    numFincaTitular = fields.String(allow_none=True)
+    pctd = fields.String(allow_none=True)
+    pisoPS = fields.String(allow_none=True)
+    pisoTitular = fields.String(allow_none=True)
+    portalPS = fields.String(allow_none=True)
+    portalTitular = fields.String(allow_none=True)
+    presionMedida = fields.String(allow_none=False)
+    propiedadEquipoMedida = fields.String(validate=OneOf(TAULA_PROPIEDAD_CONTADOR_CORRECTOR) ,allow_none=True)
+    puertaPS = fields.String(allow_none=True)
+    puertaTitular = fields.String(allow_none=True)
+    tipoContador = fields.String(allow_none=True)
+    tipoCorrector = fields.String(allow_none=True)
+    tipoPerfilConsumo = fields.String(OneOf(TAULA_PERFIL_CONSUMO), allow_none=True)
+    tipoViaPS = fields.String(allow_none=True)
+    tipoViaTitular = fields.String(allow_none=True)
+    viaPS = fields.String(allow_none=True)
+    viaTitular = fields.String(allow_none=True)
+
+class CnmcGasMeasuresSchema(Schema):
+
+    cups = fields.String(position=0, required=True, allow_none=False)
+    fechaInicioMesConsumo = fields.DateTime(required=True, position=1, allow_none=False)
+    fechaFinMesConsumo = fields.DateTime(required=True, position=2, allow_none=False)
+    codigoTarifaPeaje = fields.String(validate=OneOf(TAULA_TIPO_PEAJE), position=3, required=False, allow_none=True)
+    consumoEnWhP1 = fields.String(position=4, required=False, allow_none=True)
+    consumoEnWhP2 = fields.String(position=5, required=False, allow_none=True)
+    caudalMedioEnWhdia = fields.String(position=6, required=False, allow_none=True)
+    caudaMinimoDiario = fields.String(position=7, required=False, allow_none=True)
+    caudaMaximoDiario = fields.String(position=8, required=False, allow_none=True)
+    porcentajeConsumoNocturno = fields.String(position=9, required=False, allow_none=True)
+    codigoTipoLectura = fields.String(validate=OneOf(TAULA_TIPO_LECTURA), position=10, required=False, allow_none=True)

sippers/parsers/cnmc_gas.py

@@ -0,0 +1,152 @@
+from __future__ import absolute_import
+
+import csv
+import StringIO
+
+import pymongo
+
+from sippers import logger
+from sippers.utils import build_dict
+from sippers.parsers.parser import Parser, register
+from sippers.adapters.cnmc_gas import CnmcGasSipsAdapter, CnmcGasMeasuresAdapter
+from sippers.models.cnmc_gas import CnmcGasMeasuresSchema
+
+class CnmcGas(Parser):
+
+    # En aquest cas els llegim amb el csv.DictReader en comptes de fer-ho amb
+    # un Schema de marshmallow, ja que el csv pot contenir comes dins d'un
+    # camp si van wrapped entre cometes i el marshmallow aixo no ho contempla.
+
+    # amb csv.DictReader obtindrem un diccionari amb els headers_ps
+    # que li indiquem aqui, en comptes d'anar-los a buscar a l'Schema.
+    # El diccionari el podem utilitzar per passar-li al
+    # self.adapter.load igual que el resultat de l'esquema
+
+    pattern = '[0-9]{6}_SIPS2_PS_GAS_[a-z]*_?[a-z]*\.csv'
+    encoding = "UTF-8"
+    collection = 'cnmc_sips_gas'
+    collection_index = 'cups'
+    index_unic = True
+
+    def __init__(self, strict=False):
+
+        # l'ordre dels camps segons format cnmc
+        self.headers_ps = [
+            'codigoEmpresaDistribuidora',
+            'nombreEmpresaDistribuidora',
+            'cups',
+            'codigoProvinciaPS',
+            'desProvinciaPS',
+            'codigoPostalPS',
+            'municipioPS',
+            'desMunicipioPS',
+            'tipoViaPS',
+            'viaPS',
+            'numFincaPS',
+            'portalPS',
+            'escaleraPS',
+            'pisoPS',
+            'puertaPS',
+            'codigoPresion',
+            'codigoPeajeEnVigor',
+            'caudalMaximoDiarioEnWh',
+            'caudalHorarioEnWh',
+            'derechoTUR',
+            'fechaUltimaInspeccion',
+            'codigoResultadoInspeccion',
+            'tipoPerfilConsumo',
+            'codigoContador',
+            'calibreContador',
+            'tipoContador',
+            'propiedadEquipoMedida',
+            'codigoTelemedida',
+            'fechaUltimoMovimientoContrato',
+            'fechaUltimoCambioComercializador',
+            'informacionImpagos',
+            'idTipoTitular',
+            'idTitular',
+            'nombreTitular',
+            'apellido1Titular',
+            'apellido2Titular',
+            'codigoProvinciaTitular',
+            'desProvinciaTitular',
+            'codigoPostalTitular',
+            'municipioTitular',
+            'desMunicipioTitular',
+            'tipoViaTitular',
+            'viaTitular',
+            'numFincaTitular',
+            'portalTitular',
+            'escaleraTitular',
+            'pisoTitular',
+            'puertaTitular',
+            'esViviendaHabitual',
+            'cnae',
+            'tipoCorrector',
+            'codigoAccesibilidadContador',
+            'conectadoPlantaSatelite',
+            'pctd',
+            'presionMedida'
+        ]
+        self.adapter = CnmcGasSipsAdapter(strict=strict)
+
+    def parse_line(self, line):
+
+        # passar previament la linia pel csv reader
+        # per que agafi be els camps tot i les comes dins del camp direccio
+        # per fer-ho cal passar-la a StringIO
+        l = StringIO.StringIO(line)
+        reader = csv.DictReader(l, fieldnames=self.headers_ps, delimiter=',')
+        linia = reader.next() # nomes n'hi ha una
+
+        parsed = {'ps': {}, 'orig': line, 'collection': self.collection}
+        result, errors = self.adapter.load(linia)
+
+        if errors:
+            logger.error(errors)
+        parsed['ps'] = result
+        return parsed, errors
+
+register(CnmcGas)
+
+class CnmcGasCons(Parser):
+
+    # En el cas de les mesures, usem Schema per mantenir el format i
+    # perque no hi trobarem mes comes que les delimiters
+    pattern = '[0-9]{6}_SIPS2_CONSUMOS_GAS_[a-z]*_?[a-z]*\.csv'
+    encoding = "UTF-8"
+    collection = 'cnmc_sips_consums_gas'
+    collection_index = [("cups", pymongo.ASCENDING), ("fechaFinMesConsumo", pymongo.DESCENDING)]
+    index_unic = False
+
+
+    def __init__(self, strict=False):
+        self.schema = CnmcGasMeasuresSchema(strict=strict)
+        self.adapter = CnmcGasMeasuresAdapter(strict=strict)
+        self.measures_adapter = self.adapter
+        self.fields = []
+        self.headers = []
+        for f in sorted(self.schema.fields,
+                key=lambda f: self.schema.fields[f].metadata['position']):
+            field = self.schema.fields[f]
+            self.fields.append((f, field.metadata))
+            self.headers.append(f)
+
+    def parse_line(self, line):
+
+        l = StringIO.StringIO(line)
+        reader = csv.DictReader(l, fieldnames=self.headers, delimiter=',')
+        linia = reader.next()  # nomes n'hi ha una
+
+        parsed = {'ps': {}, 'measure_cnmc': [], 'orig': line, 'collection': self.collection}
+
+        result, errors = self.adapter.load(linia)
+
+        if errors:
+            logger.error(errors)
+
+        parsed['measure_cnmc'] = result
+
+        return parsed, errors
+
+register(CnmcGasCons)

sippers/parsers/cnmc_v2.py

@@ -3,6 +3,8 @@
 import csv
 import StringIO
 
+import pymongo
+
 from sippers import logger
 from sippers.utils import build_dict
 from sippers.parsers.parser import Parser, register
@@ -22,7 +24,9 @@ class CnmcV2(Parser):
 
     pattern = '[0-9]{6}_SIPS2_PS_ELECTRICIDAD_[a-z]*_?[a-z]*\.csv'
     encoding = "UTF-8"
-    _collection = 'cnmc_sips'
+    collection = 'cnmc_sips'
+    collection_index = 'cups'
+    index_unic = True
 
     def __init__(self, strict=False):
 
@@ -97,7 +101,7 @@ def parse_line(self, line):
         reader = csv.DictReader(l, fieldnames=self.headers_ps, delimiter=',')
         linia = reader.next() # nomes n'hi ha una
 
-        parsed = {'ps': {}, 'orig': line, 'collection': self._collection}
+        parsed = {'ps': {}, 'orig': line, 'collection': self.collection}
         result, errors = self.adapter.load(linia)
 
         if errors:
@@ -113,7 +117,9 @@ class CnmcV2Cons(Parser):
     # perque no hi trobarem mes comes que les delimiters
     pattern = '[0-9]{6}_SIPS2_CONSUMOS_ELECTRICIDAD_[a-z]*_?[a-z]*\.csv'
     encoding = "UTF-8"
-    _collection = 'cnmc_sips_consums'
+    collection = 'cnmc_sips_consums'
+    collection_index = [("cups", pymongo.ASCENDING),("fechaFinMesConsumo", pymongo.DESCENDING)]
+    index_unic = False
 
 
     def __init__(self, strict=False):
@@ -134,7 +140,7 @@ def parse_line(self, line):
         reader = csv.DictReader(l, fieldnames=self.headers, delimiter=',')
         linia = reader.next()  # nomes n'hi ha una
 
-        parsed = {'ps': {}, 'measure_cnmc': [], 'orig': line, 'collection': self._collection}
+        parsed = {'ps': {}, 'measure_cnmc': [], 'orig': line, 'collection': self.collection}
 
         result, errors = self.adapter.load(linia)
 

sippers/utils.py

@@ -69,3 +69,8 @@ def build_dict(headers, data):
 TABLA_64 = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29', '30', '31', '32', '33', '34', '35', '36', '37', '38', '39', '40', '41', '42', '43', '44', '45', '46', '47', '48', '49', '99']
 TABLA_108 = ['01','02']
 TABLA_111 = ['01', '02', '03']
+TAULA_PROPIEDAD_CONTADOR_CORRECTOR = ['04', '06', '01', '02', '11', '12', '08', '10']
+TAULA_RESULTADO_INSPECCION = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14']
+TAULA_PERFIL_CONSUMO = ['01', '02']
+TAULA_TIPO_PEAJE = ['11', '12', '13', '21', '22', '23', '24', '25', '26', '1B', '2B', '3B', '4B', '5B', '6B', '31', '32', '33', '34', '35', '41', '42', '43', '44', '45', '46', '47', 'A1', 'A2', 'A3', 'B1', 'B2', 'C1', 'C2', 'D1', 'D2', 'M1', 'M2', 'R1', 'R2', 'R3', 'R4', 'R8', 'R9', 'L0', 'L1', 'A5', 'A6', 'A7', 'B5', 'B6', 'B7', 'S1', 'S2', 'S3', 'S4', 'S5', 'S6', 'S7', 'S8', '99']
+TAULA_TIPO_LECTURA = ['0', '1', '2']