Skip to content

Commit 20d65a6

Browse files
eberlosoeberloso
authored
Merge pull request #35 from gisce/60185_migrate_gas_sips
Añadir modelos para sips de gas
2 parents 9098832 + b32f95d commit 20d65a6

File tree

6 files changed

+323
-19
lines changed

6 files changed

+323
-19
lines changed

‎sippers/adapters/cnmc_gas.py

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
from sippers.adapters import SipsAdapter, MeasuresAdapter
2+
from sippers.models.cnmc_gas import CnmcGasSipsSchema, CnmcGasMeasuresSchema
3+
from marshmallow import pre_load, fields
4+
5+
6+
class CnmcGasSipsAdapter(SipsAdapter, CnmcGasSipsSchema):
7+
'''A self.fields tenim els camps per defecte, els de SipsSchema base'''
8+
9+
@pre_load
10+
def parse_selections(self, data):
11+
selection_fields = [
12+
'propiedadEquipoMedida',
13+
'codigoResultadoInspeccion',
14+
'tipoPerfilConsumo',
15+
'codigoPeajeEnVigor'
16+
]
17+
for select_field in selection_fields:
18+
if data.get(select_field) == '':
19+
data[select_field] = None
20+
return data
21+
22+
@pre_load
23+
def parse_propiedad_equipo_medida(self, data):
24+
if data.get('propiedadEquipoMedida') and len(data.get('propiedadEquipoMedida'))==1:
25+
data['propiedadEquipoMedida'] = '0' + data['propiedadEquipoMedida']
26+
return data
27+
28+
@pre_load
29+
def fix_dates(self, data):
30+
for attr, field in self.fields.iteritems():
31+
if isinstance(field, fields.DateTime):
32+
# si ve alguna data, assumim que ve correcta
33+
if data[attr] == u'':
34+
data[attr] = None
35+
else:
36+
data[attr] += 'T00:00:00'
37+
return data
38+
39+
class CnmcGasMeasuresAdapter(MeasuresAdapter, CnmcGasMeasuresSchema):
40+
41+
@pre_load
42+
def parse_selections(self, data):
43+
selection_fields = [
44+
'codigoTarifaPeaje',
45+
'codigoTipoLectura'
46+
]
47+
for select_field in selection_fields:
48+
if data.get(select_field) == '':
49+
data[select_field] = None
50+
return data
51+
52+
@pre_load
53+
def fix_dates(self, data):
54+
for attr, field in self.fields.iteritems():
55+
if isinstance(field, fields.DateTime):
56+
# si ve alguna data, assumim que ve correcta
57+
if data[attr] == u'':
58+
data[attr] = None
59+
else:
60+
data[attr] += 'T00:00:00'
61+
return data

‎sippers/backends/mongodb.py

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1+
from __future__ import (absolute_import)
12
from sippers.backends import BaseBackend, register, urlparse
3+
from sippers.parsers.cnmc_v2 import CnmcV2, CnmcV2Cons
4+
from sippers.parsers.cnmc_gas import CnmcGas, CnmcGasCons
25
import pymongo
36

47

@@ -15,21 +18,19 @@ def __init__(self, uri=None):
1518
self.db = self.connection[self.config['db']]
1619
self.ps_collection = 'giscedata_sips_ps'
1720
self.measures_collection = 'giscedata_sips_consums'
18-
self.db[self.ps_collection].ensure_index(
19-
"name", unique=True, background=True
20-
)
21-
self.db[self.measures_collection].ensure_index(
22-
"name", background=True,
23-
)
24-
self.db['cnmc_sips'].ensure_index(
25-
"cups", unique=True, background=True
26-
)
27-
self.db['cnmc_sips_consums'].ensure_index(
28-
[
29-
("cups", pymongo.ASCENDING),
30-
("fechaFinMesConsumo", pymongo.DESCENDING)
31-
], background=True,
32-
)
21+
self.index_tables()
22+
23+
def index_tables(self):
24+
models = [CnmcV2, CnmcV2Cons, CnmcGas, CnmcGasCons]
25+
sips_table =[
26+
[self.ps_collection, 'name', True], [self.measures_collection, 'name', False]]
27+
for model in models:
28+
sips_table.append([model.collection, model.collection_index, model.index_unic])
29+
30+
for table in sips_table:
31+
self.db[table[0]].ensure_index(
32+
table[1], unique=table[2], background=True
33+
)
3334

3435
def insert(self, document):
3536
batch_insert = False

‎sippers/models/cnmc_gas.py

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# -*- coding: utf-8 -*-
2+
from marshmallow import Schema, fields
3+
from marshmallow.validate import OneOf
4+
5+
from sippers.utils import TAULA_PROPIEDAD_CONTADOR_CORRECTOR, TAULA_RESULTADO_INSPECCION, TAULA_PERFIL_CONSUMO, TAULA_TIPO_PEAJE, TAULA_TIPO_LECTURA
6+
7+
8+
class CnmcGasSipsSchema(Schema):
9+
"""Base model for SIPS
10+
"""
11+
apellido1Titular = fields.String(allow_none=True)
12+
apellido2Titular = fields.String(allow_none=True)
13+
calibreContador = fields.String(allow_none=True)
14+
caudalHorarioEnWh = fields.String(allow_none=True)
15+
caudalMaximoDiarioEnWh = fields.String(allow_none=True)
16+
cnae = fields.String(allow_none=True)
17+
codigoAccesibilidadContador = fields.String(allow_none=True)
18+
codigoContador = fields.String(allow_none=True)
19+
codigoEmpresaDistribuidora = fields.String(allow_none=True)
20+
codigoPeajeEnVigor = fields.String(validate=OneOf(TAULA_TIPO_PEAJE),allow_none=True)
21+
codigoPostalPS = fields.String(allow_none=True)
22+
codigoPostalTitular = fields.String(allow_none=True)
23+
codigoPresion = fields.String(allow_none=True)
24+
codigoProvinciaPS = fields.String(allow_none=True)
25+
codigoProvinciaTitular = fields.String(allow_none=True)
26+
codigoResultadoInspeccion = fields.String(validate=OneOf(TAULA_RESULTADO_INSPECCION), allow_none=True)
27+
codigoTelemedida = fields.String(allow_none=True)
28+
conectadoPlantaSatelite = fields.String(allow_none=True)
29+
cups = fields.String(allow_none=True)
30+
derechoTUR = fields.String(allow_none=True)
31+
desMunicipioPS = fields.String(allow_none=True)
32+
desMunicipioTitular = fields.String(allow_none=True)
33+
desProvinciaPS = fields.String(allow_none=True)
34+
desProvinciaTitular = fields.String(allow_none=True)
35+
esViviendaHabitual = fields.String(allow_none=True)
36+
escaleraPS = fields.String(allow_none=False)
37+
escaleraTitular = fields.String(allow_none=True)
38+
fechaUltimaInspeccion = fields.DateTime(allow_none=True)
39+
fechaUltimoCambioComercializador = fields.DateTime(allow_none=True)
40+
fechaUltimoMovimientoContrato = fields.DateTime(allow_none=True)
41+
idTipoTitular = fields.String(allow_none=True)
42+
idTitular = fields.String(allow_none=True)
43+
informacionImpagos = fields.String(allow_none=True)
44+
municipioPS = fields.String(allow_none=True)
45+
municipioTitular = fields.String(allow_none=True)
46+
nombreEmpresaDistribuidora = fields.String(allow_none=True)
47+
nombreTitular = fields.String(allow_none=True)
48+
numFincaPS = fields.String(allow_none=True)
49+
numFincaTitular = fields.String(allow_none=True)
50+
pctd = fields.String(allow_none=True)
51+
pisoPS = fields.String(allow_none=True)
52+
pisoTitular = fields.String(allow_none=True)
53+
portalPS = fields.String(allow_none=True)
54+
portalTitular = fields.String(allow_none=True)
55+
presionMedida = fields.String(allow_none=False)
56+
propiedadEquipoMedida = fields.String(validate=OneOf(TAULA_PROPIEDAD_CONTADOR_CORRECTOR) ,allow_none=True)
57+
puertaPS = fields.String(allow_none=True)
58+
puertaTitular = fields.String(allow_none=True)
59+
tipoContador = fields.String(allow_none=True)
60+
tipoCorrector = fields.String(allow_none=True)
61+
tipoPerfilConsumo = fields.String(OneOf(TAULA_PERFIL_CONSUMO), allow_none=True)
62+
tipoViaPS = fields.String(allow_none=True)
63+
tipoViaTitular = fields.String(allow_none=True)
64+
viaPS = fields.String(allow_none=True)
65+
viaTitular = fields.String(allow_none=True)
66+
67+
class CnmcGasMeasuresSchema(Schema):
68+
69+
cups = fields.String(position=0, required=True, allow_none=False)
70+
fechaInicioMesConsumo = fields.DateTime(required=True, position=1, allow_none=False)
71+
fechaFinMesConsumo = fields.DateTime(required=True, position=2, allow_none=False)
72+
codigoTarifaPeaje = fields.String(validate=OneOf(TAULA_TIPO_PEAJE), position=3, required=False, allow_none=True)
73+
consumoEnWhP1 = fields.String(position=4, required=False, allow_none=True)
74+
consumoEnWhP2 = fields.String(position=5, required=False, allow_none=True)
75+
caudalMedioEnWhdia = fields.String(position=6, required=False, allow_none=True)
76+
caudaMinimoDiario = fields.String(position=7, required=False, allow_none=True)
77+
caudaMaximoDiario = fields.String(position=8, required=False, allow_none=True)
78+
porcentajeConsumoNocturno = fields.String(position=9, required=False, allow_none=True)
79+
codigoTipoLectura = fields.String(validate=OneOf(TAULA_TIPO_LECTURA), position=10, required=False, allow_none=True)

‎sippers/parsers/cnmc_gas.py

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
from __future__ import absolute_import
2+
3+
import csv
4+
import StringIO
5+
6+
import pymongo
7+
8+
from sippers import logger
9+
from sippers.utils import build_dict
10+
from sippers.parsers.parser import Parser, register
11+
from sippers.adapters.cnmc_gas import CnmcGasSipsAdapter, CnmcGasMeasuresAdapter
12+
from sippers.models.cnmc_gas import CnmcGasMeasuresSchema
13+
14+
class CnmcGas(Parser):
15+
16+
# En aquest cas els llegim amb el csv.DictReader en comptes de fer-ho amb
17+
# un Schema de marshmallow, ja que el csv pot contenir comes dins d'un
18+
# camp si van wrapped entre cometes i el marshmallow aixo no ho contempla.
19+
20+
# amb csv.DictReader obtindrem un diccionari amb els headers_ps
21+
# que li indiquem aqui, en comptes d'anar-los a buscar a l'Schema.
22+
# El diccionari el podem utilitzar per passar-li al
23+
# self.adapter.load igual que el resultat de l'esquema
24+
25+
pattern = '[0-9]{6}_SIPS2_PS_GAS_[a-z]*_?[a-z]*\.csv'
26+
encoding = "UTF-8"
27+
collection = 'cnmc_sips_gas'
28+
collection_index = 'cups'
29+
index_unic = True
30+
31+
def __init__(self, strict=False):
32+
33+
# l'ordre dels camps segons format cnmc
34+
self.headers_ps = [
35+
'codigoEmpresaDistribuidora',
36+
'nombreEmpresaDistribuidora',
37+
'cups',
38+
'codigoProvinciaPS',
39+
'desProvinciaPS',
40+
'codigoPostalPS',
41+
'municipioPS',
42+
'desMunicipioPS',
43+
'tipoViaPS',
44+
'viaPS',
45+
'numFincaPS',
46+
'portalPS',
47+
'escaleraPS',
48+
'pisoPS',
49+
'puertaPS',
50+
'codigoPresion',
51+
'codigoPeajeEnVigor',
52+
'caudalMaximoDiarioEnWh',
53+
'caudalHorarioEnWh',
54+
'derechoTUR',
55+
'fechaUltimaInspeccion',
56+
'codigoResultadoInspeccion',
57+
'tipoPerfilConsumo',
58+
'codigoContador',
59+
'calibreContador',
60+
'tipoContador',
61+
'propiedadEquipoMedida',
62+
'codigoTelemedida',
63+
'fechaUltimoMovimientoContrato',
64+
'fechaUltimoCambioComercializador',
65+
'informacionImpagos',
66+
'idTipoTitular',
67+
'idTitular',
68+
'nombreTitular',
69+
'apellido1Titular',
70+
'apellido2Titular',
71+
'codigoProvinciaTitular',
72+
'desProvinciaTitular',
73+
'codigoPostalTitular',
74+
'municipioTitular',
75+
'desMunicipioTitular',
76+
'tipoViaTitular',
77+
'viaTitular',
78+
'numFincaTitular',
79+
'portalTitular',
80+
'escaleraTitular',
81+
'pisoTitular',
82+
'puertaTitular',
83+
'esViviendaHabitual',
84+
'cnae',
85+
'tipoCorrector',
86+
'codigoAccesibilidadContador',
87+
'conectadoPlantaSatelite',
88+
'pctd',
89+
'presionMedida'
90+
]
91+
self.adapter = CnmcGasSipsAdapter(strict=strict)
92+
93+
def parse_line(self, line):
94+
95+
# passar previament la linia pel csv reader
96+
# per que agafi be els camps tot i les comes dins del camp direccio
97+
# per fer-ho cal passar-la a StringIO
98+
l = StringIO.StringIO(line)
99+
reader = csv.DictReader(l, fieldnames=self.headers_ps, delimiter=',')
100+
linia = reader.next() # nomes n'hi ha una
101+
102+
parsed = {'ps': {}, 'orig': line, 'collection': self.collection}
103+
result, errors = self.adapter.load(linia)
104+
105+
if errors:
106+
logger.error(errors)
107+
parsed['ps'] = result
108+
return parsed, errors
109+
110+
register(CnmcGas)
111+
112+
class CnmcGasCons(Parser):
113+
114+
# En el cas de les mesures, usem Schema per mantenir el format i
115+
# perque no hi trobarem mes comes que les delimiters
116+
pattern = '[0-9]{6}_SIPS2_CONSUMOS_GAS_[a-z]*_?[a-z]*\.csv'
117+
encoding = "UTF-8"
118+
collection = 'cnmc_sips_consums_gas'
119+
collection_index = [("cups", pymongo.ASCENDING), ("fechaFinMesConsumo", pymongo.DESCENDING)]
120+
index_unic = False
121+
122+
123+
def __init__(self, strict=False):
124+
self.schema = CnmcGasMeasuresSchema(strict=strict)
125+
self.adapter = CnmcGasMeasuresAdapter(strict=strict)
126+
self.measures_adapter = self.adapter
127+
self.fields = []
128+
self.headers = []
129+
for f in sorted(self.schema.fields,
130+
key=lambda f: self.schema.fields[f].metadata['position']):
131+
field = self.schema.fields[f]
132+
self.fields.append((f, field.metadata))
133+
self.headers.append(f)
134+
135+
def parse_line(self, line):
136+
137+
l = StringIO.StringIO(line)
138+
reader = csv.DictReader(l, fieldnames=self.headers, delimiter=',')
139+
linia = reader.next() # nomes n'hi ha una
140+
141+
parsed = {'ps': {}, 'measure_cnmc': [], 'orig': line, 'collection': self.collection}
142+
143+
result, errors = self.adapter.load(linia)
144+
145+
if errors:
146+
logger.error(errors)
147+
148+
parsed['measure_cnmc'] = result
149+
150+
return parsed, errors
151+
152+
register(CnmcGasCons)

‎sippers/parsers/cnmc_v2.py

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
import csv
44
import StringIO
55

6+
import pymongo
7+
68
from sippers import logger
79
from sippers.utils import build_dict
810
from sippers.parsers.parser import Parser, register
@@ -22,7 +24,9 @@ class CnmcV2(Parser):
2224

2325
pattern = '[0-9]{6}_SIPS2_PS_ELECTRICIDAD_[a-z]*_?[a-z]*\.csv'
2426
encoding = "UTF-8"
25-
_collection = 'cnmc_sips'
27+
collection = 'cnmc_sips'
28+
collection_index = 'cups'
29+
index_unic = True
2630

2731
def __init__(self, strict=False):
2832

@@ -97,7 +101,7 @@ def parse_line(self, line):
97101
reader = csv.DictReader(l, fieldnames=self.headers_ps, delimiter=',')
98102
linia = reader.next() # nomes n'hi ha una
99103

100-
parsed = {'ps': {}, 'orig': line, 'collection': self._collection}
104+
parsed = {'ps': {}, 'orig': line, 'collection': self.collection}
101105
result, errors = self.adapter.load(linia)
102106

103107
if errors:
@@ -113,7 +117,9 @@ class CnmcV2Cons(Parser):
113117
# perque no hi trobarem mes comes que les delimiters
114118
pattern = '[0-9]{6}_SIPS2_CONSUMOS_ELECTRICIDAD_[a-z]*_?[a-z]*\.csv'
115119
encoding = "UTF-8"
116-
_collection = 'cnmc_sips_consums'
120+
collection = 'cnmc_sips_consums'
121+
collection_index = [("cups", pymongo.ASCENDING),("fechaFinMesConsumo", pymongo.DESCENDING)]
122+
index_unic = False
117123

118124

119125
def __init__(self, strict=False):
@@ -134,7 +140,7 @@ def parse_line(self, line):
134140
reader = csv.DictReader(l, fieldnames=self.headers, delimiter=',')
135141
linia = reader.next() # nomes n'hi ha una
136142

137-
parsed = {'ps': {}, 'measure_cnmc': [], 'orig': line, 'collection': self._collection}
143+
parsed = {'ps': {}, 'measure_cnmc': [], 'orig': line, 'collection': self.collection}
138144

139145
result, errors = self.adapter.load(linia)
140146

‎sippers/utils.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,3 +69,8 @@ def build_dict(headers, data):
6969
TABLA_64 = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29', '30', '31', '32', '33', '34', '35', '36', '37', '38', '39', '40', '41', '42', '43', '44', '45', '46', '47', '48', '49', '99']
7070
TABLA_108 = ['01','02']
7171
TABLA_111 = ['01', '02', '03']
72+
TAULA_PROPIEDAD_CONTADOR_CORRECTOR = ['04', '06', '01', '02', '11', '12', '08', '10']
73+
TAULA_RESULTADO_INSPECCION = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14']
74+
TAULA_PERFIL_CONSUMO = ['01', '02']
75+
TAULA_TIPO_PEAJE = ['11', '12', '13', '21', '22', '23', '24', '25', '26', '1B', '2B', '3B', '4B', '5B', '6B', '31', '32', '33', '34', '35', '41', '42', '43', '44', '45', '46', '47', 'A1', 'A2', 'A3', 'B1', 'B2', 'C1', 'C2', 'D1', 'D2', 'M1', 'M2', 'R1', 'R2', 'R3', 'R4', 'R8', 'R9', 'L0', 'L1', 'A5', 'A6', 'A7', 'B5', 'B6', 'B7', 'S1', 'S2', 'S3', 'S4', 'S5', 'S6', 'S7', 'S8', '99']
76+
TAULA_TIPO_LECTURA = ['0', '1', '2']

0 commit comments

Comments
 (0)