Merge pull request #38 from gisce/new_corbagen_format

Soporte para cargar ficheros CORBAGEN y soporte para TimeScaleDB

.github/pull_request_template.md

@@ -0,0 +1,19 @@
+## Objetivos
+
+-
+
+## Comportamiento antiguo
+
+-
+
+## Comportamiento nuevo
+
+- 
+
+## Relacionado
+
+*Puedes ver todas las [posibles palabras para accionesa automáticas con el mensaje](https://help.github.com/articles/closing-issues-using-keywords/)*
+
+## Checklist
+
+- [ ] Test code

.github/workflows/python2.7-app.yml

@@ -0,0 +1,42 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python 2.7
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      # You can use PyPy versions in python-version.
+      # For example, pypy2 and pypy3
+      fail-fast: false
+      matrix:
+        python-version: [ "2.7" ]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Python 2.7
+        run: |
+          sudo apt update
+          sudo apt install python2 python-pip
+          sudo update-alternatives --install /usr/bin/python python /usr/bin/python2 1
+          sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 2
+          printf '1\n' | sudo update-alternatives --config python
+          cd /usr/bin
+          sudo ln -s /usr/bin/pip2 ./pip
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade setuptools
+          pip install coveralls
+          pip install -e .
+          pip install -r requirements-dev.txt
+      - name: Test with mamba
+        run: |
+          mamba --enable-coverage

.github/workflows/python3.11-app.yml

@@ -0,0 +1,33 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python 3.11
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.11"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade setuptools
+        pip install coveralls
+        pip install -e .
+        pip install -r requirements-dev.txt
+        pip install --upgrade mamba
+    - name: Test with mamba
+      run: |
+        mamba --enable-coverage

README.md

@@ -0,0 +1,26 @@
+# CCHLOADER
+
+Eina d'importació de CCH
+
+![](https://github.com/gisce/cchloader/actions/workflows/python2.7-app.yml/badge.svg)
+![](https://github.com/gisce/cchloader/actions/workflows/python2.7-app.yml)
+
+![](https://github.com/gisce/cchloader/actions/workflows/python3.11-app.yml/badge.svg)
+![](https://github.com/gisce/cchloader/actions/workflows/python3.11-app.yml)
+
+- Eina per a importar fitxers de corba a Comercialitzadora i a Representació a mercat..
+- Aporta el coneixement i els formats necessaris per a llegir fitxers en diferents formats.
+
+## Suporta
+- `A5D`
+- `B5D`
+- `CORBAGEN`
+- `F1`
+- `F5D`
+- `MHCIL`
+- `P1`
+- `P1D`
+- `P2`
+- `P2D`
+- `REGANECU`
+- `RF5D`

cchloader/__init__.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 import os
 

cchloader/adapters/__init__.py

@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
 from marshmallow import Schema, post_load
 from marshmallow.decorators import tag_processor
 from cchloader.models import Document

cchloader/adapters/a5d.py

@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
 from cchloader.adapters import CchAdapter
 from cchloader.models.cch_autocons import CchAutoconsSchema
 from marshmallow import Schema, fields, pre_load

cchloader/adapters/b5d.py

@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
 from cchloader.adapters import CchAdapter
 from cchloader.models.cch_gennetabeta import CchGenNetaBetaSchema
 from marshmallow import Schema, fields, pre_load

cchloader/adapters/corbagen.py

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+from __future__ import absolute_import
+
+from cchloader.adapters import CchAdapter
+from cchloader.models.corbagen import CorbaGenSchema
+from marshmallow import Schema, fields, pre_load
+
+
+class CorbaGenBaseAdapter(Schema):
+    """ CORBAGEN Adapter
+    """
+
+    @pre_load
+    def fix_numbers(self, data):
+        for attr, field in self.fields.iteritems():
+            if isinstance(field, fields.Integer):
+                if not data.get(attr):
+                    data[attr] = None
+        return data
+
+    @pre_load
+    def fix_season(self, data):
+        valid_values = [0, 1]
+        season = data.get('season')
+        if season and season.isdigit() and season in map(str, valid_values):
+            data['season'] = int(season)
+        else:
+            data['season'] = None
+
+
+class CorbaGenAdapter(CorbaGenBaseAdapter, CchAdapter, CorbaGenSchema):
+    pass

cchloader/adapters/f1.py

@@ -1,7 +1,11 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
 from cchloader.adapters import CchAdapter
 from cchloader.models.f1 import F1Schema
 from marshmallow import Schema, fields, pre_load
 
+
 class F1BaseAdapter(Schema):
     """ F1 Adapter
     """

cchloader/adapters/f5d.py

@@ -1,7 +1,11 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
 from cchloader.adapters import CchAdapter
 from cchloader.models.cchfact import CchFactSchema
 from marshmallow import Schema, fields, pre_load
 
+
 class F5dBaseAdapter(Schema):
     """ F5D Adapter
     """

cchloader/adapters/mhcil.py

@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
 from cchloader.adapters import CchAdapter
 from cchloader.models.mhcil import MhcilSchema
 from marshmallow import Schema, fields, pre_load

cchloader/adapters/p1.py

@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
 from cchloader.adapters import CchAdapter
 from cchloader.models.p1 import P1Schema
 from marshmallow import Schema, fields, pre_load

cchloader/adapters/p2.py

@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
 from cchloader.adapters import CchAdapter
 from cchloader.adapters.p1 import P1BaseAdapter
 from cchloader.models.p1 import P1Schema

cchloader/adapters/p5d.py

@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
 from cchloader.adapters import CchAdapter
 from cchloader.models.cchval import CchValSchema
 from marshmallow import Schema, fields, pre_load

cchloader/adapters/reganecu.py

@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
 from cchloader.adapters import CchAdapter
 from cchloader.models.reganecu import ReganecuSchema
 from marshmallow import Schema, fields, pre_load

cchloader/backends/__init__.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
 from urlparse import urlparse as std_urlparse
@@ -53,4 +54,5 @@ def get_backend(url):
     return _AVAILABLE_BACKENDS[backend]
 
 # Import Backends
-from cchloader.backends.mongodb import MongoDBBackend
+from cchloader.backends.mongodb import MongoDBBackend
+from cchloader.backends.timescaledb import TimescaleDBBackend

cchloader/backends/base.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
 

cchloader/backends/mongodb.py

@@ -1,8 +1,11 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
 from cchloader.backends import BaseBackend, register, urlparse
-from cchloader.compress import is_compressed_file
 import pymongo
 import datetime
 
+
 class MongoDBBackend(BaseBackend):
     """MongoDB Backend
     """

cchloader/backends/timescaledb.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+from cchloader.backends import BaseBackend, register, urlparse
+import datetime
+import psycopg2
+import pytz
+
+def get_as_utc_timestamp(t):
+    timezone_utc = pytz.timezone("UTC")
+    timezone_local = pytz.timezone("Europe/Madrid")
+    return timezone_utc.normalize(timezone_local.localize(t, is_dst=False))
+
+
+class TimescaleDBBackend(BaseBackend):
+    """TimescaleDB Backend
+    """
+    collections = ['giscedata_corbagen']
+
+    def __init__(self, uri=None):
+        if uri is None:
+            uri = "timescale://localhost:5432/destral_db"
+        super(TimescaleDBBackend, self).__init__(uri)
+
+        self.uri = uri
+        self.config = urlparse(self.uri)
+        ts_con = " host=" + self.config['hostname'] + \
+                " port=" + str(self.config['port']) + \
+                " dbname=" + self.config['db'] + \
+                " user=" + self.config['username'] + \
+                " password=" + self.config['password']
+        self.db = psycopg2.connect(ts_con)
+        self.cr = self.db.cursor()
+
+
+    def insert(self, document):
+        for collection in document.keys():
+            if collection in self.collections:
+                cch = document.get(collection)
+                if cch:
+                    cch.backend = self
+                    cch.collection = collection
+                    self.insert_cch(cch)
+
+
+    def insert_cch(self, cch):
+        collection = cch.collection
+        document = cch.backend_data
+        document.update({
+            'create_at': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+            'update_at': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+            'utc_timestamp': get_as_utc_timestamp(document['datetime']).strftime('%Y-%m-%d %H:%M:%S')
+        })
+        if 'validated' in document and type(document['validated']) == bool:
+            document['validated'] = 1 if document['validated'] else 0
+
+        if 'datetime' in document and type(document['datetime']) == datetime.datetime:
+            document['datetime'] = document['datetime'].strftime('%Y-%m-%d %H:%M:%S')
+
+        if 'name' in document and type(document['name']) == type(u''):
+            document['name'] = document['name'].encode('utf-8')
+
+        placeholders = ', '.join(['%s'] * len(document))
+        columns = ', '.join(document.keys())
+        sql = "INSERT INTO %s ( %s ) VALUES ( %s ) RETURNING *" % (collection, columns, placeholders)
+        self.cr.execute(sql, document.values())
+        oid = self.cr.fetchone()[0]
+        self.db.commit()
+        return oid
+
+    def get(self, collection, filters, fields=None):
+        raise Exception("Not implemented cchloader.backend.timescale.get()")
+
+    def disconnect(self):
+        self.db = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.disconnect()
+
+
+register("timescale", TimescaleDBBackend)

cchloader/cli/__init__.py

@@ -1,10 +1,12 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
 import click
 from cchloader.file import CchFile, PackedCchFile
 from cchloader.backends import get_backend
 from cchloader.compress import is_compressed_file
 
+
 @click.group()
 def cchloader():
     pass

cchloader/compress.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
 
 import struct
 import zipfile
 import gzip
 import bz2
 
+
 class CompressedFile (object):
     magic = None
     file_type = None

cchloader/exceptions.py

@@ -1,2 +1,3 @@
+# -*- coding: utf-8 -*-
 class ParserNotFoundException(Exception):
     pass