90% done

Author: adiled

.gitignore

@@ -0,0 +1 @@
+node_modules

README.md

@@ -0,0 +1,11 @@
+
+# Architecture Philosophy
+
+This bot employs a 'Divide and Conquer' strategy.
+
+**What it doesn't do:** Grab all the document numbers first and then scrape them
+
+**What it does:** Grab a bunch of document numbers, suppose 100 from 5 pages. Scrape them, continue where left.
+
+---
+

backlog.json

@@ -0,0 +1 @@
+{"first":{"doc":"SCE0000144093728","page":-1},"last":{"doc":"DCR0000001000809","page":-1}}

index.js

@@ -0,0 +1,101 @@
+var table_name = 'documents'
+
+var fs = require('fs')
+var url = require('url')
+
+var store = require('./store')
+var scraper = require('./scraper')
+var indexer = require('./indexer')
+
+var log = require('./logger')
+
+
+// Load backlog to store
+
+log.load()
+
+
+// Set up data dump file
+
+iso_date = new Date()
+store.data_filename = 'data'+iso_date+'.sql'
+
+sql_insert_query = "INSERT INTO "+table_name+" () VALUES "
+
+try {
+  fs.writeFileSync(store.data_filename, sql_insert_query)
+}
+
+catch(e) {
+  console.log(e)
+  process.exit(1)
+}
+
+
+// Write to dump file when scraper queue gets empty
+// Restart the process
+
+scraper.on('drain',function(){
+  fs.appendFile('message.txt', JSON.stringify(store.documents, null, '\t'), function(err) {
+    if (err) throw err
+      console.log('Data File: Write Complete')
+  })
+
+  console.log(store.backlog.last.doc)
+})
+
+indexer.on('drain', function() {
+  
+})
+
+
+
+/** Queue Pages **/
+
+start_indexer()
+
+
+/** Queue Certificates **/
+
+// Pass uri_list
+
+scraper.queue(['http://www.adene.pt/sce/certificados/SCE0000144093728', 'http://www.adene.pt/sce/certificados/DCR0000001000809'])
+
+
+/** On Exit **/
+
+process.on('exit', function(code) {
+
+  console.log('\n## EXITING - Code '+code+' ##')
+
+  // Save from store
+  log.save()
+});
+
+
+var lib = {
+
+  get_index_uri: function(page_number) {
+    return constants.index.uri+'page='+page_number+'&'+constants.index.querystring
+  },
+
+  start_indexer: function(page_number, offset) {
+
+    page_list = []
+
+    for(i=page_number; i<offset; i++) {
+      page_list.push(get_index_uri(i))
+    }
+
+    indexer.queue(page_list)
+
+  }
+
+}
+
+var constants = {
+  index: {
+    uri: 'http://www.adene.pt/sce/micro/certificados-energeticos?',
+    querystring: 'tipo_cert=Todos&tipo_ed=Todos&morada=&concelho=all&distrito=all&freguesia=all&conservatoria=&conservatoria_nr=&artigo=&fracao=&numero=&op=Pesquisar&form_build_id=form-qpN7d8_HPQqSQJGhFxB024FLI8tBZLX_naofWt_Mwlo&form_id=certificados_webservice_form'
+  }
+}

indexer.js

@@ -0,0 +1,21 @@
+/** INDEXER **/
+var store = require('./store')
+
+var Crawler = require('crawler')
+
+var indexer = new Crawler({
+  maxConnections: 1,
+
+  callback: function(err, res, done) {
+
+    var $ = res.$
+
+    store.doc_uri_list
+
+    done()
+
+  }
+
+})
+
+exports = module.exports = indexer

logger.js

@@ -0,0 +1,65 @@
+var fs = require('fs')
+var store = require('./store')
+
+var log = module.exports = {
+
+  filename: 'backlog.json',
+
+  struct: {
+    first: {
+      doc: null,
+      page: null
+    },  
+    last: {
+      doc: null,
+      page: null
+    }
+  },
+
+  create: function() {
+
+    fs.open(this.filename, 'wx', function(err, fd) {
+      if (err) {
+        if (err.code === "EEXIST") {
+          console.log('[Logger] Log file already exists');
+          return;
+        } else throw err
+      }
+
+      else {
+        fs.writeFile(fd, JSON.stringify(log.struct), function(err) {
+          if(err)
+            console.log('[Log File] Write Failed')
+        })
+      }
+      
+    })
+
+  },
+
+  save: function() {
+
+    try {
+      fs.writeFileSync('./'+this.filename, JSON.stringify(store.backlog))
+    } catch(e) {
+      console.log('[Log File] Dump failed')
+    }
+
+  },
+
+  load: function() {
+
+    try {
+      data = fs.readFileSync('./'+this.filename);
+      store.backlog = JSON.parse(data.toString())
+    }
+
+    catch(err) {
+      throw err
+    }
+
+  }
+}
+
+
+log.create()

message.txt

@@ -0,0 +1,11 @@
+{
+  "name": "adene-bot",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC"
+}

package.json

@@ -0,0 +1,115 @@
+var Crawler = require('crawler')
+var store = require('./store')
+var fs = require('fs')
+
+var scraper = new Crawler({
+  maxConnections: 10,
+
+  callback: function(err, res, done){
+
+    if(err)
+      console.log(err)
+
+    else {
+
+      var $ = res.$
+
+        // Registry Object
+        var ident_string = $('.perito-item-ident').text().trim()
+        var ident = registry_data(ident_string)
+
+        // Energy Class
+        var img_str = $('#cert-image').attr("src").trim()
+        var img_name = img_str.substring(img_str.lastIndexOf("/")+1, img_str.length)
+        var energy_class = img_name.split('.')[0]
+
+        // Verify dates
+        var issue_date = $(".perito-item-data-emissao *")[0].next.data.trim()
+        if(issue_date.length < 10)
+          issue_date = "NULL"
+
+        var expiry_date = $(".perito-item-data-emissao *")[0].next.data.trim()
+        if(expiry_date.length < 10)
+          expiry_date = "NULL"
+
+
+        // Compile final data
+        var doc = {
+          doc_number: $('#doc_certificado strong').text(),
+          type: $(".perito-item:nth-of-type(3)").clone().children().remove().end().text().trim(),
+          info: $('#texto_certificado > span').text().trim(),
+          energy_class: energy_class,
+          address: $(".perito-item:nth-of-type(4)").clone().children().remove().end().text().trim(),
+          location: $(".perito-item-localidade *")[0].next.data.trim(),
+          town: $(".perito-item-freguesia *")[0].next.data.trim(),
+          country: $(".perito-item-concelho *")[0].next.data.trim(),
+          issue_date: $(".perito-item-data-emissao *")[0].next.data.trim(),
+          expiry_date: $(".perito-item-validade *")[0].next.data.trim(),
+          expert_name: $(".perito-item-nome *")[0].next.data.trim(),
+          expert_number: $(".perito-item-cod *")[0].next.data.trim(),
+          reg_name: ident.name,
+          reg_number: ident.number
+        }
+
+        // Add to the SQL insert statement
+        var sql_valstring = '('
+        for(var key in doc) {
+          sql_valstring += "'"+doc[key]+"',"
+        }
+        sql_valstring = sql_valstring.slice(0, -1)
+        sql_valstring += '),'
+
+        try {
+          fs.appendFileSync('./'+store.data_filename, sql_valstring)
+        }
+
+        catch(err) {
+          console.log(err)
+          return 0
+        }
+
+        // Save doc to dump
+        store.documents.push(doc)
+
+        // Mark last saved doc
+        store.backlog.last.doc = doc.doc_number
+
+        // Mark first doc
+        if(!store.flag_first_read) {
+          store.flag_first_read = true
+          store.backlog.first.doc = doc.doc_number
+        }
+
+        done()
+
+      }
+    }
+  })
+
+/** Strip out info from registry information string **/
+
+function registry_data(string) {
+
+  string = string.replace(/\s\s+/g, ' ')
+
+  var reg_name_rx = /Imóvel descrito na Conservatória do (.*?) sob o nº/gi
+  var reg_no_rx = /sob o nº (.*)/gi
+
+  try {
+    var reg_name = reg_name_rx.exec(string)[1]
+    var reg_no = reg_no_rx.exec(string)[1]
+  }
+
+  catch(e) {
+    var reg_name = '-'
+    var reg_no = '-'
+  }
+
+  return {
+    name: reg_name,
+    number: reg_no
+  }
+
+}
+
+exports = module.exports = scraper

scraper.js

@@ -0,0 +1,27 @@
+module.exports = {
+
+  flag_first_read: false,
+  data_filename: '',
+
+  documents: {
+
+    uri_list: [],
+    data: []
+
+  },
+
+  backlog: {
+
+    first: {
+      doc: 0,
+      page: -1
+    },
+    
+    last: {
+      doc: 0,
+      page: -1
+    }
+
+  }
+
+}