diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..91ed15d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.dockerignore +Dockerfile* +docker-compose* +/build +/.cache +/log +/tmp +/var +.git +.idea +.swp +tags diff --git a/.env b/.env index 2531ff1..8cb4b7e 100644 --- a/.env +++ b/.env @@ -1,3 +1,4 @@ URL_BASE=http://localhost:8001 KEY=jajaparaquequieressabereso123! -PHANTOMJS=phantomjs \ No newline at end of file +PHANTOMJS=phantomjs +# APP_BOT_NAME=odiometro diff --git a/.gitignore b/.gitignore index e0d580f..75c5d93 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ node_modules public/node_modules public/.sass-cache -config/database_odiometro.json config/database_*.json -config/twitter_*.json \ No newline at end of file +config/twitter_*.json +var +.DS_Store \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6ade291 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +## Dockerfile for building development image +FROM node:14.12-alpine3.12 +LABEL maintainer "Igor Pellegrini " + +ENV NODE_ENV=development + +# Install system dependencies +# bzip2 to install phantomjs; bash to run entrypoint +USER root +RUN apk update && apk add --no-cache --upgrade bash bzip2 + +WORKDIR /home/node/app +RUN chown node:node -R /home/node/app + +# copy custom entrypoint + related functions +COPY --chown=node:node ./docker/bash_functions.sh /scripts/bash_functions.sh +COPY --chown=node:node ./docker/entrypoint.sh /usr/local/bin/docker-entrypoint.sh + +# copy installation manifests fist (to allow caching NPM installation) +COPY --chown=node:node ./package.json /home/node/app/package.json +COPY --chown=node:node ./public/package.json /home/node/app/public/package.json + +# set user to run NPM scripts +# RUN npm config set user node + +# Install application +USER node + +RUN npm install +RUN cd public && npm install + +# copy the rest of the application code +COPY --chown=node:node . /home/node/app + +# RUN ./home/node/app/public/node_modules/grunt/bin/grunt + +EXPOSE $PORT + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +CMD ["/usr/local/bin/docker-entrypoint.sh", "npm", "start"] diff --git a/Dockerfile-database b/Dockerfile-database new file mode 100644 index 0000000..836ebd1 --- /dev/null +++ b/Dockerfile-database @@ -0,0 +1,24 @@ +## Dockerfile for building development image +FROM mariadb:10.4-focal +LABEL maintainer "Igor Pellegrini " + +USER root + +# install netcat to run the `wait-for-mysql` readyness probe function +RUN apt-get -q update \ + && apt-get -q --yes install netcat \ + && apt-get clean + +RUN chown -R mysql:mysql /var/lib/mysql + +COPY --chown=mysql:mysql ./db/app-schema.sql /schemas/app-schema.sql +COPY --chown=mysql:mysql ./docker/bash_functions.sh /scripts/bash_functions.sh + +# initialize the project database, when running container (create database, import schema, run migrations) +# MySql/MariaDB images support putting files into `docker-entrypoint-initdb.d/` +# to load *.sql or execute initialization *.sh scripts. +COPY --chown=mysql:mysql ./docker/init-from-DB_NAMES.sh /docker-entrypoint-initdb.d/init-from-DB_NAMES.sh + +USER mysql + +# CMD ["--user=root","--explicit_defaults_for_timestamp"] \ No newline at end of file diff --git a/README.md b/README.md index 1af39a0..e60ae4c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -ODIÓMETRO -================================ +# ODIÓMETRO +--- Ya disponible en https://odiometro.es @@ -11,58 +11,75 @@ y el amplio uso de insultos, descalificaciones, etc. que se producen en Twitter. La web hace uso de las siguientes tecnologías: * NodeJS / Express * Socket.io -* MySQL +* MySQL (if having problems, stick to 5.x; see [mysqljs#1962](https://github.com/mysqljs/mysql/pull/1962)) * VueJS * SASS * Grunt +* *(opcional)* Docker +## Instalación +--- +1. Clonar el repositorio +```git clone git@github.com:ojoven/odiometro.git``` -Instalación -------------- +### **Docker setup:** +Para instalar en un contenedor de Docker… -1. Clonar el repositorio +2. Configurar las credenciales de la [API de Twitter](https://developer.twitter.com/en/docs/twitter-api/getting-started/guide) con las environment variables `APP_TWITTER_`, en ``docker-compose.yml``. -```git clone git@github.com:ojoven/odiometro.git``` +3. En *docker-compose.yml*, asegúrate de que `INIT_DB_NAMES` lista los nombres de la base de datos a inicializar, separados por una coma (por ejemplo, `INIT_DB_NAMES=odiometro,amorometro`). -2. Desde la raíz del proyecto (instalará express, socket y otras librerías) +4. Ejecute `docker-compose up` desde el directorio raíz del proyecto. +**NOTE:** Necesitas tener [Docker](https://www.docker.com/get-started) ya instalado. Si tienes problemas, [abre un issue](https://github.com/ojoven/odiometro/issues) o intenta la configuración manual. -```npm install``` +> Por defecto, el `docker-compose.yml` está configurado para inicializar: +> 1 contenedor de base de datos (compartido entre instancias) +> 1..N contenedores, para cada versión de la aplicación (odómetro, amorómetro u otros). +> Comente los servicios no deseados, o configúrelos para su versión personalizada. +> Cada instancia de la aplicación puede ser alcanzada en una porta diferente (3001, 3002; see `PORT` env variable). -3. También desde la carpeta `/public` (instalará grunt y plugins, vue) +5. Accede a la aplicación de *odiometro* en http://localhost:8081, y a la aplicación de *amorometro* en http://localhost:8082 -```cd public && npm install``` +**Nota:** Los file de configuración correspondientes a cada bot deben existir; por ejemplo, para *odiometro* crea `database_odiometro.json` y `twitter_odiometro.json` y `odiometro.json`. Los dos primeros pueden dejarse vacíos (`{}`), cuando se utilizan variables del sistema. -4. Crea una base de datos e importa el dump en `db/odiometro.sql` +### **Local setup:** +De lo contrario, siga los pasos a continuación para instalar en la máquina local. -```mysql -u [username] -p [dbname] < db/odiometro.sql``` +2. Configurar las credenciales de la [API de Twitter](https://developer.twitter.com/en/docs/twitter-api/getting-started/guide) en `/config/twitter_odiometro.json` (o `twitter_[dbname].json`) -4.2 Configura los datos de tu DB en /config/database_odiometro.json o si estás creando el odiómetro en tu país database_odiometro.pais.json +3. Desde la raíz del proyecto (instalará express, socket y otras librerías) +```npm install``` -```mysql -u [username] -p [dbname] < db/odiometro.sql``` +4. También desde la carpeta `/public` (instalará grunt y plugins, vue) +```cd public && npm install``` -5. Corre el grunt desde /public +5. Crear una base de datos y configurar la aplicación para acceder a ella -```grunt``` +5.2 Configure les credenciales de tu DB desde `/config/database_odiometro.json`; o `database_[dbname].pais.json`, si estás creando el odiómetro en tu país) -6. Lanza la app +5.3 Importar el esquema a su base de datos +```mysql -u [username] -p [dbname] < db/app-schema.sql``` -```node app.js``` +6. Corre el grunt desde */public* +```grunt``` -7. Si has creado tu propia versión de país (o de otro estilo tipo amorómetro) +7. Lanza la app +```node app.js``` -```node app.js [nombredebot]``` +8. Si has creado tu propia versión de país (o de otro estilo tipo amorómetro) +```node app.js [nombredebot]```. Alternativamente, configura la variable de sistema `APP_BOT_NAME`, en *.env*. -IMPORTANTE -------------------------------- +### IMPORTANTE +--- Tendrás que crear una app en Twitter y rellenar los datos de consumer key y secret, y access_token y secret en twitter_odiometro.json o twitter_odiometro.pais.json. No sé bien cómo funciona la API de Twitter ahora, ya que estuvieron implementando bastantes restricciones que igual no afectan a apps antiguas pero sí a las nuevas. Si tienes algún problema o sabes cómo va, no dudes en escribir en las Issues. -Notas adicionales de desarrollo -------------------------------- +## Notas adicionales de desarrollo +--- * Los componentes Vue del frontend se encuentran en `js/src/app/components` * Otros archivos interesantes son: @@ -76,6 +93,6 @@ Notas adicionales de desarrollo -Contacta ------------- +## Contacta +--- Si tienes alguna duda, sugerencia, crítica o palabra de odio, puedes contactarme en https://twitter.com/ojoven diff --git a/app.js b/app.js index 7bbf822..10b38be 100644 --- a/app.js +++ b/app.js @@ -1,3 +1,4 @@ +require('dotenv').config(); /* |-------------------------------------------------------------------------- | INITIALIZATION @@ -6,13 +7,12 @@ // Bot var args = process.argv.slice(2); -var defaultBot = 'odiometro'; +var defaultBot = process.env.APP_BOT_NAME || 'odiometro'; var bot = (args && typeof args[0] !== "undefined") ? args[0] : defaultBot; // Path and ENV var path = require('path'); global.appRoot = path.resolve(__dirname); -require('dotenv').config(); global.urlBase = process.env.URL_BASE; global.phantomJsBin = process.env.PHANTOMJS; global.botName = bot; @@ -23,6 +23,7 @@ var express = require('express'), app = express(); var port = process.env.PORT || global.botConfig.port; +var external_port = process.env.EXTERNAL_PORT || port; var io = require('socket.io').listen(app.listen(port)); require('./config')(app, io); @@ -45,7 +46,7 @@ var Tweet = require("./app/models/Tweet.js"); var Historic = require("./app/models/Historic.js"); // Logging -console.log('Odiometro is running on http://localhost:' + port); +console.log('Odiometro is running on http://localhost:' + external_port); // Vars var numberTweets, mostHatedUser, mostHatedUsersLastTweet, mostHatefulUser, mostHatefulUserTweet, mostHatefulUserTweetId; diff --git a/app/lib/database.js b/app/lib/database.js index d98fe8b..cad4b27 100644 --- a/app/lib/database.js +++ b/app/lib/database.js @@ -4,6 +4,7 @@ var mysql = require('mysql'); var Tweet = require("../models/Tweet.js"); // DB helper +var databaseName = process.env.APP_DATABASE_NAME || dbConfig.database || 'odiometro'; var numTweetsToSave = 5; var numRetweetsToSave = 10; var numTweetsToStore = 50; @@ -12,17 +13,20 @@ var retweetsToSave = []; var tweetsToStore = []; var database = { - connection: mysql.createConnection({ - host: dbConfig.host, - user: dbConfig.user, - password: dbConfig.password, - database: dbConfig.database + pool: mysql.createPool({ + connectionLimit : 15, + // fallback chain for the database config + // - prefer environment variables, when running on docker (see docker-compose.yml) + // - precedence is given to local config file, if present under config/ folder + host: process.env.APP_DATABASE_HOST || dbConfig.host, + user: process.env.APP_DATABASE_USER || dbConfig.user, + password: process.env.APP_DATABASE_PASSWORD || dbConfig.password, + database: databaseName, + charset: process.env.APP_DATABASE_CONNECTION_CHARSET || dbConfig.charset || 'utf8mb4' }) }; database.initialize = function () { - - this.connection.connect(); }; /** TWEETS **/ @@ -37,16 +41,16 @@ database.saveTweet = function (tweet) { var values = ''; tweetsToSave.forEach(function (tweet, index) { var tweetText = that.escapeSingleQuotes(tweet.text); - values += '(null, \'' + tweetText + '\', \'' + tweet.id_str + '\', \'' + tweet.user.screen_name + '\', \' ' + database.currentDateTimeInMySQLFormat() + ' \')'; + values += '(null, \'' + tweetText + '\', \'' + tweet.id_str + '\', \'' + tweet.user.screen_name + '\', \'{}\', \' ' + database.currentDateTimeInMySQLFormat() + ' \')'; if (index !== tweetsToSave.length - 1) values += ','; }); tweetsToSave = []; - var query = 'INSERT INTO ' + dbConfig.database + '.tweets VALUES' + values; + var query = 'INSERT INTO ' + databaseName + '.tweets VALUES' + values; console.log('Save last ' + numTweetsToSave + ' tweets'); - this.connection.query(query, function (error, results, fields) { + this.pool.query(query, function (error, results, fields) { if (error) { console.log(error); throw error; @@ -68,16 +72,16 @@ database.saveRetweet = function (tweet) { var values = ''; retweetsToSave.forEach(function (tweet, index) { var retweetText = that.escapeSingleQuotes(tweet.retweeted_status.text); - values += '(null, \'' + tweet.retweeted_status.id_str + '\', \'' + tweet.retweeted_status.user.screen_name + '\', \'' + retweetText + '\', \' ' + database.currentDateTimeInMySQLFormat() + ' \')'; + values += '(null, \'' + tweet.retweeted_status.id_str + '\', \'' + tweet.retweeted_status.user.screen_name + '\', \'' + retweetText + '\', \'{}\', \' ' + database.currentDateTimeInMySQLFormat() + ' \')'; if (index !== retweetsToSave.length - 1) values += ','; }); retweetsToSave = []; - var query = 'INSERT INTO ' + dbConfig.database + '.retweets VALUES' + values; + var query = 'INSERT INTO ' + databaseName + '.retweets VALUES' + values; console.log('Save last ' + numRetweetsToSave + ' retweets'); - this.connection.query(query, function (error, results, fields) { + this.pool.query(query, function (error, results, fields) { if (error) { console.log(error); throw error; @@ -94,11 +98,11 @@ database.getNumberOfTweetsInLastMinute = function (callback) { var numberOfTweets = 0; var timeInMinutes = 1; var dateMysql = this.getDateTimeInMySQLFormatXMinutesAgo(timeInMinutes); - that.connection.query('SELECT COUNT(*) AS count FROM tweets WHERE published >= \'' + dateMysql + '\'', function (error, results, fields) { + that.pool.query('SELECT COUNT(*) AS count FROM tweets WHERE published >= \'' + dateMysql + '\'', function (error, results, fields) { if (results !== undefined) { numberOfTweets = results[0].count; - that.connection.query('SELECT COUNT(*) AS count FROM retweets WHERE published >= \'' + dateMysql + '\'', function (error, results, fields) { + that.pool.query('SELECT COUNT(*) AS count FROM retweets WHERE published >= \'' + dateMysql + '\'', function (error, results, fields) { numberOfTweets += results[0].count; callback(numberOfTweets); @@ -113,8 +117,8 @@ database.getNumberOfTweetsInLastMinute = function (callback) { database.getLastTweetFromDatabase = function (callback) { - var query = 'SELECT * FROM ' + dbConfig.database + '.tweets ORDER BY published DESC LIMIT 0,1'; - this.connection.query(query, function (error, results, fields) { + var query = 'SELECT * FROM ' + databaseName + '.tweets ORDER BY published DESC LIMIT 0,1'; + this.pool.query(query, function (error, results, fields) { var lastTweet = results[0]; callback(lastTweet); }); @@ -127,9 +131,9 @@ database.cleanOldData = function (timeInMinutes) { var dateMysql = this.getDateTimeInMySQLFormatXMinutesAgo(timeInMinutes); // Clean old data from the different tables - this.connection.query('DELETE FROM ' + dbConfig.database + '.tweets WHERE published < \'' + dateMysql + '\''); - this.connection.query('DELETE FROM ' + dbConfig.database + '.retweets WHERE published < \'' + dateMysql + '\''); - this.connection.query('DELETE FROM ' + dbConfig.database + '.users WHERE published < \'' + dateMysql + '\''); + this.pool.query('DELETE FROM ' + databaseName + '.tweets WHERE published < \'' + dateMysql + '\''); + this.pool.query('DELETE FROM ' + databaseName + '.retweets WHERE published < \'' + dateMysql + '\''); + this.pool.query('DELETE FROM ' + databaseName + '.users WHERE published < \'' + dateMysql + '\''); }; @@ -180,8 +184,8 @@ database.saveTweetStore = function (tweet) { tweetsToStore = []; - var query = 'INSERT INTO ' + dbConfig.database + '.tweets_store VALUES' + values; - this.connection.query(query); + var query = 'INSERT INTO ' + databaseName + '.tweets_store VALUES' + values; + this.pool.query(query); console.log('Store last ' + numTweetsToStore + ' tweets'); } @@ -192,7 +196,7 @@ database.saveRetweetStore = function (retweet) { var retweetStore = Tweet.parseRetweetForStore(retweet); - var query = 'INSERT INTO ' + dbConfig.database + '.retweets_store VALUES(null, ' + + var query = 'INSERT INTO ' + databaseName + '.retweets_store VALUES(null, ' + '\'' + retweetStore.id_str + '\', ' + '\'' + retweetStore.user_id_str + '\', ' + '\'' + retweetStore.user_screen_name + '\', ' + @@ -200,7 +204,7 @@ database.saveRetweetStore = function (retweet) { '\'' + database.currentDateTimeInMySQLFormat() + '\'' + ')'; - this.connection.query(query); + this.pool.query(query); } /** USERS **/ @@ -210,7 +214,7 @@ database.saveUsers = function (users) { var that = this; users.forEach(function (user) { - that.connection.query('INSERT INTO ' + dbConfig.database + '.users VALUES(null, \'' + user + '\', \' ' + database.currentDateTimeInMySQLFormat() + ' \')'); + that.pool.query('INSERT INTO ' + databaseName + '.users VALUES(null, \'' + user + '\', \' ' + database.currentDateTimeInMySQLFormat() + ' \')'); }); }; @@ -219,7 +223,7 @@ database.saveUsers = function (users) { database.getUserImage = function (user, callback) { var query = 'SELECT * FROM `user_images` WHERE screen_name = \'' + user + '\''; - this.connection.query(query, function (error, results, fields) { + this.pool.query(query, function (error, results, fields) { if (!results) { callback(false); @@ -234,7 +238,7 @@ database.saveUserImage = function (user, userImage) { if (!user || !userImage) return; - this.connection.query('INSERT INTO ' + dbConfig.database + '.user_images VALUES(null, \'' + user + '\', \'' + userImage + '\', \' ' + database.currentDateTimeInMySQLFormat() + ' \')'); + this.pool.query('INSERT INTO ' + databaseName + '.user_images VALUES(null, \'' + user + '\', \'' + userImage + '\', \' ' + database.currentDateTimeInMySQLFormat() + ' \')'); }; @@ -243,7 +247,7 @@ database.getMostHatedUser = function (callback) { var timeInMinutes = 1; var dateMysql = this.getDateTimeInMySQLFormatXMinutesAgo(timeInMinutes); var query = 'SELECT `user`, COUNT(`user`) AS `user_occurrence` FROM `users` WHERE published > \'' + dateMysql + '\' GROUP BY `user` ORDER BY `user_occurrence` DESC LIMIT 1'; - this.connection.query(query, function (error, results, fields) { + this.pool.query(query, function (error, results, fields) { if (!results) { callback(false); @@ -260,7 +264,7 @@ database.getMostHatedUserExampleMultipleTweets = function (user, callback) { // FIRST WE RETRIEVE THE EXAMPLE FROM SIMPLE TWEETS var query = 'SELECT * FROM `tweets_store` WHERE text LIKE \'%' + user + '%\' ORDER BY `published` DESC LIMIT 10'; - this.connection.query(query, function (error, results, fields) { + this.pool.query(query, function (error, results, fields) { if (results) { @@ -281,7 +285,7 @@ database.getMostHatedUserExampleTweet = function (user, callback) { // FIRST WE RETRIEVE THE EXAMPLE FROM SIMPLE TWEETS var query = 'SELECT * FROM `tweets` WHERE tweet LIKE \'%' + user + '%\' ORDER BY `published` DESC LIMIT 1'; - this.connection.query(query, function (error, results, fields) { + this.pool.query(query, function (error, results, fields) { if (results) { @@ -292,7 +296,7 @@ database.getMostHatedUserExampleTweet = function (user, callback) { // IF NO RESULTS WITH TWEETS, WE SEARCH ON RETWEETS var query = 'SELECT * FROM `retweets` WHERE retweeted_text LIKE \'%' + user + '%\' ORDER BY `published` DESC LIMIT 1'; - that.connection.query(query, function (error, results, fields) { + that.pool.query(query, function (error, results, fields) { if (results) { var tweet = results[0]; @@ -315,7 +319,7 @@ database.getMostHatefulUserAndTweet = function (callback) { // FIRST WE SEARCH FOR THE HATEFUL USER ON RETWEETS, AS IT'S ON THEM WHERE THE INFLUENCE HAPPENS var query = 'SELECT `retweeted_user` AS `user`, `retweeted_id` AS `id_str`, `retweeted_text` AS `text`, COUNT(`id`) AS `user_occurrence` FROM `retweets` WHERE retweeted_text LIKE \'%@%\' AND published > \'' + dateMysql + '\' GROUP BY `retweeted_user` ORDER BY `user_occurrence` DESC LIMIT 1'; - that.connection.query(query, function (error, results, fields) { + that.pool.query(query, function (error, results, fields) { if (results) { var user = results[0]; @@ -325,7 +329,7 @@ database.getMostHatefulUserAndTweet = function (callback) { // IF NO RETWEETS WITH HATE, WE SEARCH ON SIMPLE TWEETS var query = 'SELECT `screen_name` AS `user`, `id_str`, `tweet` AS `text`, COUNT(`id`) AS `user_occurrence` FROM `tweets` WHERE tweet LIKE \'%@%\' AND published > \'' + dateMysql + '\' GROUP BY `screen_name` ORDER BY `user_occurrence` DESC LIMIT 1'; - that.connection.query(query, function (error, results, fields) { + that.pool.query(query, function (error, results, fields) { if (results) { var user = results[0]; @@ -345,7 +349,7 @@ database.getHistoricData = function (dateStart, dateEnd, callback) { '`hateful_user`, `hateful_user_tweet_text`, `hateful_user_tweet_id`, `date`' + 'FROM `historic` WHERE date >= \'' + dateStart + '\' AND date <= \'' + dateEnd + '\' ORDER BY `date` ASC'; - this.connection.query(query, function (error, results, fields) { + this.pool.query(query, function (error, results, fields) { callback(results); }); @@ -358,7 +362,7 @@ database.getHistoricNumberTweets = function (callback) { var query = 'SELECT `id`, `number_tweets`, `date`' + 'FROM `historic` ORDER BY `date` DESC LIMIT 0, ' + limit; - this.connection.query(query, function (error, results, fields) { + this.pool.query(query, function (error, results, fields) { callback(results); }); @@ -373,8 +377,8 @@ database.saveHistoricData = function ( hatedUserExampleTweetText = this.escapeSingleQuotes(hatedUserExampleTweetText); hatefulUserTweetText = this.escapeSingleQuotes(hatefulUserTweetText); - this.connection.query( - 'INSERT INTO ' + dbConfig.database + '.historic VALUES(null, \'' + numberTweets + '\', \'' + + this.pool.query( + 'INSERT INTO ' + databaseName + '.historic VALUES(null, \'' + numberTweets + '\', \'' + hatedUser + '\', \'' + hatedUserExampleTweetText + '\', \'' + hatedUserExampleTweetId + '\', \'' + hatedUserExampleTweetUser + '\', \'' + hatefulUser + '\', \'' + hatefulUserTweetText + '\', \'' + hatefulUserTweetId + '\', \' ' + database.currentDateTimeInMySQLFormat() + ' \')', diff --git a/config/database_odiometro.pais.json.example b/config/database_odiometro.pais.json.example index 3c13377..85f9af1 100644 --- a/config/database_odiometro.pais.json.example +++ b/config/database_odiometro.pais.json.example @@ -1,6 +1,6 @@ { - "host": "---", - "user": "---", - "password": "---", - "database": "---" + "host": "database", + "user": "root", + "password": "mysql_pass", + "database": "odiometro" } \ No newline at end of file diff --git a/db/odiometro.sql b/db/app-schema.sql similarity index 94% rename from db/odiometro.sql rename to db/app-schema.sql index 727fd2d..06fffeb 100644 --- a/db/odiometro.sql +++ b/db/app-schema.sql @@ -17,16 +17,18 @@ SET time_zone = "+00:00"; /*!40101 SET NAMES utf8mb4 */; -- --- Base de datos: `odiometro` +-- Base de datos: `amorometro` -- +-- CREATE DATABASE IF NOT EXISTS `amorometro` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci + -- -------------------------------------------------------- -- -- Estructura de tabla para la tabla `historic` -- -CREATE TABLE `historic` ( +CREATE TABLE IF NOT EXISTS `historic` ( `id` int(11) NOT NULL, `number_tweets` int(5) NOT NULL, `hated_user` varchar(32) COLLATE utf8mb4_unicode_ci DEFAULT NULL, @@ -45,7 +47,7 @@ CREATE TABLE `historic` ( -- Estructura de tabla para la tabla `retweets` -- -CREATE TABLE `retweets` ( +CREATE TABLE IF NOT EXISTS `retweets` ( `id` int(11) NOT NULL, `retweeted_id` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL, `retweeted_user` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL, @@ -60,7 +62,7 @@ CREATE TABLE `retweets` ( -- Estructura de tabla para la tabla `retweets_store` -- -CREATE TABLE `retweets_store` ( +CREATE TABLE IF NOT EXISTS `retweets_store` ( `id` int(11) NOT NULL, `id_str` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL, `user_id_str` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL, @@ -75,7 +77,7 @@ CREATE TABLE `retweets_store` ( -- Estructura de tabla para la tabla `tweets` -- -CREATE TABLE `tweets` ( +CREATE TABLE IF NOT EXISTS `tweets` ( `id` int(11) NOT NULL, `tweet` mediumtext COLLATE utf8mb4_unicode_ci NOT NULL, `id_str` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL, @@ -90,7 +92,7 @@ CREATE TABLE `tweets` ( -- Estructura de tabla para la tabla `tweets_store` -- -CREATE TABLE `tweets_store` ( +CREATE TABLE IF NOT EXISTS `tweets_store` ( `id` int(11) NOT NULL, `id_str` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL, `text` mediumtext COLLATE utf8mb4_unicode_ci NOT NULL, @@ -117,7 +119,7 @@ CREATE TABLE `tweets_store` ( -- Estructura de tabla para la tabla `users` -- -CREATE TABLE `users` ( +CREATE TABLE IF NOT EXISTS `users` ( `id` int(11) NOT NULL, `user` varchar(32) COLLATE utf8_bin NOT NULL, `published` datetime NOT NULL @@ -129,7 +131,7 @@ CREATE TABLE `users` ( -- Estructura de tabla para la tabla `user_images` -- -CREATE TABLE `user_images` ( +CREATE TABLE IF NOT EXISTS `user_images` ( `id` int(11) NOT NULL, `screen_name` varchar(32) NOT NULL, `image_url` varchar(256) NOT NULL, diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c805b73 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,76 @@ +--- +version: '2' + +services: + database: + tty: true + environment: + ### Specify the names of the databases to create + # (corresponding to the app bot-names) + - INIT_DB_NAMES=odiometro,amorometro + ### Set default credentials + - MYSQL_ROOT_USER=root + - MYSQL_ROOT_PASSWORD=mysql_pass + # - ALLOW_EMPTY_PASSWORD=yes + build: + context: . + dockerfile: ./Dockerfile-database + # volumes: + ### Backup databases to host machine + # (remember: init script runs just when /var/lib/mysql is empty; + # try to delete the folder, if incurring in problems) + # - ./var/mysql:/var/lib/mysql + ### MySql image supports docker-entrypoint-initdb.d/ to load .sql or .sh, + # - ./init-from-DB_NAMES.sh:/docker-entrypoint-initdb.d/init-from-DB_NAMES.sh + ### Cache apt-get (for debug purposes) + # - ./var/apt/cache:/var/cache/apt/archives + + odiometro: + tty: true # Enables debugging capabilities when attached to this container. + environment: + - PORT=8001 + - EXTERNAL_PORT=8081 + - NODE_ENV=development + - APP_BOT_NAME=odiometro + - APP_URL=http://localhost:8081 + # - APP_DATABASE_URL=mysql://root:mysql_pass@database/odiometro + - APP_DATABASE_HOST=database + - APP_DATABASE_NAME=odiometro + - APP_DATABASE_USER=root + - APP_DATABASE_PASSWORD=mysql_pass + working_dir: /home/node/app + build: . + depends_on: + - database + ports: + - 8081:8001 # external:docker + volumes: + - ./:/home/node/app + - /home/node/app/node_modules # DO NOT remove: maintains a volume for node_modules/ dir, + # even when mounting over it (from host to /home/node/app dir) + # (otherwise is overridden and fails with MODULE_NOT_FOUND) + + amorometro: + tty: true # Enables debugging capabilities when attached to this container. + environment: + - PORT=8002 + - EXTERNAL_PORT=8082 + - NODE_ENV=development + - APP_BOT_NAME=amorometro + - APP_URL=http://localhost:8082 + # - APP_DATABASE_URL=mysql://root:mysql_pass@database/amorometro + - APP_DATABASE_HOST=database + - APP_DATABASE_NAME=amorometro + - APP_DATABASE_USER=root + - APP_DATABASE_PASSWORD=mysql_pass + working_dir: /home/node/app + build: . + depends_on: + - database + ports: + - 8082:8002 # external:docker + volumes: + - ./:/home/node/app + - /home/node/app/node_modules # DO NOT remove: maintains a volume for node_modules/ dir, + # even when mounting over it (from host to /home/node/app dir) + # (otherwise is overridden and fails with MODULE_NOT_FOUND) diff --git a/docker/bash_functions.sh b/docker/bash_functions.sh new file mode 100644 index 0000000..9d673cb --- /dev/null +++ b/docker/bash_functions.sh @@ -0,0 +1,176 @@ + +# see master/4/debian-10/prebuildfs/opt/bitnami/base/functions +# from github.com/bitnami/bitnami-docker-express +[[ ${BASH_DEBUG:-false} = true ]] && set -x + +# Constants +MODULE="$(basename "$0")" +#!/bin/bash +# +# Library for logging functions + +# Constants +RESET='\033[0m' +RED='\033[38;5;1m' +GREEN='\033[38;5;2m' +YELLOW='\033[38;5;3m' +MAGENTA='\033[38;5;5m' +CYAN='\033[38;5;6m' + +# Functions + +######################## +# Print to STDERR +# Arguments: +# Message to print +# Returns: +# None +######################### +stderr_print() { + # 'is_boolean_yes' is defined in libvalidations.sh, but depends on this file so we cannot source it + local bool="${BITNAMI_QUIET:-false}" + # comparison is performed without regard to the case of alphabetic characters + shopt -s nocasematch + if ! [[ "$bool" = 1 || "$bool" =~ ^(yes|true)$ ]]; then + printf "%b\\n" "${*}" >&2 + fi +} + +######################## +# Log message +# Arguments: +# Message to log +# Returns: +# None +######################### +log() { + stderr_print "${CYAN}${MODULE:-} ${MAGENTA}$(date "+%T.%2N ")${RESET}${*}" +} +######################## +# Log an 'info' message +# Arguments: +# Message to log +# Returns: +# None +######################### +info() { + log "${GREEN}INFO ${RESET} ==> ${*}" +} +######################## +# Log message +# Arguments: +# Message to log +# Returns: +# None +######################### +warn() { + log "${YELLOW}WARN ${RESET} ==> ${*}" +} +######################## +# Log an 'error' message +# Arguments: +# Message to log +# Returns: +# None +######################### +error() { + log "${RED}ERROR${RESET} ==> ${*}" +} +######################## +# Log a 'debug' message +# Globals: +# BITNAMI_DEBUG +# Arguments: +# None +# Returns: +# None +######################### +debug() { + # 'is_boolean_yes' is defined in libvalidations.sh, but depends on this file so we cannot source it + local bool="${BITNAMI_DEBUG:-false}" + # comparison is performed without regard to the case of alphabetic characters + shopt -s nocasematch + if [[ "$bool" = 1 || "$bool" =~ ^(yes|true)$ ]]; then + log "${MAGENTA}DEBUG${RESET} ==> ${*}" + fi +} + +######################## +# Indent a string +# Arguments: +# $1 - string +# $2 - number of indentation characters (default: 4) +# $3 - indentation character (default: " ") +# Returns: +# None +######################### +indent() { + local string="${1:-}" + local num="${2:?missing num}" + local char="${3:-" "}" + # Build the indentation unit string + local indent_unit="" + for ((i = 0; i < num; i++)); do + indent_unit="${indent_unit}${char}" + done + echo "$string" | sed "s/^/${indent_unit}/" +} + + + +######################## +# Wait for service to be reachable +# Arguments: +# $1 - host (default: localhost) +# $2 - port (default: 3306) +# $3 - maximum number of retries (default: 10) +# Returns: +# None +######################### +wait_for_db() { + if ! [[ -n $SKIP_DB_WAIT && $SKIP_DB_WAIT -gt 0 ]] + then + local host=${1:-'localhost'} + local port=${2:-'3306'} + local ip_address + ip_address=$(getent ahosts "$host" | grep STREAM | head -n 1 | cut -d ' ' -f 1) + + info "Connecting to $host server at $ip_address:$port.\n" + + max_count=${3:-10} + counter=0 + until nc -z "$host" "$port"; do + counter=$((counter+1)) + if [ $counter == $max_count ]; then + error "Couldn't connect to $host server.\n" + return 1; + fi + info "Trying to connect to $host server at $ip_address:$port. Attempt $counter.\n" + sleep 5 + done + info "Connected to $host server." + fi +} + +# migrate_db() { +# if ! [[ -n $SKIP_DB_MIGRATE && $SKIP_DB_MIGRATE -gt 0 ]] && [[ -f .sequelizerc ]]; then +# info "Applying database migrations (sequelize db:migrate)." +# sequelize db:migrate +# fi +# } + +# add_nodemon_support() { +# info "Adding nodemon npm module (dev)." +# npm install nodemon --save-dev +# sed -i 's;"start".*;"start": "node ./bin/www", "development": "nodemon ./bin/www";' package.json +# } + +# install_packages() { +# if ! dependencies_up_to_date; then +# if ! [[ -n $SKIP_NPM_INSTALL && $SKIP_NPM_INSTALL -gt 0 ]] && [[ -f package.json ]]; then +# info "Installing npm packages." +# npm install +# fi +# fi +# } + diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..5c0c4a2 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e +source /scripts/bash_functions.sh + +## Entry instructions + +info "Wait for $APP_DATABASE_HOST:3306" +wait_for_db "${APP_DATABASE_HOST}" "3306" 6 + +exec "$@" diff --git a/docker/init-from-DB_NAMES.sh b/docker/init-from-DB_NAMES.sh new file mode 100755 index 0000000..2121480 --- /dev/null +++ b/docker/init-from-DB_NAMES.sh @@ -0,0 +1,32 @@ +#!/bin/bash +source /scripts/bash_functions.sh + +# To execute on MariaDB + +# Debug commands +# service mysql status +# mysqladmin -u"$MYSQL_ROOT_USER" ping +# mysql -uroot -p$MYSQL_ROOT_PASSWORD -e '\q' +# mysql -uroot -p$MYSQL_ROOT_PASSWORD 'mysql' -e 'SHOW DATABASES' +# mysql -uroot -p$MYSQL_ROOT_PASSWORD 'mysql' -e 'select * from mysql.user' +# wait_for_db "127.0.0.1" "3306" 5 + +printf '========================================================================================\n' +info 'Initialize custom databases..' +info '[ you can define the desired databases as INIT_DB_NAMES="odiometro,amoreometro,.." ]' + +IFS=',' read -ra db_names <<< "${INIT_DB_NAMES}" # split database-names by comma +for db_name in "${db_names[@]}" +do + info "Create database $db_name\n" + # printf "mysql -uroot -p$MYSQL_ROOT_PASSWORD 'mysql' -e \"CREATE DATABASE $db_name\"\n" + mysql -uroot -p$MYSQL_ROOT_PASSWORD 'mysql' -e "CREATE DATABASE $db_name" + info "Apply schema to database \`$db_name\`\n" + mysql -uroot -p$MYSQL_ROOT_PASSWORD "$db_name" < /schemas/app-schema.sql + + mysql -uroot -p$MYSQL_ROOT_PASSWORD 'mysql' -e "SHOW TABLES FROM $db_name" +done + +mysql -uroot -p$MYSQL_ROOT_PASSWORD 'mysql' -e 'SHOW DATABASES' + +printf '========================================================================================\n' diff --git a/package.json b/package.json index a216068..2581ca3 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "description": "Measure hate tweets in real time on Twitter", "main": "app.js", "scripts": { - "test": "mocha" + "test": "mocha", + "start": "node app.js" }, "keywords": [ "node", @@ -28,4 +29,4 @@ "chai": "^4.2.0", "mocha": "^7.2.0" } -} \ No newline at end of file +}