Merge pull request #20 from jpbnetley/feature/config-file(#15)

Feature/config file(#15)

README.md

@@ -1,21 +1,30 @@
 # Mongo-csv-importer
 Imports csv items into mongo
 
-# Environment variables
+#Injecting environment variables
+
+Environment variables can be injected in 2 ways.
+1. A config file.
+2. The system environment variables.
+
+When using the config file, create a config file `application.conf` located under  
+`src/main/resources`
+
+## Required Environment variables
 ```
-mongo_address=;
-mongo_port=;
-mongo_db_name=;
+mongo_address=
+mongo_port=
+mongo_db_name=
 ```
 
 ## Optional auth environment variables
 ```
-mongo_auth_uname=;
-mongo_auth_pw=;
+mongo_auth_uname=
+mongo_auth_pw=
 ```
 
-## Usage
-When the user is prompted for their input to the path, the Raw path to the files should be  
+# Usage
+When the user gets prompted for their input to the path, the raw path to the files should be  
 entered.
 eg: `c:/users/username/desktop/csvFiles`  
-The file under this directory should be `.csv`
+The files under this directory should be `.csv`

build.sbt

@@ -3,11 +3,13 @@ name := "Mongo-csv-importer"
 version := "0.1"
 
 libraryDependencies ++= Seq(
-  "org.mongodb.scala"           %% "mongo-scala-driver" % "2.9.0",
-  "io.monix"                    %% "monix"              % "3.2.1",
-  "org.typelevel"               %% "cats-core"          % "2.1.1",
-  "ch.qos.logback"              % "logback-classic"     % "1.2.3",
-  "com.typesafe.scala-logging"  %% "scala-logging"      % "3.9.2"
+  "org.mongodb.scala"           %% "mongo-scala-driver"     % "2.9.0",
+  "io.monix"                    %% "monix"                  % "3.2.1",
+  "org.typelevel"               %% "cats-core"              % "2.1.1",
+  "ch.qos.logback"              % "logback-classic"         % "1.2.3",
+  "com.typesafe.scala-logging"  %% "scala-logging"          % "3.9.2",
+  "com.typesafe"                % "config"                  % "1.4.0",
+  "com.beachape"                %% "enumeratum"             % "1.6.0"
 )
 
 scalaVersion := "2.13.2"

src/main/resources/application.conf

@@ -0,0 +1,5 @@
+mongo-address=localhost
+mongo-port=27017
+mongo-auth-uname=john.doe
+mongo-auth-pw=alessonadaykeepsthedocktoraway
+mongo-db-name=northwind

src/main/resources/logback.xml

@@ -5,6 +5,7 @@
         </layout>
     </appender>
     <logger name="com.base22" level="TRACE"/>
+    <logger name="Mongo csv importer" level="TRACE"/>
     <root level="error">
         <appender-ref ref="STDOUT"/>
     </root>

src/main/scala/Main.scala

@@ -1,22 +1,26 @@
-import Util.DataBuilder.Processing
-import Util.Database
-import Util.File.FileHelper
-import Util.UserPrompt._
+import util.dataBuilder.Processing
+import util.databaseProvider
+import util.file.FileHelper
+import util.UserPrompt._
 import cats.data.EitherT
 import cats.effect.ExitCode
 import monix.eval.{Task, TaskApp}
-import Util.ImplicitConversions._
+import org.mongodb.scala.MongoDatabase
+import util.ImplicitConversions._
+import util.config.ConfigHandler
+import util.databaseProvider.MongoDB
 
 object Main extends TaskApp {
    override def run(args: List[String]): Task[ExitCode] = {
-     val database: Database.MongoDB.type = Database.MongoDB
+     val database: MongoDB.type = databaseProvider.MongoDB
 
      (for {
+      config        <- EitherT.fromEither[Task](ConfigHandler.init)
+      mongoClient   =  database.getMongoClient(config)
+      db            <- EitherT.liftF[Task, Exception, MongoDatabase](database.getDatabase(config, mongoClient))
       userInput     <- EitherT(promptUser())
       csvFiles      <- EitherT(FileHelper.getCsvFiles(userInput.folderPath, userInput.skipFiles))
       orderedFiles  =  csvFiles.zipWithIndex.toOrderedFile()
-      mongoClient   <- EitherT.fromEither[Task](database.getMongoClient)
-      db            <- EitherT(database.getDatabase(mongoClient))
       _             <- EitherT(Processing.csvFiles(orderedFiles)(db))
     } yield {
        database.close(mongoClient)

src/main/scala/Util/ErrorHandler.scala → src/main/scala/util/ErrorHandler.scala

@@ -1,6 +1,6 @@
-package Util
+package util
 
-import Util.Logging.log
+import util.Logging.log
 import monix.eval.Task
 
 case object ErrorHandler {
@@ -24,6 +24,11 @@ case object ErrorHandler {
     Left(new Exception(message, e))
   }
 
+  def errorL[A](exception: Exception): Left[Exception, A] = {
+    log.error(exception.getMessage)
+    Left(exception)
+  }
+
   def error(message: String): Exception = {
     log.error(message)
     new Exception(message)

src/main/scala/util/ImplicitConversions.scala

@@ -0,0 +1,25 @@
+package util
+
+import com.typesafe.config.Config
+import util.models.OrderedFile
+
+import scala.reflect.io.File
+
+object ImplicitConversions {
+  implicit class listFiles(files : List[(File, Int)]) {
+    def toOrderedFile: List[OrderedFile] = {
+      files.map { case (file, index) =>
+        OrderedFile(index, file)
+      }
+    }
+  }
+
+  implicit class RichConfig(val underlying: Config) extends AnyVal {
+    def getOptionalString(path: String): Option[String] = if (underlying.hasPath(path)) {
+      Some(underlying.getString(path))
+    } else {
+      None
+    }
+  }
+}
+

src/main/scala/Util/Logging.scala → src/main/scala/util/Logging.scala

@@ -1,4 +1,4 @@
-package Util
+package util
 
 import com.typesafe.scalalogging.Logger
 import org.slf4j.LoggerFactory

src/main/scala/Util/UserPrompt.scala → src/main/scala/util/UserPrompt.scala

@@ -1,12 +1,12 @@
-package Util
+package util
 
-import Util.File.FileHelper
-import Util.Models.UserInput
+import util.file.FileHelper
+import util.models.UserInput
 import cats.data.EitherT
 import cats.implicits._
 import monix.eval.Task
-import Util.Logging.log
-import Util.ErrorHandler._
+import util.Logging.log
+import util.ErrorHandler._
 
 object UserPrompt {
 

src/main/scala/util/config/ConfigFile.scala

@@ -0,0 +1,44 @@
+package util.config
+
+import util.databaseProvider.SystemConfigProperties
+import util.models.SystemConfigPropertiesResponse
+import com.typesafe.config.ConfigFactory
+import util.ErrorHandler._
+import cats.implicits._
+import util.Logging.log
+import util.ImplicitConversions.RichConfig
+import util.models.enums.MongoDbEnvironmentNames
+
+case object ConfigFile extends SystemConfigProperties {
+  /** validate of the configeration method is present
+    *
+    * @return Boolean, true if it exists
+    */
+  override def exists: Boolean = {
+    val exist = !ConfigFactory.load().isEmpty
+    log.info(s"Check if config file: application.conf exists: $exist")
+    exist
+  }
+
+  /** extracts the config variables
+    *
+    * @return SystemConfigPropertiesResponse
+    */
+  override def extractConfig: Either[Exception, SystemConfigPropertiesResponse] = {
+    try {
+      SystemConfigPropertiesResponse(
+        ConfigFactory.load().getString(MongoDbEnvironmentNames.address.value),
+        ConfigFactory.load().getInt(MongoDbEnvironmentNames.port.value),
+        ConfigFactory.load().getString(MongoDbEnvironmentNames.name.value),
+        ConfigFactory.load().getOptionalString(MongoDbEnvironmentNames.username.value),
+        ConfigFactory.load().getOptionalString(MongoDbEnvironmentNames.password.value).map(_.toCharArray)
+      ).asRight[Exception]
+
+    } catch {
+      case e: Exception => errorL(e)
+    }
+
+  }
+
+}
+

src/main/scala/util/config/ConfigHandler.scala

@@ -0,0 +1,18 @@
+package util.config
+
+import util.models.SystemConfigPropertiesResponse
+import util.ErrorHandler._
+import util.databaseProvider.SystemConfigProperties
+
+case object ConfigHandler {
+  def init: Either[Exception, SystemConfigPropertiesResponse] = {
+    val environments: List[SystemConfigProperties] = List(ConfigFile, EnvironmentVariables)
+
+    getEnvironment(environments)
+  }
+
+  private def getEnvironment(environments: List[SystemConfigProperties]): Either[Exception, SystemConfigPropertiesResponse]  = {
+    environments.find(_.exists).map(_.extractConfig)
+      .getOrElse(errorL("No environments are defined"))
+  }
+}

src/main/scala/util/config/EnvironmentVariables.scala

@@ -0,0 +1,41 @@
+package util.config
+
+import util.ErrorHandler.error
+import util.databaseProvider.SystemConfigProperties
+import util.models.SystemConfigPropertiesResponse
+import cats.implicits._
+import util.Logging.log
+import util.models.enums.MongoDbEnvironmentNames
+
+case object EnvironmentVariables extends SystemConfigProperties {
+  /** validate of the configeration method is present
+    *
+    * @return Boolean, true if it exists
+    */
+  override def exists: Boolean = {
+    val exist = extractConfig.isRight
+    log.info(s"Check if env vars exists: $exist")
+    exist
+  }
+
+  /** extracts the config variables
+    *
+    * @return SystemConfigPropertiesResponse
+    */
+  override def extractConfig: Either[Exception, SystemConfigPropertiesResponse] = {
+    for {
+      address       <- Either.fromOption(sys.env.get(MongoDbEnvironmentNames.address.value), error("mongo address not found"))
+      port          <- Either.fromOption(sys.env.get(MongoDbEnvironmentNames.port.value).map(_.toInt), error("mongo port not found"))
+      databaseName  <- Either.fromOption(sys.env.get(MongoDbEnvironmentNames.name.value), error("environment variables not set for db name"))
+    } yield {
+      val user          = sys.env.get(MongoDbEnvironmentNames.username.value)
+      val password      = sys.env.get(MongoDbEnvironmentNames.password.value).map(_.toCharArray)
+
+      SystemConfigPropertiesResponse(address, port, databaseName, user, password)
+    }
+  }
+}
+
+
+
+

src/main/scala/Util/DataBuilder/DataBuilder.scala → src/main/scala/util/dataBuilder/DataBuilder.scala

@@ -1,9 +1,9 @@
-package Util.DataBuilder
+package util.dataBuilder
 import org.bson.Document
 import cats.implicits._
 import monix.eval.Task
-import Util.Logging.log
-import Util.ErrorHandler._
+import util.Logging.log
+import util.ErrorHandler._
 
 object DataBuilder {
 
@@ -119,6 +119,8 @@ object DataBuilder {
       }.mkString.trim
     }
 
-    jsonObjects.map{row => println{row.mkString.trim};Document.parse(row)}
+    jsonObjects.map { row =>
+      println(row.mkString.trim)
+      Document.parse(row)}
   }
 }

src/main/scala/Util/DataBuilder/Processing.scala → src/main/scala/util/dataBuilder/Processing.scala

@@ -1,16 +1,16 @@
-package Util.DataBuilder
+package util.dataBuilder
 
-import Util.DataBuilder.DataBuilder.buildMongoDocuments
-import Util.File.FileHelper
+import util.dataBuilder.DataBuilder.buildMongoDocuments
+import util.file.FileHelper
 import cats.data.EitherT
 import cats.implicits._
 import monix.eval.Task
 import org.bson.Document
 
 import scala.concurrent.Await
 import scala.concurrent.duration.Duration
-import Util.ErrorHandler._
-import Util.Models.OrderedFile
+import util.ErrorHandler._
+import util.models.OrderedFile
 import org.mongodb.scala.MongoDatabase
 
 object Processing {