pipeline

#!/bin/bash
# This script runs a specified pipeline from the config. That pipeline then executes the builders in the order as specified in the config.
# It determines how to execute the builders by looking in their README.md file and grapping the commands from a defined pattern.

# -------------------------------------------------------------

# Usage
# pipeline <configFile> <name of pipeline configured in the config>

# Example usage
# ./pipeline Germany-Hamburg pipeline_pbfgraphmbtiles

# Example usages with options

# Example usage --dry-run (does not execute anything, just simulates)
# ./pipeline Germany-Hamburg pipeline_pbfgraphmbtiles --dry-run

# Example usage --stderr (useful for logging to a file or abother logging mechanism)
# ./pipeline Germany-Hamburg pipeline_pbfgraphmbtiles --stderr 2>log.txt

# Example usage --stderr with --markdown (useful for logging to a markdown file)
# ./pipeline Germany-Hamburg pipeline_pbfgraphmbtiles  --stderr --dry-run --markdown 2>log.md

# Of course options can be combined like
# ./pipeline Germany-Hamburg pipeline_pbfgraphmbtiles --dry-run --stderr --markdown
# to make a simulation and to save a markdown formatted file containing a markdown formatted log

# -------------------------------------------------------------

# That defined pattern is as follows:
# 1. The script looks for a line containing the following content: ## How to use
# 2. The script extracts all content as long as it does not find another line containing ## or as long it does not reach the end of all lines
# 3. In the extract it looks for a line containing the following content: ```bash
# 4. The script extracts all content from the extract as long as it does not find a line containing ``` or as long it does not reach the end of all lines
# 5. It has now the commands. Each line correspondends to an element in the array (array delimited by new line)

# Pattern matching based on an example:
# -------------------------------------------------------------

# We have the following content from a README file we've read:

## MBTiles Builder
# 
# | &nbsp;                                 | &nbsp;                                                       |
# | -------------------------------------- | ------------------------------------------------------------ |
# | The following extensions depend on it  | [tileserver](https://github.com/trufi-association/trufi-server/tree/main/extensions/tileserver) |
# | This depends on the following builders | [map-pbf-builder](../map-pbf-builder)                        |
# 
### Description
# 
# Generates the background map tiles in the `*.mbtile` format used by many tile serving services like [OpenMapTiles](https://github.com/openmaptiles/openmaptiles) what we use for our extension [tileserver](https://github.com/trufi-association/trufi-server/tree/main/extensions/tileserver). Maps generated using this builder need to credit `© OpenMapTiles © OpenStreetMap contributors`. Attribution won't be applied automatically. This builder uses [OpenMapTiles](https://github.com/openmaptiles/openmaptiles) underneath.
# 
### How to use
# 
# This tool will generate a `.mbtile` file which contains the necessary data for generating map background tiles either on the fly (upon request by the client) or static using the builder [*Static Map Tiles Builder*](./static-map-tiles-builder). Run the following commands:
# 
# ```bash
# export $(cat ../$envfile | xargs)
# bash generate_mbtiles.sh
# ```
# 
# - The `.mbtile` file out is located at `../data/<Country-City>/tileserver/region/myregion.osm.pbf`
# 

# The first extract will be:

### How to use
# 
# This tool will generate a `.mbtile` file which contains the necessary data for generating map background tiles either on the fly (upon request by the client) or static using the builder [*Static Map Tiles Builder*](./static-map-tiles-builder). Run the following commands:
# 
# ```bash
# export $(cat ../$envfile | xargs)
# bash generate_mbtiles.sh
# ```
# 
# - The `.mbtile` file out is located at `../data/<Country-City>/tileserver/region/myregion.osm.pbf`
# 

# The second extract will be:

# export $(cat ../$envfile | xargs)
# bash generate_mbtiles.sh

# -------------------------------------------------------------

configFile="$1"
configFile=${configFile//".env"/};
noColor=$(echo "$@" | grep "\-\-no-color")
dryRun=$(echo "$@" | grep "\-\-dry-run")
toStderr=$(echo "$@" | grep "\-\-stderr")
markdown=$(echo "$@" | grep "\-\-markdown")

export $(cat "./config/$configFile.env" | xargs)
export configFile=$configFile
export pipelineName=$2

pipeline="${!pipelineName}"
pipeline=${pipeline//;/ };
pipeline=( $pipeline )

extract=()
input=()


if [ -z "$noColor" ]; then # coloring is allowed
	titleColor="\033[0;33m"
 	primaryColor="\033[0;34m"
 	secondaryColor="\033[0;35m"
 	textColor=""
 	colorEnd="\033[0;m"
fi

if [ -n "$markdown" ]; then # markdown mode
	italicText="<ul>"
	italicTextEnd="</ul>"
	markdownHeading="#"
fi

echoError() {
	if [ -n "$toStderr" ]; then
		echo -e "$1" 1>&2
	fi
}

echoTitle() {
	echo -e "${titleColor}$1${colorEnd}"
	echoError "$1"
}

echoPrimary() {
	echo -e "${primaryColor}$1${colorEnd}"
	echoError "$1"
}

echoSecondary() {
	echo -e "${secondaryColor}$1${colorEnd}"
	echoError "$1"
}

echoText() {
	echo -e "${textColor}$1${colorEnd}"
	echoError "$1"
}

extractFromArray() { # extraction algorithm
	start_find="$1"
	end_find="$2"
	extract=()
	addmode=""
	# taken from https://linuxize.com/post/how-to-read-a-file-line-by-line-in-bash/ and modified
	for line in "${input[@]}"; do
		start=`echo $line | grep -i "$start_find"`
		end=`echo $line | grep -i "$end_find"`
		if ! [ -z "$addmode" ]; then
			if ! [ -z "$end" ]; then # if the line contains $end then
				addmode="" # unset add marker
				continue
			fi
			line=${line//" "/"#"}
			extract+=( "$line" )
		else
			# if add marker hasn't been set
			if ! [ -z "$start" ]; then # beginning of the entry we want to extract found
				addmode="true" # set add marker
				#echo "  start"
				continue
			fi
		fi
	done
}

startDateTime=$(date +"%x %X %Z")
echoTitle "${markdownHeading} Run report: pipeline '$pipelineName' from $startDateTime"
echoTitle ""

echoText "Starting pipeline \`$pipelineName\` for environment \`$configFile\` ..."
echoText "Date & Time started: $startDateTime"
if ! [ -z "$dryRun" ]; then # simulation
	echoText ""
	echoText "This is a dry-run. In that mode no commands are executed actually!"
	echoText ""
fi

if [ -n "$markdown" ]; then # markdown mode
	echoError ""
	echoError "- [Executing builders](#executing-builders)"
	for builder in "${pipeline[@]}"; do
		echoError "  - [$builder](#$builder)"
	done
	echoError "- [Clean up](#clean-up)"
	echoError ""
fi

getCommandsToExecute() {
	extract=()
	input=()
	
	local readmeHyperlink="'./$builder/README.md'"
	if [ -n "$markdown" ]; then
		readmeHyperlink="[README](./$builder/README.md)"
	fi
	echoText "${italicText}Reading run configuration from \`$builder\`'s $readmeHyperlink ...${italicTextEnd}"
	# read file and convert its content in an array delimited by new line
	readarray input < "./README.md"
	
	extractFromArray "## How to use" "##"
	input=(${extract[@]})
	extract=()
	
	extractFromArray "\`\`\`bash" "\`\`\`"
	
}

echoPrimary "${markdownHeading}${markdownHeading} Executing builders"
echoPrimary ""

for builder in "${pipeline[@]}"; do
	echoSecondary "${markdownHeading}${markdownHeading}${markdownHeading} $builder"
	echoText ""
	cd $builder
	getCommandsToExecute $builder
	for cmd in "${extract[@]}"; do
		cmd=${cmd//"#"/" "}
		echoText "${italicText}Executing command${italicTextEnd} \`$cmd\` ..."
		if [ -z "$dryRun" ]; then # no simulation
			eval $cmd
		else # simulation
			echoText "> The output of command execution does not exist in the simulation mode"
			echoText ""
		fi
	done
	cd ../
done

echoPrimary "${markdownHeading}${markdownHeading} Clean up"
echoPrimary ""
echoText "${italicText}Executing command${italicTextEnd} \`sudo chown -R $USER:$USER ./data --verbose\`"
if [ -z "$dryRun" ]; then # no simulation
	sudo chown -R $USER:$USER ./data --verbose
else
	echoText "> The output of command execution does not exist in the simulation mode"
	echoText ""
fi