Fix issue #19: Handle similarly named column headers in CSV merger

internal/merger.go

@@ -33,29 +33,35 @@ func (m *Merger) CombineCSVFiles(filenames []string, cols []string, outputFilena
 
 func (m *Merger) combine(w *csv.Writer, files []string, columns []string) {
 	//first try at this will be a naive impl:
-	// 1. read in records of each input file, write columns with matching headers; load everything into memory
-	log.Debug("columns to keep", "columns", columns)
-	for _, f := range files {
-		reader := csv.NewReader(openFile(f))
-		records, _ := reader.ReadAll()
-		rows := make([][]string, len(records))
-		indexes := ColumnIndexes(records[0], columns)
-		for i := 0; i < len(records); i++ {
-			var cols []string
-			for _, col := range indexes {
-				cols = append(cols, records[i][col]) //the columns we are using in the output file
-			}
-			rows[i] = append(rows[i], cols...) //the rows for this file
-		}
-
-		err := w.WriteAll(rows)
-		if err != nil {
-			LogPanic("", err)
-		}
-		w.Flush()
-		fmt.Printf("%v <- %s\n", m.OutputFileName, f)
-	}
-}
+ func (m *Merger) combine(w *csv.Writer, files []string, columns []string) {
+ 	log.Debug("columns to keep", "columns", columns)
+ 	headerMap := make(map[string]int)
+ 	for _, f := range files {
+ 		reader := csv.NewReader(openFile(f))
+ 		records, _ := reader.ReadAll()
+ 		rows := make([][]string, len(records))
+ 		indexes := ColumnIndexes(records[0], columns)
+ 		for i := 0; i < len(records); i++ {
+ 			var cols []string
+ 			for _, col := range indexes {
+ 				if _, exists := headerMap[records[0][col]]; exists {
+ 					cols[headerMap[records[0][col]]] = records[i][col]
+ 				} else {
+ 					headerMap[records[0][col]] = len(cols)
+ 					cols = append(cols, records[i][col]) //the columns we are using in the output file
+ 				}
+ 			}
+ 			rows[i] = append(rows[i], cols...) //the rows for this file
+ 		}
+
+ 		err := w.WriteAll(rows)
+ 		if err != nil {
+ 			LogPanic("", err)
+ 		}
+ 		w.Flush()
+ 		fmt.Printf("%v <- %s\n", m.OutputFileName, f)
+ 	}
+ }
 
 // AppendCSVFiles appends the files in the array to the output file (writer)
 func (m *Merger) AppendCSVFiles(w *csv.Writer, files []string) {
@@ -138,4 +144,4 @@ func copyTo(r *csv.Reader, w *csv.Writer) {
 	for line, b := readline(r); b; line, b = readline(r) {
 		writeLine(w, line)
 	}
-}
+}

internal/merger_test.go

@@ -23,4 +23,4 @@ func TestCombine(t *testing.T) {
 	//fmt.Print(w)
 
 	approvals.VerifyString(t, w.String())
-}
+}