Alpha - entropy checking (#5)

Adds entropy checking feature

Author: necrophonic

.gitignore

@@ -25,4 +25,5 @@ _testmain.go
 0
 
 pre-commit
-main
+main
+build

CHANGELOG.md

@@ -1,5 +1,14 @@
-### Unreleased
-  - Add regex line test for `----BEGIN CERTIFICATE----`
+# Changelog
 
-### 0.1.0 2018-02-07
-  - Initial release
+## Unreleased
+
+## 0.2.0 2018-03-28
+
+- add regex line test for `----BEGIN CERTIFICATE----`
+- add entropy check into core (optional)
+- add `DC_ENTROPY_EXPERIMENT` environment option to activate entropy checking
+- add support for multi-os cross compilation in Makefile
+
+## 0.1.0 2018-02-07
+
+- initial release

Makefile

@@ -0,0 +1,20 @@
+platforms = windows/amd64 darwin/amd64
+package = cmd/pre-commit/main.go
+binary = build/pre-commit
+
+help:
+	@ echo "See Makefile for options"
+
+compile:
+	@ for platform in $(platforms); do 										  \
+		platform_split=($${platform//\// }); 								  \
+		GOOS=$${platform_split[0]}; 										  \
+		GOARCH=$${platform_split[1]}; 										  \
+		output_name=$(binary)'_'$$GOOS'-'$$GOARCH;							  \
+		if [ $$GOOS = "windows" ]; then 									  \
+			output_name+='.exe'; 											  \
+		fi;	 																  \
+		echo "Build for $$platform -> $$output_name";						  \
+		env GOOS=$$GOOS GOARCH=$$GOARCH go build -o $$output_name $(package); \
+	done
+	

README.md

@@ -89,6 +89,18 @@ If you're VERY SURE these files are ok, rerun commit with --no-verify
 **NB** Currently if you update the pre-commit script in your templates, you will
 need to manually re-copy it into each repo that uses it.
 
+
+## Experimental Entropy Checking
+
+By default, the `pre-commit` tool won't use entropy checking on patch strings. If you
+wish to enable this functionality, please set the `DC_ENTROPY_EXPERIMENT` environment
+variable.
+
+```shell
+$ export DC_ENTROPY_EXPERIMENT=1
+```
+
+
 License
 =======
 

cmd/pre-commit/main.go

@@ -22,13 +22,20 @@ var (
 )
 
 func main() {
+
 	flag.Parse()
 
 	if *target == "" {
 		*target = "."
 	}
 	fmt.Printf("Running precommit diff check on '%s'\n", *target)
 
+	// Import environmental feature flags
+	if useEntropyFeature := os.Getenv("DC_ENTROPY_EXPERIMENT"); useEntropyFeature == "1" {
+		fmt.Println("i) Experimental entropy checking enabled")
+		diffcheck.UseEntropy = true
+	}
+
 	// Get where we are so we can get back
 	ex, err := os.Executable()
 	if err != nil {

diffcheck/diffcheck.go

@@ -11,6 +11,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/ONSdigital/git-diff-check/entropy"
 	"github.com/ONSdigital/git-diff-check/rule"
 )
 
@@ -47,6 +48,10 @@ type (
 var (
 	// Matches the first offset in the old and new diff
 	reOffset = regexp.MustCompile("^@@ -(\\d+).* \\+(\\d+).* @@")
+
+	// UseEntropy is a feature flag that, if set true, enables experimental
+	// string entropy testing
+	UseEntropy = false
 )
 
 const (
@@ -162,12 +167,20 @@ func checkLineBytes(line []byte, position int) (bool, []Warning) {
 
 	warnings := []Warning{}
 
+	// Normal line rulesets
 	for _, rule := range rule.Sets["line"] {
 		if rule.Regex.Match(line) {
 			warnings = append(warnings, Warning{Type: "line", Description: rule.Caption, Line: position})
 		}
 	}
 
+	// Entropy check
+	if UseEntropy {
+		if ok, _ := entropy.Check(line); !ok {
+			warnings = append(warnings, Warning{Type: "line", Description: "Possible key in high entropy string", Line: position})
+		}
+	}
+
 	if len(warnings) > 0 {
 		return false, warnings
 	}

diffcheck/diffcheck_test.go

@@ -17,6 +17,9 @@ type testCase struct {
 
 func TestSnoopPatch(t *testing.T) {
 
+	// Enable the feature flag to assume entropy usage in these tests
+	diffcheck.UseEntropy = true
+
 	for _, tc := range testCases {
 
 		t.Logf("Given a patch containing %s", tc.Name)
@@ -36,16 +39,22 @@ func TestSnoopPatch(t *testing.T) {
 		}
 
 		for i, expected := range tc.ExpectedReports {
+			if i >= len(reports) {
+				break
+			}
 			gotReport := reports[i]
 
 			shouldEqual("path", gotReport.Path, expected.Path, t)
 			shouldEqual("old path", gotReport.OldPath, expected.OldPath, t)
 
 			if len(expected.Warnings) != len(gotReport.Warnings) {
-				t.Errorf("Incorrect number of warnings in report, got %d, expected %d", len(expected.Warnings), len(gotReport.Warnings))
+				t.Errorf("Incorrect number of warnings in report, got %d, expected %d", len(gotReport.Warnings), len(expected.Warnings))
 			}
 
 			for j, expWarning := range expected.Warnings {
+				if j >= len(gotReport.Warnings) {
+					break
+				}
 				gotWarning := gotReport.Warnings[j]
 
 				shouldEqual("type", gotWarning.Type, expWarning.Type, t)
@@ -126,6 +135,11 @@ index 0000000..e69de29
 						Line:        6,
 						Description: "Possible AWS Access Key",
 					},
+					{
+						Type:        "line",
+						Line:        7,
+						Description: "Possible key in high entropy string",
+					},
 				},
 			},
 		},
@@ -138,6 +152,7 @@ index e69de29..92251f8 100644
 
 # Shhh
 aws=AKIA7362373827372737
+secret=ZWVTjPQSdhwRgl204Hc51YCsritMIzn8B=/p9UyeX7xu6KkAGqfm3FJ+oObLDNEva
 		`),
 	},
 }

entropy/entropy.go

@@ -0,0 +1,86 @@
+// Package entropy contains functions for checking the entropy of given data
+package entropy
+
+import (
+	"bytes"
+	"math"
+	"strings"
+)
+
+// Define entropy thresholds over which a string is considered complex enough
+// to be a potential key
+const (
+	Base64Threshold = 4.5
+	HexThreshold    = 3.0
+)
+
+const (
+	consider = 20 // When scanning for strings, only consider >= this value length
+)
+
+// CalculateShannon calculates the shannon entropy for a block of data
+// - http://blog.dkbza.org/2007/05/scanning-data-for-entropy-anomalies.html
+func CalculateShannon(data []byte) float64 {
+	if len(data) == 0 {
+		return 0.0
+	}
+	entropy := 0.0
+	pX := 0.0
+	for x := 0; x < 256; x++ {
+		pX = float64(bytes.Count(data, []byte(string(x)))) / float64(len(data))
+		if pX > 0 {
+			entropy += -pX * math.Log2(pX)
+		}
+	}
+	return entropy
+}
+
+// Check searches through a given block of data to attempt to identify high
+// entropy blocks. Returns true and number of matching strings if found
+func Check(b []byte) (bool, int) {
+	found := [][]byte{}
+
+	// Offset where we started reading the data - indexes from
+	// 1 instead of zero otherwise we'll capture a spurious leading
+	// byte into the slice
+	// start := 1
+	start := -1
+
+	// Base64 strings
+	for i, tok := range b {
+		if !isBase64Byte(tok) || i+1 == len(b) {
+			if i-start >= consider {
+				s := b[start+1 : i]
+				if e := CalculateShannon(s); e > Base64Threshold {
+					found = append(found, s)
+				}
+			}
+			start = i
+		}
+	}
+
+	start = -1
+
+	// Hex strings
+	for i, tok := range b {
+		if !isHexByte(tok) || i+1 == len(b) {
+			if i-start >= consider {
+				s := b[start+1 : i]
+				if e := CalculateShannon(s); e > HexThreshold {
+					found = append(found, s)
+				}
+			}
+			start = i
+		}
+	}
+
+	return len(found) == 0, len(found)
+}
+
+func isBase64Byte(b byte) bool {
+	return strings.Contains("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=", string(b))
+}
+
+func isHexByte(b byte) bool {
+	return strings.Contains("ABCDEFabcdef0123456789", string(b))
+}

entropy/entropy_test.go

@@ -0,0 +1,90 @@
+// Package entropy_test is for testing the entropy package
+//
+// !IMPORTANT! - none of the keys or strings listed in this file are real keys. They
+// 				 are generated purely to test this package and MUST NOT be used
+//				 anywhere else as actual credentials
+package entropy_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/ONSdigital/git-diff-check/entropy"
+)
+
+var (
+	highBase64 = [][]byte{
+		[]byte(`ZWVTjPQSdhwRgl204Hc51YCsritMIzn8B=/p9UyeX7xu6KkAGqfm3FJ+oObLDNEva`),
+		[]byte(`hSXAQy9D1J0hkCQy0tKBCxnpcOQCPeM54RFXZLJE`),
+		[]byte(`secret=ZWVTjPQSdhwRgl204Hc51YCsritMIzn8B=/p9UyeX7xu6KkAGqfm3FJ+oObLDNEva`),
+		[]byte(`aws:ZWVTjPQSdhwRgl204Hc51YCsritMIzn8B=/p9UyeX7xu6KkAGqfm3FJ+oObLDNEva`),
+	}
+	highHex = [][]byte{
+		[]byte(`b3A0a1FDfe86dcCE945B72`),
+	}
+)
+
+func ExampleCalculateShannon() {
+	password := []byte("verysecret")
+	if entropy.CalculateShannon(password) < entropy.Base64Threshold {
+		fmt.Println("Password not complex enough!")
+	}
+}
+
+func TestCalculateEntropy(t *testing.T) {
+
+	t.Log("Check base64 data")
+	for _, b := range highBase64 {
+		if e := entropy.CalculateShannon(b); e < entropy.Base64Threshold {
+			t.Errorf("Got entropy %f, expected > %f", e, entropy.Base64Threshold)
+		}
+	}
+
+	t.Log("Check hex data")
+	for _, b := range highHex {
+		if e := entropy.CalculateShannon(b); e < entropy.HexThreshold {
+			t.Errorf("Got entropy %f, expected > %f", e, entropy.HexThreshold)
+		}
+	}
+}
+
+func TestCheck(t *testing.T) {
+
+	exampleBlock := []byte(`+// CheckPatchLine takes a line from a patch hunk and tests it for naughty patterns
++func CheckPatchLine(line []byte) (bool, []Warning) {
++       warnings := []Warning{}
++		aws := []byte("hSXAQy9D1J0hkCQy0tKBCxnpcOQCPeM54RFXZLJE")
++
++ 		// Log in with secret: ZWVTjPQSdhwRgl204Hc51YCsritMIzn8B=/p9UyeX7xu6KkAGqfm3FJ+oObLDNEva
++
++       for _, rule := range linePatterns {
++               if found := rule.Regex.FindAll(line, -1); len(found) > 0 {
++                       for _, f := range found {
++                               // TODO Ignore exclusions
++                               if string(f) != "b3A0a1FDfe86dcCE945B72" {
++                                       warnings = append(warnings, Warning{Type: "line", Line: -1})
++                               }`)
+
+	ok, n := entropy.Check(exampleBlock)
+	if ok {
+		t.Error("Expected 'not ok'")
+	}
+	if n != 3 {
+		t.Errorf("Expected warnings, got %d, expected 3", n)
+	}
+
+	for _, b := range highBase64 {
+		ok, _ := entropy.Check(b)
+		if ok {
+			t.Error("Expected failed base64 entropy check")
+		}
+	}
+
+	for _, b := range highHex {
+		ok, _ := entropy.Check(b)
+		if ok {
+			t.Error("Expected failed hex entropy check")
+		}
+	}
+
+}

rule/rule_test.go

@@ -0,0 +1,16 @@
+package rule_test
+
+import (
+	"testing"
+
+	"github.com/ONSdigital/git-diff-check/rule"
+)
+
+func TestInitRules(t *testing.T) {
+	// The rulesets should be populated into rule.Sets as part of the package
+	// init() method. We won't test for specific rules but the amount loaded
+	// should be non-zero
+	if len(rule.Sets) == 0 {
+		t.Error("Failed to initialise rulesets")
+	}
+}