cmd fuzz implementation (#34)

Author: peakle

Makefile

@@ -1,5 +1,5 @@
 NOW=`date '+%Y.%m.%d %H:%M:%S'`
-OS=`uname -i -v`
+OS=`uname`
 AFTER_COMMIT=`git rev-parse HEAD`
 VERSION=0.3.0
 

cmd/phpsmith/cmd_fuzz.go

@@ -1,12 +1,164 @@
 package main
 
-import "flag"
+import (
+	"bytes"
+	"context"
+	"flag"
+	"log"
+	"math/rand"
+	"os"
+	"os/signal"
+	"runtime"
+	"strconv"
+	"syscall"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"golang.org/x/sync/errgroup"
+
+	"github.com/quasilyte/phpsmith/cmd/phpsmith/interpretator"
+)
+
+type executor func(ctx context.Context, filename string, seed int64) ([]byte, error)
+
+var executors = []executor{
+	interpretator.RunPHP,
+	interpretator.RunKPHP,
+}
 
 func cmdFuzz(args []string) error {
 	fs := flag.NewFlagSet("phpsmith fuzz", flag.ExitOnError)
+	flagConcurrency := fs.Int("flagConcurrency", 1,
+		"Number of concurrent runners. Defaults to the half number of available CPU cores.")
+	flagSeed := fs.Int64("seed", 0,
+		`a seed to be used during the code generation, 0 means "randomized seed"`)
+	flagOutputDir := fs.String("o", "phpsmith_out",
+		`output dir`)
+
 	_ = fs.Parse(args)
 
-	// TODO
+	concurrency := *flagConcurrency
+	dir := *flagOutputDir
+	seed := *flagSeed
+
+	if concurrency == 1 && runtime.NumCPU()/2 > 1 {
+		concurrency = runtime.NumCPU() / 2
+	}
+
+	if seed == 0 {
+		seed = time.Now().Unix()
+	}
+
+	interrupt := make(chan os.Signal, 1)
+	signalNotify(interrupt)
+
+	eg, ctx := errgroup.WithContext(context.Background())
+	ctx, cancel := context.WithCancel(ctx)
+
+	go func() {
+		<-interrupt
+		cancel()
+	}()
+
+	dirCh := make(chan string, concurrency)
+	for i := 0; i < concurrency; i++ {
+		eg.Go(func() error {
+			return runner(ctx, dirCh, seed)
+		})
+	}
+
+	randomizer := rand.New(rand.NewSource(time.Now().Unix()))
+out:
+	for {
+		newDir := dir + strconv.Itoa(randomizer.Int())
+		if err := generate(newDir, seed); err != nil {
+			log.Println("on generate: ", err)
+			continue
+		}
+
+		select {
+		case dirCh <- newDir:
+		case <-ctx.Done():
+			break out
+		}
+	}
+
+	if err := eg.Wait(); err != nil {
+		log.Println("on errorGroup Execution: ", err)
+	}
 
 	return nil
 }
+
+func runner(ctx context.Context, dirs <-chan string, seed int64) error {
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case dir := <-dirs:
+			if err := fuzzingProcess(ctx, dir, seed); err != nil {
+				log.Println("on fuzzingProcess:", err)
+			}
+			log.Println("dir processed:", dir)
+		}
+	}
+}
+
+type ExecutorOutput struct {
+	Output string
+	Error  string
+}
+
+func fuzzingProcess(ctx context.Context, dir string, seed int64) error {
+	var results = make(map[int]ExecutorOutput, len(executors))
+	for i, ex := range executors {
+		var errMsg string
+		r, err := ex(ctx, dir, seed)
+
+		if err != nil {
+			errMsg = err.Error()
+		}
+
+		grepExceptions(r, seed)
+		results[i] = ExecutorOutput{
+			Output: string(r),
+			Error:  errMsg,
+		}
+	}
+
+	if diff := cmp.Diff(results[0].Output, results[1].Output); diff != "" {
+		l, err := os.OpenFile("./"+dir+"/log", os.O_RDWR|os.O_CREATE, 0700)
+		if err != nil {
+			log.Println("-----------------------------")
+			log.Printf("diff: %s\t, seed: %d\t\n", diff, seed)
+			log.Printf("out: %s\terr: %s\t\n", results[0].Output, results[0].Error)
+			log.Printf("out: %s\terr: %s\t\n", results[1].Output, results[1].Error)
+		} else {
+			defer l.Close()
+
+			logger := log.New(l, "", 0)
+			logger.Printf("diff: %s\t, seed: %d\t\n", diff, seed)
+			logger.Printf("out: %s\terr: %s\t\n", results[0].Output, results[0].Error)
+			logger.Printf("out: %s\terr: %s\t\n", results[1].Output, results[1].Error)
+		}
+	}
+
+	return nil
+}
+
+func signalNotify(interrupt chan<- os.Signal) {
+	signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
+}
+
+var exceptionPatterns = [][]byte{
+	[]byte("uncaught exception"),
+	[]byte("fatal error"),
+}
+
+func grepExceptions(s []byte, seed int64) {
+	for _, pattern := range exceptionPatterns {
+		if bytes.Contains(bytes.ToLower(s), pattern) {
+			log.Println("found exception pattern: on seed:", seed)
+		}
+	}
+}

cmd/phpsmith/cmd_generate.go

@@ -15,19 +15,24 @@ import (
 
 func cmdGenerate(args []string) error {
 	fs := flag.NewFlagSet("phpsmith generate", flag.ExitOnError)
-	flagSeed := fs.Uint64("seed", 0,
+	flagSeed := fs.Int64("seed", 0,
 		`a seed to be used during the code generation, 0 means "randomized seed"`)
 	flagOutputDir := fs.String("o", "phpsmith_out",
 		`output dir`)
 	_ = fs.Parse(args)
 
-	randomSeed := int64(*flagSeed)
-	if randomSeed == 0 {
-		randomSeed = time.Now().Unix()
+	var seed int64
+	if *flagSeed == 0 {
+		seed = time.Now().Unix()
 	}
+
+	return generate(*flagOutputDir, seed)
+}
+
+func generate(dir string, randomSeed int64) error {
 	random := rand.New(rand.NewSource(randomSeed))
 
-	if err := os.MkdirAll(*flagOutputDir, 0o700); err != nil {
+	if err := os.MkdirAll(dir, 0o700); err != nil && !os.IsExist(err) {
 		return err
 	}
 
@@ -36,14 +41,16 @@ func cmdGenerate(args []string) error {
 	printerConfig := &irprint.Config{
 		Rand: random,
 	}
+
 	for _, f := range program.RuntimeFiles {
-		fullname := filepath.Join(*flagOutputDir, f.Name)
+		fullname := filepath.Join(dir, f.Name)
 		if err := os.WriteFile(fullname, f.Contents, 0o664); err != nil {
 			return fmt.Errorf("create %s file: %w", fullname, err)
 		}
 	}
+
 	for _, f := range program.Files {
-		fullname := filepath.Join(*flagOutputDir, f.Name)
+		fullname := filepath.Join(dir, f.Name)
 		fileContents := makeFileContents(f, printerConfig)
 		if err := os.WriteFile(fullname, fileContents, 0o664); err != nil {
 			return fmt.Errorf("create %s file: %w", fullname, err)

cmd/phpsmith/interpretator/kphp.go

@@ -0,0 +1,54 @@
+package interpretator
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"log"
+	"os/exec"
+	"sync"
+)
+
+var mu sync.Mutex
+
+func RunKPHP(ctx context.Context, dir string, seed int64) ([]byte, error) {
+	var (
+		outBuffer bytes.Buffer
+		errBuffer bytes.Buffer
+	)
+
+	binaryName := dir + "/" + dir
+
+	if err := func() error {
+		mu.Lock()
+		defer mu.Unlock()
+
+		compileCmd := exec.CommandContext(ctx, "kphp", "--mode", "cli", "-o", binaryName, dir+"/main.php")
+		compileCmd.Stdout, compileCmd.Stderr = &outBuffer, &errBuffer
+
+		if err := compileCmd.Run(); err != nil {
+			if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() > 1 {
+				log.Printf("non one or zero exit code found: %d, seed: %d \n", exitErr.ExitCode(), seed)
+			}
+			return err
+		}
+		return nil
+	}(); err != nil {
+		return nil, fmt.Errorf("on compile kphp: stdOut: %s, stdErr: %s", outBuffer.String(), errBuffer.String())
+	}
+
+	outBuffer.Reset()
+	errBuffer.Reset()
+
+	runCmd := exec.CommandContext(ctx, binaryName)
+	runCmd.Stdout, runCmd.Stderr = &outBuffer, &errBuffer
+
+	if err := runCmd.Run(); err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() > 1 {
+			log.Printf("non one or zero exit code found: %d, seed: %d \n", exitErr.ExitCode(), seed)
+		}
+		return nil, fmt.Errorf("on run kphp binary: stdOut: %s, stdErr: %s", outBuffer.String(), errBuffer.String())
+	}
+
+	return outBuffer.Bytes(), nil
+}

cmd/phpsmith/interpretator/php.go

@@ -0,0 +1,27 @@
+package interpretator
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"log"
+	"os/exec"
+)
+
+func RunPHP(ctx context.Context, dir string, seed int64) ([]byte, error) {
+	var (
+		outBuffer bytes.Buffer
+		errBuffer bytes.Buffer
+	)
+	phpCmd := exec.CommandContext(ctx, "php", "-f", dir+"/main.php")
+	phpCmd.Stdout, phpCmd.Stderr = &outBuffer, &errBuffer
+
+	if err := phpCmd.Run(); err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() > 1 {
+			log.Printf("non one or zero exit code found: %d, seed: %d \n", exitErr.ExitCode(), seed)
+		}
+		return nil, fmt.Errorf("on run php: stdOut: %s, stdErr: %s", outBuffer.String(), errBuffer.String())
+	}
+
+	return outBuffer.Bytes(), nil
+}

go.mod

@@ -2,4 +2,9 @@ module github.com/quasilyte/phpsmith
 
 go 1.17
 
-require github.com/cespare/subcmd v1.1.0
+require (
+	github.com/cespare/subcmd v1.1.0
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+)
+
+require github.com/google/go-cmp v0.5.8 // indirect

go.sum

@@ -1,2 +1,6 @@
 github.com/cespare/subcmd v1.1.0 h1:r60BAqAKOGcBjxHmV9/WYvq5Qbp3xW9ByB+fRjtty9U=
 github.com/cespare/subcmd v1.1.0/go.mod h1:wnVjukiuhSlhZSgGHUilbkHykG7Oglb0sJXpUQ+MoUw=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=