diff --git a/pkg/cmd/corset/check.go b/pkg/cmd/corset/check.go index 994430b36..65f11d2c5 100644 --- a/pkg/cmd/corset/check.go +++ b/pkg/cmd/corset/check.go @@ -18,7 +18,6 @@ import ( "os" "runtime" "runtime/pprof" - "slices" cmd_util "github.com/LFDT-Lineth/zkc/pkg/cmd/corset/util" "github.com/LFDT-Lineth/zkc/pkg/cmd/corset/view" @@ -221,21 +220,18 @@ func CheckTrace[F field.Element[F]](ir string, schema sc.AnySchema[F], builder i cfg CheckConfig, trace tr.Trace[F]) bool { // begin performance measurement var ( - mapping = module.IdentityMap[F](schema.Modules().Collect()...) - stats = util.NewPerfStats() - recoverable bool - errs []error + mapping = module.IdentityMap[F](schema.Modules().Collect()...) + stats = util.NewPerfStats() + errs []error ) // trace, errs = builder.Build(schema, trace) - // Check whether any invalid traces - recoverable = !slices.Contains(trace, nil) // Log cost of expansion stats.Log("Expanding trace columns") // Report any errors reportErrors(ir, errs) // Check whether considered unrecoverable - if !recoverable || len(errs) > 0 { + if len(errs) > 0 { return false } // diff --git a/pkg/cmd/corset/debug/schema.go b/pkg/cmd/corset/debug/schema.go index 86cfe9077..d817f2dc9 100644 --- a/pkg/cmd/corset/debug/schema.go +++ b/pkg/cmd/corset/debug/schema.go @@ -162,7 +162,7 @@ func printStaticContents[F field.Element[F]](module schema.Module[F], verbose bo } } -func printRegisters[F any](module schema.Module[F], prefix string, filter func(register.Register) bool) { +func printRegisters[F field.Element[F]](module schema.Module[F], prefix string, filter func(register.Register) bool) { var ( regT string ) @@ -190,7 +190,7 @@ func printRegisters[F any](module schema.Module[F], prefix string, filter func(r } } -func countRegisters[F any](module schema.Module[F], filter func(register.Register) bool) uint { +func countRegisters[F field.Element[F]](module schema.Module[F], filter func(register.Register) bool) uint { var count = uint(0) // for _, r := range module.Registers() { @@ -212,7 +212,7 @@ func requiresSpacing[F field.Element[F]](c schema.Constraint[F]) bool { return false } -func isEmptyModule[F any](module schema.Module[F]) bool { +func isEmptyModule[F field.Element[F]](module schema.Module[F]) bool { return len(module.Registers()) == 0 && module.Constraints().Count() == 0 && module.Assignments().Count() == 0 diff --git a/pkg/cmd/zkc/trace.go b/pkg/cmd/zkc/trace.go index 6a40b0ce6..0e1648098 100644 --- a/pkg/cmd/zkc/trace.go +++ b/pkg/cmd/zkc/trace.go @@ -16,18 +16,23 @@ import ( "encoding/hex" "fmt" "os" + "slices" "strconv" "strings" "github.com/LFDT-Lineth/zkc/pkg/cmd/corset" "github.com/LFDT-Lineth/zkc/pkg/schema/module" "github.com/LFDT-Lineth/zkc/pkg/trace" + "github.com/LFDT-Lineth/zkc/pkg/util" + "github.com/LFDT-Lineth/zkc/pkg/util/collection/array" + "github.com/LFDT-Lineth/zkc/pkg/util/collection/hash" "github.com/LFDT-Lineth/zkc/pkg/util/field" "github.com/LFDT-Lineth/zkc/pkg/util/field/bls12_377" "github.com/LFDT-Lineth/zkc/pkg/util/field/gf251" "github.com/LFDT-Lineth/zkc/pkg/util/field/gf8209" "github.com/LFDT-Lineth/zkc/pkg/util/field/koalabear" "github.com/LFDT-Lineth/zkc/pkg/util/termio" + "github.com/LFDT-Lineth/zkc/pkg/util/word" "github.com/LFDT-Lineth/zkc/pkg/zkc/vm" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -58,6 +63,10 @@ var traceFlags FlagChecks func runTraceCmd[F field.Element[F]](cmd *cobra.Command, args []string, field field.Config) { var ( + statsCfg traceStatsConfig[F] + // + moduleSummarisers = moduleSummarisers[F]() + // build = GetBuildConfig[F](cmd, field) // outputFile file for trace outputFile = GetString(cmd, "output") @@ -72,6 +81,8 @@ func runTraceCmd[F field.Element[F]](cmd *cobra.Command, args []string, field fi // extract sharding config sharding = GetString(cmd, "sharding") // + includes = GetStringArray(cmd, "include") + // trace trace.Trace[F] outputs map[string][]byte ) @@ -82,6 +93,13 @@ func runTraceCmd[F field.Element[F]](cmd *cobra.Command, args []string, field fi fmt.Println("error: \"trace\" does not support fast mode (use \"execute\" instead)") os.Exit(1) } + // Configure stats + statsCfg.human = !GetFlag(cmd, "raw") + statsCfg.maxCellWidth = GetUint(cmd, "cell-width") + statsCfg.summarisers = array.Filter(moduleSummarisers, func(m ModuleSummariser[F]) bool { + return slices.Contains(includes, m.Name) + }) + statsCfg.sortedBy = util.Some(GetUint(cmd, "sort")) // Configure tracing traceConfig := vm.DEFAULT_TRACE_CONFIG. WithPadding(build.padding). @@ -111,8 +129,8 @@ func runTraceCmd[F field.Element[F]](cmd *cobra.Command, args []string, field fi // print trace statistics (if requested). Only meaningful when a trace was // actually generated (i.e. no execution errors). if stats && len(errors) == 0 { - printTraceStats(trace...) - printModuleStats(trace...) + printTraceStats(statsCfg, trace...) + printModuleStats(statsCfg, trace...) } // print entire trace (if requested). Unlike the inspector, there is no way // to reveal a module which was hidden, so everything carrying data is shown @@ -160,15 +178,20 @@ func runTraceCmd[F field.Element[F]](cmd *cobra.Command, args []string, field fi //nolint:errcheck func init() { - rootCmd.AddCommand(traceCmd) traceCmd.Flags().StringP("output", "o", "", "specify output file for writing trace") traceCmd.Flags().String("sharding", "", "specify sharding strategy") traceCmd.Flags().BoolP("check", "c", false, "check generated trace against constraints") traceCmd.Flags().Bool("stats", false, "show overall stats for the generated trace") + traceCmd.Flags().BoolP("raw", "r", false, "show raw stats (rather than human-readable stats like 1K 234M 2G, etc)") + traceCmd.Flags().Uint("sort", 1, "sort table column") + traceCmd.Flags().Uint("cell-width", 32, "specify maximum display width for a cell") + traceCmd.Flags().StringArrayP("include", "i", []string{"columns", "lines", "cells", "bytes"}, + fmt.Sprintf("specify information to include in module summaries: %s", moduleSummariserOptions[koalabear.Element]())) traceCmd.Flags().BoolP("print", "p", false, "print the generated trace") traceCmd.Flags().Bool("sequential", false, "force sequential tracing") - traceCmd.Flags().BoolP("inspect", "i", false, "open the generated trace in the interactive inspector") + traceCmd.Flags().Bool("inspect", false, "open the generated trace in the interactive inspector") traceCmd.PersistentFlags().UintP("batch", "b", 1024, "specify batch size for constraint checking") + rootCmd.AddCommand(traceCmd) } func parseShardingConfig(spec string) vm.ShardingStrategy { @@ -203,10 +226,6 @@ func publicModule(name module.Name) bool { return !strings.HasPrefix(name, "$") } -// Column bit-width buckets reported by printTraceStats, matching those shown by -// the corset "trace --stats" command. -var traceStatBuckets = []struct{ lo, hi uint }{{1, 8}, {9, 16}, {17, 32}, {33, 128}, {129, 256}} - const ( oneK = 1000 oneM = oneK * oneK @@ -218,159 +237,206 @@ const ( // traced cells (both human-readable and raw) plus a breakdown of columns by // bit-width. Columns backed by field elements (i.e. native registers, which // have no fixed bit-width) are reported separately. -func printTraceStats[F field.Element[F]](shards ...trace.Shard[F]) { - for _, shard := range shards { - printShardStats(shard) +func printTraceStats[F field.Element[F]](cfg traceStatsConfig[F], shards ...trace.Shard[F]) { + // Render it. + tbl := termio.NewFormattedTable(3, uint(len(shards)+1)) + // + tbl.SetRow(0, termio.NewText("shard"), termio.NewText("cells"), termio.NewText("bytes")) + tbl.SetRule(1) + // + for i, shard := range shards { + var ( + cells, bytes = getShardStats(shard) + sid = fmt.Sprintf("%d", i) + ) + // + cs := humanCount(cfg.human, cells) + bs := humanCount(cfg.human, bytes) + // + tbl.SetRow(uint(i+1), termio.NewText(sid), termio.NewText(cs), termio.NewText(bs)) } + // + tbl.SetMaxWidths(64) + tbl.Print(AnsiEscapes) } -func printShardStats[F field.Element[F]](shard trace.Shard[F]) { +func getShardStats[F field.Element[F]](shard trace.Shard[F]) (uint64, uint64) { var ( - cells uint - counts = make([]uint, len(traceStatBuckets)) - native uint + cells uint64 + bytes uint64 ) // Tally cells and per-column bit-widths across all modules. for mid := range shard.Width() { mod := shard.Module(mid) - cells += mod.Width() * mod.Height() - // - for _, reg := range mod.Descriptor().Columns { - bitwidth := reg.Bitwidth - // Native (field-element) limbs have no fixed bit-width. - if bitwidth.IsEmpty() { - native++ - continue - } - // Otherwise, place the limb in its matching bit-width bucket. - for i, b := range traceStatBuckets { - if w := bitwidth.Unwrap(); w >= b.lo && w <= b.hi { - counts[i]++ - break - } - } - } - } - // Assemble the stats table. - rows := [][2]string{ - {"Cells", humanCount(cells)}, - {"Cells (raw)", fmt.Sprintf("%d", cells)}, - } - // - for i, b := range traceStatBuckets { - rows = append(rows, [2]string{fmt.Sprintf("Columns (%d..%d bits)", b.lo, b.hi), fmt.Sprintf("%d", counts[i])}) - } - // - if native > 0 { - rows = append(rows, [2]string{"Columns (native)", fmt.Sprintf("%d", native)}) - } - // Render it. - tbl := termio.NewFormattedTable(2, uint(len(rows))) - // - for i, row := range rows { - tbl.SetRow(uint(i), termio.NewText(row[0]), termio.NewText(row[1])) + cells += uint64(mod.Width()) * uint64(mod.Height()) + bytes += moduleBytesSummariser(mod) } // - tbl.SetMaxWidths(64) - tbl.Print(AnsiEscapes) + return cells, bytes } // humanCount formats a (potentially large) count using K/M/G suffixes, matching // the corset trace command's cell-count formatting. -func humanCount(total uint) string { +func humanCount(enable bool, total uint64) string { switch { - case total > oneG: + case enable && total > oneG: return fmt.Sprintf("%.01fG", float64(total)/oneG) - case total > oneM: + case enable && total > oneM: return fmt.Sprintf("%.01fM", float64(total)/oneM) - case total > oneK: + case enable && total > oneK: return fmt.Sprintf("%.01fK", float64(total)/oneK) default: return fmt.Sprintf("%d", total) } } -// Per-module summary column titles, matching those shown by the corset "trace -// --modules" command. -var moduleStatTitles = []string{"columns", "lines", "bitwidth", "cells", "nonzero", "bytes"} +type traceStatsConfig[F field.Element[F]] struct { + human bool + summarisers []ModuleSummariser[F] + sortedBy util.Option[uint] + maxCellWidth uint +} // printModuleStats prints a per-module summary for a raw (row-major) trace, much // like the corset trace command's module listing. For each module it reports // the column count, line (row) count, total bit-width, total cells, non-zero // cells and total bytes. Native (field-element) limbs, which have no fixed // bit-width, are excluded from the bit-width and byte totals. -func printModuleStats[F field.Element[F]](shards ...trace.Shard[F]) { +func printModuleStats[F field.Element[F]](cfg traceStatsConfig[F], shards ...trace.Shard[F]) { for _, shard := range shards { - printShardModuleStats(shard) + printShardModuleStats(cfg, shard) } } -func printShardModuleStats[F field.Element[F]](shard trace.Shard[F]) { +func printShardModuleStats[F field.Element[F]](cfg traceStatsConfig[F], shard trace.Shard[F]) { var ( n = shard.Width() - tbl = termio.NewFormattedTable(uint(len(moduleStatTitles))+1, n+1) + tbl = termio.NewFormattedTable(uint(len(cfg.summarisers))+1, n+1) ) // Set column titles (leaving the top-left cell blank, as corset does). - for i, title := range moduleStatTitles { - tbl.Set(uint(i)+1, 0, termio.NewText(title)) + for i, s := range cfg.summarisers { + tbl.Set(uint(i)+1, 0, termio.NewText(s.Name)) } - // Compute a summary row for each module. + // for mid := range n { var ( - mod = shard.Module(mid) - columns = mod.Width() - lines = mod.Height() - bitwidth uint - nonzero uint - bytes uint + mod = shard.Module(mid) + row = make([]termio.FormattedText, len(cfg.summarisers)+1) ) - // Sum per-limb bit-widths and byte requirements. - for _, reg := range mod.Descriptor().Columns { - if bw := reg.Bitwidth; bw.HasValue() { - w := bw.Unwrap() - bitwidth += w - bytes += byteWidth(w) * lines - } - } - // Count non-zero cells. - for cid := range columns { - col := mod.Column(cid) + // + row[0] = termio.NewText(mod.Name()) + // + for i, summary := range cfg.summarisers { + var count = summary.Summary(mod) // - for rid := range lines { - if !col.Get(rid).IsZero() { - nonzero++ - } - } + row[i+1] = termio.NewText(humanCount(cfg.human, count)) } // - tbl.SetRow(mid+1, - termio.NewText(mod.Name()), - termio.NewText(fmt.Sprintf("%d", columns)), - termio.NewText(fmt.Sprintf("%d", lines)), - termio.NewText(fmt.Sprintf("%d", bitwidth)), - termio.NewText(fmt.Sprintf("%d", columns*lines)), - termio.NewText(fmt.Sprintf("%d", nonzero)), - termio.NewText(fmt.Sprintf("%d", bytes)), - ) + tbl.SetRow(mid+1, row...) } // - tbl.SetMaxWidths(64) + tbl.SetMaxWidths(cfg.maxCellWidth) // Separate the summary stats (above) from the per-module stats (below) with a // horizontal rule as wide as the module table. fmt.Println(strings.Repeat("-", int(tbl.PrintedWidth()))) // Sort modules (descending) by cell count, skipping the title row. - tbl.Sort(1, termio.NewTableSorter().SortNumericalColumn(4).Invert()) + if cfg.sortedBy.HasValue() { + sorter := termio.NewTableSorter(). + SortNumericalColumn(cfg.sortedBy.Unwrap()). + Invert() + // + tbl.Sort(1, sorter) + } + // tbl.Print(AnsiEscapes) } -// byteWidth returns the number of bytes required to hold a value of the given -// bit-width (i.e. the bit-width rounded up to the nearest byte). -func byteWidth(bitwidth uint) uint { - w := bitwidth / 8 +// ============================================================================ +// Module Summarisers +// ============================================================================ + +// ModuleSummariser abstracts the notion of a function which summarises the +// contents of a given column. +type ModuleSummariser[F field.Element[F]] struct { + Name string + Description string + Summary func(trace.Module[F]) uint64 +} + +// Used to show the available options on the command-line. +func moduleSummariserOptions[F field.Element[F]]() string { + summarisers := "\n" + // + for _, s := range moduleSummarisers[F]() { + summarisers = fmt.Sprintf("%s--- %s (%s)\n", summarisers, s.Name, s.Description) + } + // + return summarisers +} + +// moduleSummarisers provides a list of suitable summarisers. +func moduleSummarisers[F field.Element[F]]() []ModuleSummariser[F] { + return []ModuleSummariser[F]{ + {"columns", "column count for module", moduleColumnSummariser[F]}, + {"lines", "line count for module", moduleLineSummariser[F]}, + {"cells", "total number of cells traced for module", moduleCellSummariser[F]}, + {"bytes", "total number of bytes used to hold trace", moduleBytesSummariser[F]}, + {"unique", "total number of unique cells traced for module", moduleUniqueSummariser[F]}, + } +} + +func moduleColumnSummariser[F field.Element[F]](mod trace.Module[F]) uint64 { + return uint64(mod.Width()) +} + +func moduleCellSummariser[F field.Element[F]](mod trace.Module[F]) uint64 { + return uint64(mod.Height()) * uint64(mod.Width()) +} + +func moduleLineSummariser[F field.Element[F]](mod trace.Module[F]) uint64 { + return uint64(mod.Height()) +} + +func moduleBytesSummariser[F field.Element[F]](mod trace.Module[F]) uint64 { + var count uint64 + // + for i := range mod.Descriptor().Columns { + var data = mod.Column(uint(i)) + // + if data != nil { + count += uint64(data.Bytes()) + } + } + // + return count +} + +func moduleUniqueSummariser[F field.Element[F]](mod trace.Module[F]) uint64 { + var count uint64 // - if bitwidth%8 != 0 { - w++ + for i := range mod.Descriptor().Columns { + var data = mod.Column(uint(i)) + // + if data != nil { + count += uniqueElementsSummariser(data) + } } // - return w + return count +} + +func uniqueElementsSummariser[F field.Element[F]](data array.Array[F]) uint64 { + // + elems := hash.NewSet[word.BigEndian](data.Len() / 2) + // Add all the elements + for i := uint(0); i < data.Len(); i++ { + var ( + ith = data.Get(i) + word word.BigEndian + ) + // + elems.Insert(word.SetBytes(ith.Bytes())) + } + // Done + return uint64(elems.Size()) } diff --git a/pkg/corset/compiler.go b/pkg/corset/compiler.go index 9a1c9ce34..37f2c3df6 100644 --- a/pkg/corset/compiler.go +++ b/pkg/corset/compiler.go @@ -21,6 +21,7 @@ import ( "github.com/LFDT-Lineth/zkc/pkg/ir/mir" "github.com/LFDT-Lineth/zkc/pkg/schema" "github.com/LFDT-Lineth/zkc/pkg/schema/register" + "github.com/LFDT-Lineth/zkc/pkg/util/field" "github.com/LFDT-Lineth/zkc/pkg/util/file" "github.com/LFDT-Lineth/zkc/pkg/util/source" "github.com/LFDT-Lineth/zkc/pkg/util/word" @@ -184,7 +185,7 @@ func constructSourceModule(schema schema.AnySchema[word.BigEndian], scope *compi // Determine the reference reference in the schema which corresponds with a // given (Corset) path. -func determineRegisterRef[F any](path file.Path, sc schema.AnySchema[F], env compiler.GlobalEnvironment, +func determineRegisterRef[F field.Element[F]](path file.Path, sc schema.AnySchema[F], env compiler.GlobalEnvironment, ) register.Ref { var ( mid schema.ModuleId diff --git a/pkg/ir/air/gadgets/bitwidth.go b/pkg/ir/air/gadgets/bitwidth.go index 35e07f908..4968a73bd 100644 --- a/pkg/ir/air/gadgets/bitwidth.go +++ b/pkg/ir/air/gadgets/bitwidth.go @@ -222,7 +222,7 @@ func (p *typeDecomposition[F]) AddSource(source register.Ref) { // Compute computes the values of columns defined by this assignment. // This requires computing the value of each byte column in the decomposition. func (p *typeDecomposition[F]) Compute(tr trace.Shard[F], schema sc.AnySchema[F], -) ([]array.MutArray[F], error) { +) ([]array.Array[F], error) { // Read inputs sources := assignment.ReadRegistersRef(tr, p.sources...) // Combine all sources @@ -328,11 +328,11 @@ func determineLimbSplit(bitwidth uint) (uint, uint) { // Combine all values from the given source registers into a single array of // data, whilst eliminating duplicates. -func combineSources[F field.Element[F]](bitwidth uint, sources []array.Array[F]) array.MutArray[F] { +func combineSources[F field.Element[F]](bitwidth uint, sources []array.Array[F]) array.Array[F] { // var ( n = sources[0].Len() - arr = array.Alloc[F](bitwidth, 0) + arr = array.Alloc[F](bitwidth) seen = hash.NewSet[F](n) ) // Add all values @@ -345,30 +345,30 @@ func combineSources[F field.Element[F]](bitwidth uint, sources []array.Array[F]) // record have seen item seen.Insert(ith) // append item - arr.Append(ith) + arr = arr.Append(ith) } } } // Done - return arr + return arr.Build() } -func computeDecomposition[F field.Element[F]](loWidth, hiWidth uint, vArr array.MutArray[F], -) []array.MutArray[F] { +func computeDecomposition[F field.Element[F]](loWidth, hiWidth uint, vArr array.Array[F], +) []array.Array[F] { // var ( - vLoArr = array.Alloc[F](loWidth, vArr.Len()) - vHiArr = array.Alloc[F](hiWidth, vArr.Len()) + vLoArr = array.Alloc[F](loWidth) + vHiArr = array.Alloc[F](hiWidth) ) // for i := range vArr.Len() { ith := vArr.Get(i) lo, hi := decompose(loWidth, ith) - vLoArr.Set(i, lo) - vHiArr.Set(i, hi) + vLoArr = vLoArr.Append(lo) + vHiArr = vHiArr.Append(hi) } // - return []array.MutArray[F]{vArr, vLoArr, vHiArr} + return []array.Array[F]{vArr, vLoArr.Build(), vHiArr.Build()} } // Decompose a given field element into its least and most significant limbs, diff --git a/pkg/ir/assignment/computed_register.go b/pkg/ir/assignment/computed_register.go index 059500de7..95eec5223 100644 --- a/pkg/ir/assignment/computed_register.go +++ b/pkg/ir/assignment/computed_register.go @@ -70,11 +70,10 @@ func (p *ComputedRegister[F]) Bounds(mid sc.ModuleId) util.Bounds { // creates a new column which contains the result of evaluating a given // expression on each row. func (p *ComputedRegister[F]) Compute(tr trace.Shard[F], schema sc.AnySchema[F], -) ([]array.MutArray[F], error) { +) ([]array.Array[F], error) { var ( trModule = tr.Module(p.Module) scModule = schema.Module(p.Module) - // Determine multiplied height height = trModule.Height() // FIXME: using a large bitwidth here ensures the underlying data is @@ -83,16 +82,21 @@ func (p *ComputedRegister[F]) Compute(tr trace.Shard[F], schema sc.AnySchema[F], // values outside the range of the computed register, but which we still // want to check are actually rejected (i.e. since they are simulating what // an attacker might do). - data = array.Alloc[F](math.MaxUint, height) - // Run computation - err = fwdComputation(height, data, p.Expr, trModule, scModule, p.Module) + data = array.Alloc[F](math.MaxUint) ) - // Sanity check - if err != nil { - return nil, err + // Forwards computation + for i := range height { + val, err := p.Expr.EvalAt(i, trModule, scModule) + // error check + if err != nil { + e := fmt.Sprintf("%s for %s", err.Error(), p.Expr.Lisp(false, scModule).String(true)) + return nil, constraint.NewInternalFailure[F](scModule.Name(), p.Module, i, e) + } + // Write data + data = data.Append(val) } // Done - return []array.MutArray[F]{data}, err + return []array.Array[F]{data.Build()}, nil } // Consistent performs some simple checks that the given assignment is @@ -155,20 +159,3 @@ func (p *ComputedRegister[F]) Lisp(schema sc.AnySchema[F]) sexp.SExp { p.Expr.Lisp(false, module), }) } - -func fwdComputation[F field.Element[F]](height uint, data array.MutArray[F], expr term.Evaluable[F], - trMod trace.Module[F], scMod register.Map, ctx sc.ModuleId) error { - // Forwards computation - for i := range height { - val, err := expr.EvalAt(i, trMod, scMod) - // error check - if err != nil { - e := fmt.Sprintf("%s for %s", err.Error(), expr.Lisp(false, scMod).String(true)) - return constraint.NewInternalFailure[F](scMod.Name(), ctx, i, e) - } - // Write data - data.Set(i, val) - } - // - return nil -} diff --git a/pkg/ir/assignment/pseudo_inverse.go b/pkg/ir/assignment/pseudo_inverse.go index 179c42fd4..11838b0d5 100644 --- a/pkg/ir/assignment/pseudo_inverse.go +++ b/pkg/ir/assignment/pseudo_inverse.go @@ -57,29 +57,19 @@ func (e *PseudoInverse[F]) Bounds(mid schema.ModuleId) util.Bounds { } // Compute performs the inversion. -func (e *PseudoInverse[F]) Compute(tr trace.Shard[F], schema schema.AnySchema[F]) ([]array.MutArray[F], error) { +func (e *PseudoInverse[F]) Compute(tr trace.Shard[F], schema schema.AnySchema[F]) ([]array.Array[F], error) { var ( trModule = tr.Module(e.Target.Module()) scModule = schema.Module(e.Target.Module()) - err error ) - // Determine multiplied height - height := trModule.Height() - // FIXME: using a large bitwidth here ensures the underlying data is - // represented using a full field element, rather than e.g. some smaller - // number of bytes. This is needed to handle reject tests which can produce - // values outside the range of the computed register, but which we still - // want to check are actually rejected (i.e. since they are simulating what - // an attacker might do). - data := array.Alloc[F](math.MaxUint, height) // Expand the trace - data, err = invert(data, e.Expr, trModule, scModule) + data, err := invert(e.Expr, trModule, scModule) // Sanity check if err != nil { return nil, err } // Done - return []array.MutArray[F]{data}, err + return []array.Array[F]{data}, err } // Consistent performs some simple checks that the given assignment is @@ -157,23 +147,29 @@ func (e *PseudoInverse[F]) RequiredCells(row int, mid trace.ModuleId) *set.AnySo } func invert[F field.Element[F]]( - data array.MutArray[F], expr term.Evaluable[F], trMod trace.Module[F], scMod schema.Module[F], -) (array.MutArray[F], error) { +) (array.Array[F], error) { + // FIXME: this process is relatively inefficient for modules with + // signifciant padding. In particular, because it enumerates each padding + // value individually, rather than treating them as one large chunk. + data := make([]F, trMod.Height()) // Forwards computation - for i := range data.Len() { + for i := range trMod.Height() { val, err := expr.EvalAt(i, trMod, scMod) // error check if err != nil { - return data, err + return nil, err } // - data.Set(i, val) + data[i] = val } - // + // Apply the batch inversion algorithm. field.BatchInvert(data) - // - return data, nil + // NOTE: using a large bitwidth here ensures the underlying data is + // represented using a full field element, rather than e.g. some smaller + // number of bytes. This is necessary since the inverse of a value can + // occupy anything. + return array.NewStaticArray(math.MaxUint, data...).Build(), nil } diff --git a/pkg/ir/builder/alignment.go b/pkg/ir/builder/alignment.go index cf6d3d59b..b1f72b74e 100644 --- a/pkg/ir/builder/alignment.go +++ b/pkg/ir/builder/alignment.go @@ -23,12 +23,6 @@ import ( "github.com/LFDT-Lineth/zkc/pkg/util/field" ) -// ArrayModule provides a convenient alias. -type ArrayModule[F field.Element[F]] = *trace.CompactModule[F] - -// ArrayTrace provides a convenient alias. -type ArrayTrace[F field.Element[F]] = *trace.Array[F, ArrayModule[F]] - // PaddingStrategy captures the notion of an algorithm that determines how much front padding is added to each module // when expanding a trace (see TraceBuilder.WithPadding). type PaddingStrategy func(height, multiplier uint) uint @@ -58,11 +52,11 @@ type Config struct { // not. Specifically, expanding traces don't need to include data for computed // columns, since these will be added during expansion. func AlignAndPad[F field.Element[F]](config Config, schema sc.AnySchema[F], tr trace.Shard[F], -) (ArrayTrace[F], []error) { +) (trace.Shard[F], []error) { // var ( errors []error - modules = make([]ArrayModule[F], schema.Width()) + modules = make([]trace.Module[F], schema.Width()) modmap = make(map[string]uint) seen = make([]bool, tr.Width()) ) @@ -88,7 +82,7 @@ func AlignAndPad[F field.Element[F]](config Config, schema sc.AnySchema[F], tr t // all computed) is legitimately allowed to have no presence in // the trace at all. Any genuinely missing data is detected // below, on a column-by-column basis. - trMod = trace.NewCompactModule[F](trace.NewModuleDescriptor(scMod.Name(), nil)) + trMod = trace.NewModule[F](trace.NewModuleDescriptor(scMod.Name(), nil)) } // Align trace modules[i], errs = alignModule(config, scMod, trMod) @@ -109,11 +103,11 @@ func AlignAndPad[F field.Element[F]](config Config, schema sc.AnySchema[F], tr t // errors = append(errors, errs...) // Done - return trace.NewArray(modules), errors + return trace.NewShard(modules), errors } func alignModule[F field.Element[F]](config Config, scMod sc.Module[F], trMod trace.Module[F], -) (ArrayModule[F], []error) { +) (trace.Module[F], []error) { var ( errors []error width = uint(len(scMod.Registers())) @@ -121,7 +115,7 @@ func alignModule[F field.Element[F]](config Config, scMod sc.Module[F], trMod tr // modules to ensure they are consistent. height uint descriptors = make([]trace.ColumnDescriptor, width) - columns = make([]array.MutArray[F], width) + columns = make([]array.Array[F], width) regmap = make(map[string]uint) seen = make([]bool, trMod.Width()) ) @@ -156,7 +150,7 @@ func alignModule[F field.Element[F]](config Config, scMod sc.Module[F], trMod tr errs = append(errs, fmt.Errorf("duplicate column '%s' in module '%s' of trace", ith.Name, trMod.Name())) } else { // Clone underlying data - columns[cid] = trMod.MutColumn(i) + columns[cid] = trMod.Column(i) // Mark column as seen seen[cid] = true // Update maximum height @@ -181,5 +175,5 @@ func alignModule[F field.Element[F]](config Config, scMod sc.Module[F], trMod tr } } // Done - return trace.NewCompactModule(trace.NewModuleDescriptor(scMod.Name(), descriptors), columns...), errors + return trace.NewModule(trace.NewModuleDescriptor(scMod.Name(), descriptors), columns...), errors } diff --git a/pkg/ir/builder/expander.go b/pkg/ir/builder/expander.go index c051a5ea6..c1113d66b 100644 --- a/pkg/ir/builder/expander.go +++ b/pkg/ir/builder/expander.go @@ -19,6 +19,7 @@ import ( "github.com/LFDT-Lineth/zkc/pkg/schema/register" "github.com/LFDT-Lineth/zkc/pkg/util/collection/bit" "github.com/LFDT-Lineth/zkc/pkg/util/collection/iter" + "github.com/LFDT-Lineth/zkc/pkg/util/field" ) // Expander encapsulates key state required in order to expand traces safely @@ -27,7 +28,7 @@ import ( // However, some assignments must be run before others. For example, if one // assignment depends upon a column which is computed by another, then the // latter must go first. -type Expander[F any] struct { +type Expander[F field.Element[F]] struct { // Width records the number of modules in the schema. width uint // Set of assignments yet to run @@ -39,7 +40,7 @@ type Expander[F any] struct { } // NewExpander constructs a new trace expander for a given set of assignments. -func NewExpander[F any](width uint, assignments iter.Iterator[sc.Assignment[F]]) Expander[F] { +func NewExpander[F field.Element[F]](width uint, assignments iter.Iterator[sc.Assignment[F]]) Expander[F] { var ( notReady bit.Set expanding bit.Set diff --git a/pkg/ir/builder/expansion.go b/pkg/ir/builder/expansion.go index b9d4a3b10..6b807b477 100644 --- a/pkg/ir/builder/expansion.go +++ b/pkg/ir/builder/expansion.go @@ -17,6 +17,7 @@ import ( sc "github.com/LFDT-Lineth/zkc/pkg/schema" "github.com/LFDT-Lineth/zkc/pkg/schema/register" + "github.com/LFDT-Lineth/zkc/pkg/trace" "github.com/LFDT-Lineth/zkc/pkg/util" "github.com/LFDT-Lineth/zkc/pkg/util/collection/array" "github.com/LFDT-Lineth/zkc/pkg/util/field" @@ -25,7 +26,8 @@ import ( // TraceExpansion expands a given trace according to a given schema. More // specifically, that means computing the actual values for any assignments. // This is done using a straightforward sequential algorithm. -func TraceExpansion[F field.Element[F]](config Config, schema sc.AnySchema[F], trace ArrayTrace[F]) error { +func TraceExpansion[F field.Element[F]](config Config, schema sc.AnySchema[F], tr trace.Shard[F], +) (trace.Shard[F], error) { // var ( err error @@ -33,36 +35,39 @@ func TraceExpansion[F field.Element[F]](config Config, schema sc.AnySchema[F], t // if config.Parallel { // Run (parallel) trace expansion - err = ParallelTraceExpansion(config.BatchSize, schema, trace) + tr, err = ParallelTraceExpansion(config.BatchSize, schema, tr) } else { - err = SequentialTraceExpansion(schema, trace) + tr, err = SequentialTraceExpansion(schema, tr) } // - return err + return tr, err } // SequentialTraceExpansion expands a given trace according to a given schema. // More specifically, that means computing the actual values for any // assignments. This is done using a straightforward sequential algorithm. -func SequentialTraceExpansion[F field.Element[F]](schema sc.AnySchema[F], trace ArrayTrace[F]) error { +func SequentialTraceExpansion[F field.Element[F]](schema sc.AnySchema[F], tr trace.Shard[F]) (trace.Shard[F], error) { var ( err error expander = NewExpander(schema.Width(), schema.Assignments()) + modules = extractModules(tr) ) + // Allocate new trace from expanded modules + tr = trace.NewShard(modules) // Compute each assignment in turn for !expander.Done() { - var cols []array.MutArray[F] + var cols []array.Array[F] // Get next assignment ith := expander.Next(1)[0] // Compute ith assignment(s) - if cols, err = ith.Compute(trace, schema); err != nil { - return err + if cols, err = ith.Compute(tr, schema); err != nil { + return tr, err } // Fill all computed columns - fillComputedColumns(ith.RegistersWritten(), cols, trace) + fillComputedColumns(ith.RegistersWritten(), cols, modules) } // Done - return nil + return tr, nil } // ParallelTraceExpansion performs trace expansion using concurrently executing @@ -70,12 +75,16 @@ func SequentialTraceExpansion[F field.Element[F]](schema sc.AnySchema[F], trace // continuous approach. This is for two reasons: firstly, the latter would // require locks that would slow down evaluation performance; secondly, the vast // majority of jobs are run in the very first wave. -func ParallelTraceExpansion[F field.Element[F]](batchsize uint, schema sc.AnySchema[F], trace ArrayTrace[F]) error { +func ParallelTraceExpansion[F field.Element[F]](batchsize uint, schema sc.AnySchema[F], tr trace.Shard[F], +) (trace.Shard[F], error) { var ( batchNum = 0 // expander = NewExpander(schema.Width(), schema.Assignments()) + modules = extractModules(tr) ) + // Allocate new trace from expanded modules + tr = trace.NewShard(modules) // Iterate until all assignments processed. for !expander.Done() { var ( @@ -84,17 +93,17 @@ func ParallelTraceExpansion[F field.Element[F]](batchsize uint, schema sc.AnySch ) // Process all assignments in this wave in parallel using a worker pool. results := array.ParallelMap(batch, func(_ uint, ith sc.Assignment[F]) columnBatch[F] { - cols, err := ith.Compute(trace, schema) + cols, err := ith.Compute(tr, schema) return columnBatch[F]{ith.RegistersWritten(), cols, err} }) // Check for errors and fill computed columns into the trace. for _, r := range results { if r.err != nil { // Fail immediately - return r.err + return tr, r.err } // - fillComputedColumns(r.targets, r.columns, trace) + fillComputedColumns(r.targets, r.columns, modules) } // Log stats about this batch stats.Log(fmt.Sprintf("Expansion batch %d (remaining %d)", batchNum, expander.Count())) @@ -102,22 +111,34 @@ func ParallelTraceExpansion[F field.Element[F]](batchsize uint, schema sc.AnySch batchNum++ } // Done - return nil + return tr, nil +} + +// extractModules copies out the modules underlying a given trace, so they can +// be progressively updated (via fillComputedColumns) as expansion proceeds. +func extractModules[F field.Element[F]](tr trace.Shard[F]) []trace.Module[F] { + modules := make([]trace.Module[F], tr.Width()) + // + for i := range modules { + modules[i] = tr.RawModule(uint(i)) + } + // + return modules } // Fill a set of columns with their computed results. The column index is that // of the first column in the sequence, and subsequent columns are index // consecutively. -func fillComputedColumns[F field.Element[F]](refs []register.Ref, cols []array.MutArray[F], trace ArrayTrace[F]) { +func fillComputedColumns[F field.Element[F]](refs []register.Ref, cols []array.Array[F], modules []trace.Module[F]) { // Add all columns for i, ref := range refs { var ( - rid = ref.Column().Unwrap() - module = trace.RawModule(ref.Module()) - col = cols[i] + mid = ref.Module() + rid = ref.Column().Unwrap() + col = cols[i] ) - // Expand it - module.Expand(rid, col) + // Expand it, recording the updated module. + modules[mid] = modules[mid].Expand(rid, col) } } @@ -126,7 +147,7 @@ type columnBatch[F field.Element[F]] struct { // Target registers for this batch targets []register.Ref // The computed columns in this batch. - columns []array.MutArray[F] + columns []array.Array[F] // An error (should one arise) err error } diff --git a/pkg/ir/builder/padding.go b/pkg/ir/builder/padding.go index 2fe4ea997..9b03ba7c6 100644 --- a/pkg/ir/builder/padding.go +++ b/pkg/ir/builder/padding.go @@ -30,13 +30,13 @@ import ( // directly from the trace given to AlignAndPad, and that trace can be shared // with (and reused by) the caller, so the modules returned from here must // never retain that aliasing. -func padModules[F field.Element[F]](config Config, schema sc.AnySchema[F], mods []ArrayModule[F], -) ([]ArrayModule[F], []error) { +func padModules[F field.Element[F]](config Config, schema sc.AnySchema[F], mods []trace.Module[F], +) ([]trace.Module[F], []error) { var ( // Determine the set of minimal trace sizes minimums = determineMinimumTraceHeight(schema) - columns, errs = flattenTrace(schema, trace.NewArray(mods)) - data []array.MutArray[F] + columns, errs = flattenTrace(schema, trace.NewShard(mods)) + data []array.Array[F] mapfn = paddingMapFn(config, schema, mods, minimums) ) // @@ -54,13 +54,13 @@ func padModules[F field.Element[F]](config Config, schema sc.AnySchema[F], mods // Columns belonging to a static module, and columns which are not yet // assigned (e.g. an unfilled computed column, prior to expansion), are passed // through unchanged. -func paddingMapFn[F field.Element[F]](config Config, schema sc.AnySchema[F], mods []ArrayModule[F], - minimums []uint) func(uint, trace.ColumnRef) array.MutArray[F] { +func paddingMapFn[F field.Element[F]](config Config, schema sc.AnySchema[F], mods []trace.Module[F], + minimums []uint) func(uint, trace.ColumnRef) array.Array[F] { // - return func(_ uint, p trace.ColumnRef) array.MutArray[F] { + return func(_ uint, p trace.ColumnRef) array.Array[F] { var ( mid = p.Module() - col = mods[mid].MutColumn(p.Column().Unwrap()) + col = mods[mid].Column(p.Column().Unwrap()) scMod = schema.Module(mid) ) // @@ -69,7 +69,6 @@ func paddingMapFn[F field.Element[F]](config Config, schema sc.AnySchema[F], mod } // var ( - zero F height = mods[mid].Height() // calculate taget height, whilst ensuring minimum enforced. target = config.Padding(max(minimums[mid], height), 1) @@ -80,7 +79,7 @@ func paddingMapFn[F field.Element[F]](config Config, schema sc.AnySchema[F], mod front = target - height } // - return col.Pad(front, 0, zero) + return col.Pad(front) } } @@ -118,25 +117,25 @@ func determineMinimumTraceHeight[F field.Element[F]](schema sc.AnySchema[F]) []u // produced by mapping over the (module,column) pairs from flattenTrace -- // back into their enclosing modules, using each module's original descriptor // (which padding never changes). -func rebuildModules[F field.Element[F]](mods []ArrayModule[F], columns []trace.ColumnRef, - padded []array.MutArray[F]) []ArrayModule[F] { +func rebuildModules[F field.Element[F]](mods []trace.Module[F], columns []trace.ColumnRef, + padded []array.Array[F]) []trace.Module[F] { var ( - result = make([]ArrayModule[F], len(mods)) - buffers = make([][]array.MutArray[F], len(mods)) + result = make([]trace.Module[F], len(mods)) + buffers = make([][]array.Array[F], len(mods)) ) // Regroup padded columns by their enclosing module. for i, p := range columns { var mid = p.Module() // if buffers[mid] == nil { - buffers[mid] = make([]array.MutArray[F], mods[mid].Width()) + buffers[mid] = make([]array.Array[F], mods[mid].Width()) } // buffers[mid][p.Column().Unwrap()] = padded[i] } // Reconstruct each module using its original descriptor. for mid, mod := range mods { - result[mid] = trace.NewCompactModule(mod.Descriptor(), buffers[mid]...) + result[mid] = trace.NewModule(mod.Descriptor(), buffers[mid]...) } // return result diff --git a/pkg/ir/mir/schema.go b/pkg/ir/mir/schema.go index 3a2056d7d..a60fbabdf 100644 --- a/pkg/ir/mir/schema.go +++ b/pkg/ir/mir/schema.go @@ -38,12 +38,12 @@ type ( Schema[F field.Element[F]] = schema.UniformSchema[F, Module[F]] // Term represents the fundamental for arithmetic expressions in the MIR // representation. - Term[F any] interface { + Term[F field.Element[F]] interface { term.Expr[F, Term[F]] } // LogicalTerm represents the fundamental for logical expressions in the MIR // representation. - LogicalTerm[F any] interface { + LogicalTerm[F field.Element[F]] interface { term.Logical[F, LogicalTerm[F]] } // Computation captures the notion of computations used in a small number of places. diff --git a/pkg/ir/schema_builder.go b/pkg/ir/schema_builder.go index d346f1d1d..2a9a12766 100644 --- a/pkg/ir/schema_builder.go +++ b/pkg/ir/schema_builder.go @@ -25,7 +25,7 @@ import ( // BuildableModule embodies the notion of a module which can be initialised from // the various required components. This provides a useful way for constructing // modules once all the various pieces of information have been finalised. -type BuildableModule[F any, C schema.Constraint[F], M any] interface { +type BuildableModule[F field.Element[F], C schema.Constraint[F], M any] interface { Init(name module.Name, public, private, synthetic, native, static bool) M // Add one or more assignments to this buildable module AddAssignments(assignments ...schema.Assignment[F]) diff --git a/pkg/ir/term/computation.go b/pkg/ir/term/computation.go index 2dee65234..76825094d 100644 --- a/pkg/ir/term/computation.go +++ b/pkg/ir/term/computation.go @@ -20,13 +20,13 @@ import ( // Computation represents an "unbound" term. That is, it captures any possible // term (i.e. rather than a fixed set as for MIR or AIR, etc). -type Computation[F any] interface { +type Computation[F field.Element[F]] interface { Expr[F, Computation[F]] } // LogicalComputation represents an "unbound" term. That is, it captures any // possible term (i.e. rather than a fixed set as for MIR or AIR, etc). -type LogicalComputation[F any] interface { +type LogicalComputation[F field.Element[F]] interface { Logical[F, LogicalComputation[F]] } diff --git a/pkg/ir/term/term.go b/pkg/ir/term/term.go index b4ca0ceb7..1ab1d5449 100644 --- a/pkg/ir/term/term.go +++ b/pkg/ir/term/term.go @@ -42,7 +42,7 @@ type Contextual interface { // produce an evaluation point. For example, expressions in the // Mid-Level or Arithmetic-Level IR can all be evaluated at rows of a // table. -type Evaluable[F any] interface { +type Evaluable[F field.Element[F]] interface { util.Boundable Contextual // EvalAt evaluates this expression in a given tabular context. @@ -75,7 +75,7 @@ type Shiftable[T any] interface { } // Expr represents a component of an MIR/AIR expression. -type Expr[F any, T any] interface { +type Expr[F field.Element[F], T any] interface { Contextual Shiftable[T] Evaluable[F] @@ -98,7 +98,7 @@ type Costable interface { // Evaluable is that, for historical reasons, logical constraints cannot be // Evaluable (i.e. because they return multiple values, rather than a single // value). However, such constraints remain testable. -type Testable[F any] interface { +type Testable[F field.Element[F]] interface { util.Boundable Contextual // TestAt evaluates this expression in a given tabular context and checks it @@ -114,7 +114,7 @@ type Testable[F any] interface { // Logical represents a term which can be tested for truth or falsehood. // For example, an equality comparing two arithmetic terms is a logical term. -type Logical[F any, T any] interface { +type Logical[F field.Element[F], T any] interface { Contextual Shiftable[T] Testable[F] diff --git a/pkg/ir/term/util.go b/pkg/ir/term/util.go index d11bf19c9..bfd11eeff 100644 --- a/pkg/ir/term/util.go +++ b/pkg/ir/term/util.go @@ -62,7 +62,7 @@ func isOne[F field.Element[F], T Expr[F, T]](term T) bool { return false } -func lispOfLogicalTerms[F any, T Logical[F, T]](global bool, mapping register.Map, op string, +func lispOfLogicalTerms[F field.Element[F], T Logical[F, T]](global bool, mapping register.Map, op string, exprs []T) sexp.SExp { // arr := make([]sexp.SExp, 1+len(exprs)) @@ -79,7 +79,8 @@ func lispOfLogicalTerms[F any, T Logical[F, T]](global bool, mapping register.Ma return sexp.NewList(arr) } -func lispOfTerms[F any, E any, T Expr[F, E]](global bool, mapping register.Map, op string, exprs []T) sexp.SExp { +func lispOfTerms[F field.Element[F], E any, T Expr[F, E]](global bool, mapping register.Map, op string, exprs []T, +) sexp.SExp { arr := make([]sexp.SExp, 1+len(exprs)) arr[0] = sexp.NewSymbol(op) // Translate arguments diff --git a/pkg/ir/trace_builder.go b/pkg/ir/trace_builder.go index 8a693c659..d75ad552b 100644 --- a/pkg/ir/trace_builder.go +++ b/pkg/ir/trace_builder.go @@ -136,7 +136,7 @@ func (tb TraceBuilder[F]) buildShard(schema sc.AnySchema[F], shard uint, tf trac ) (tr trace.Shard[F], errs []error) { // var ( - atr builder.ArrayTrace[F] + atr trace.Shard[F] // config = builder.Config{ Parallel: false, @@ -147,19 +147,20 @@ func (tb TraceBuilder[F]) buildShard(schema sc.AnySchema[F], shard uint, tf trac ) // Apply trace alignment and padding if atr, errs = builder.AlignAndPad(config, schema, tf); len(errs) > 0 { - return nil, errs + return atr, errs } // Apply trace expansion (if requested) if tb.expand { + var err error // Expand trace - if err := builder.TraceExpansion(config, schema, atr); err != nil { - return nil, append(errs, err) + if atr, err = builder.TraceExpansion(config, schema, atr); err != nil { + return atr, append(errs, err) } // Validate expanded trace if tb.validate { // Run (parallel) trace validation - if errs := builder.TraceValidation(config, schema, atr); len(errs) > 0 { - return nil, errs + if errs = builder.TraceValidation(config, schema, atr); len(errs) > 0 { + return atr, errs } } } diff --git a/pkg/schema/assignment.go b/pkg/schema/assignment.go index 67cf9db49..03848bffa 100644 --- a/pkg/schema/assignment.go +++ b/pkg/schema/assignment.go @@ -17,6 +17,7 @@ import ( "github.com/LFDT-Lineth/zkc/pkg/trace" "github.com/LFDT-Lineth/zkc/pkg/util" "github.com/LFDT-Lineth/zkc/pkg/util/collection/array" + "github.com/LFDT-Lineth/zkc/pkg/util/field" "github.com/LFDT-Lineth/zkc/pkg/util/source/sexp" ) @@ -26,7 +27,7 @@ import ( // have an associated assignment. A good example of an assignment is computed // the multiplicative inverse of a column in order to implement a non-zero // check. -type Assignment[F any] interface { +type Assignment[F field.Element[F]] interface { // For the given module, determine any well-definedness bounds implied by // this assignment in both the negative (left) or positive (right) // directions. For example, consider an expression such as "(shift X -1)". @@ -39,7 +40,7 @@ type Assignment[F any] interface { // assignment depends must exist (e.g. are either inputs or have been // computed already). Computed columns do not exist in the original trace, // but are added during trace expansion to form the final trace. - Compute(trace.Shard[F], AnySchema[F]) ([]array.MutArray[F], error) + Compute(trace.Shard[F], AnySchema[F]) ([]array.Array[F], error) // Consistent applies a number of internal consistency checks. Whilst not // strictly necessary, these can highlight otherwise hidden problems as an aid // to debugging. diff --git a/pkg/schema/constraint.go b/pkg/schema/constraint.go index afe5ab70f..49cf035a6 100644 --- a/pkg/schema/constraint.go +++ b/pkg/schema/constraint.go @@ -23,12 +23,13 @@ import ( "github.com/LFDT-Lineth/zkc/pkg/util" "github.com/LFDT-Lineth/zkc/pkg/util/collection" "github.com/LFDT-Lineth/zkc/pkg/util/collection/array" + "github.com/LFDT-Lineth/zkc/pkg/util/field" "github.com/LFDT-Lineth/zkc/pkg/util/source/sexp" ) // Constraint represents an element which can "accept" a trace, or either reject // with an error (or eventually perhaps report a warning). -type Constraint[F any] interface { +type Constraint[F field.Element[F]] interface { // Accepts determines whether a given (local) constraint accepts a given set // of traces or not. If not, a failure is produced. Observe that, for // global constraints, this is a no-op. diff --git a/pkg/schema/constraint/failure.go b/pkg/schema/constraint/failure.go index 19640083d..a4f89c60e 100644 --- a/pkg/schema/constraint/failure.go +++ b/pkg/schema/constraint/failure.go @@ -18,11 +18,12 @@ import ( "github.com/LFDT-Lineth/zkc/pkg/schema" "github.com/LFDT-Lineth/zkc/pkg/trace" "github.com/LFDT-Lineth/zkc/pkg/util/collection/set" + "github.com/LFDT-Lineth/zkc/pkg/util/field" ) // InternalFailure is a generic mechanism for reporting failures, particularly // as arising from evaluation of a given expression. -type InternalFailure[F any] struct { +type InternalFailure[F field.Element[F]] struct { // Handle of the failing constraint handle string // Module in which constraint failed. @@ -34,7 +35,7 @@ type InternalFailure[F any] struct { } // NewInternalFailure constructs a new internal failure object. -func NewInternalFailure[F any](handle string, ctx schema.ModuleId, row uint, +func NewInternalFailure[F field.Element[F]](handle string, ctx schema.ModuleId, row uint, err string) *InternalFailure[F] { // return &InternalFailure[F]{handle, ctx, row, err} diff --git a/pkg/schema/constraint/lookup/failure.go b/pkg/schema/constraint/lookup/failure.go index be7e762e4..ed33b6a34 100644 --- a/pkg/schema/constraint/lookup/failure.go +++ b/pkg/schema/constraint/lookup/failure.go @@ -18,10 +18,11 @@ import ( "github.com/LFDT-Lineth/zkc/pkg/schema" "github.com/LFDT-Lineth/zkc/pkg/trace" "github.com/LFDT-Lineth/zkc/pkg/util/collection/set" + "github.com/LFDT-Lineth/zkc/pkg/util/field" ) // Failure provides structural information about a failing lookup constraint. -type Failure[F any] struct { +type Failure[F field.Element[F]] struct { // Handle of the failing constraint LookupHandle string // SourceId gives the set identifier of the source diff --git a/pkg/schema/constraint/ranged/failure.go b/pkg/schema/constraint/ranged/failure.go index b85a0a64a..85cc69e28 100644 --- a/pkg/schema/constraint/ranged/failure.go +++ b/pkg/schema/constraint/ranged/failure.go @@ -19,10 +19,11 @@ import ( "github.com/LFDT-Lineth/zkc/pkg/schema/register" "github.com/LFDT-Lineth/zkc/pkg/trace" "github.com/LFDT-Lineth/zkc/pkg/util/collection/set" + "github.com/LFDT-Lineth/zkc/pkg/util/field" ) // Failure provides structural information about a failing type constraint. -type Failure[F any] struct { +type Failure[F field.Element[F]] struct { // Handle of the failing constraint RangeHandle string // Enclosing context diff --git a/pkg/schema/constraint/util.go b/pkg/schema/constraint/util.go index 7a2ccde6f..912346574 100644 --- a/pkg/schema/constraint/util.go +++ b/pkg/schema/constraint/util.go @@ -18,12 +18,14 @@ import ( "github.com/LFDT-Lineth/zkc/pkg/ir/term" "github.com/LFDT-Lineth/zkc/pkg/schema" "github.com/LFDT-Lineth/zkc/pkg/trace" + "github.com/LFDT-Lineth/zkc/pkg/util/field" ) // CheckConsistent performs a simple consistency check for terms in a given // module. Specifically, to check that: (1) the module exists; (2) all used // registers existing with then given module. -func CheckConsistent[F any, E term.Contextual](module uint, schema schema.AnySchema[F], terms ...E) []error { +func CheckConsistent[F field.Element[F], E term.Contextual](module uint, schema schema.AnySchema[F], terms ...E, +) []error { var errs []error // Sanity check module if module >= schema.Width() { @@ -49,7 +51,7 @@ func CheckConsistent[F any, E term.Contextual](module uint, schema schema.AnySch // DetermineHandle is a very simple helper which determines a suitable qualified // name for the given constraint handle. -func DetermineHandle[F any](handle string, ctx schema.ModuleId, tr trace.Shard[F]) string { +func DetermineHandle[F field.Element[F]](handle string, ctx schema.ModuleId, tr trace.Shard[F]) string { modName := tr.Module(ctx).Name() // return trace.QualifiedColumnName(modName, handle) diff --git a/pkg/schema/constraint/vanishing/failure.go b/pkg/schema/constraint/vanishing/failure.go index b739f4b49..e196a2dba 100644 --- a/pkg/schema/constraint/vanishing/failure.go +++ b/pkg/schema/constraint/vanishing/failure.go @@ -20,10 +20,11 @@ import ( "github.com/LFDT-Lineth/zkc/pkg/trace" "github.com/LFDT-Lineth/zkc/pkg/util/collection/array" "github.com/LFDT-Lineth/zkc/pkg/util/collection/set" + "github.com/LFDT-Lineth/zkc/pkg/util/field" ) // Failure provides structural information about a failing vanishing constraint. -type Failure[F any] struct { +type Failure[F field.Element[F]] struct { // Handle of the failing constraint VanishingHandle string // Constraint expression diff --git a/pkg/schema/module.go b/pkg/schema/module.go index 5d65ef05f..c218d7789 100644 --- a/pkg/schema/module.go +++ b/pkg/schema/module.go @@ -55,7 +55,7 @@ type ModuleView interface { // Module represents a "table" within a schema which contains zero or more rows // for a given set of registers. -type Module[F any] interface { +type Module[F field.Element[F]] interface { ModuleView // Assignments returns an iterator over the assignments of this module. // These are the computations used to assign values to all computed columns diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index 418af5911..7c2844b5e 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -18,16 +18,17 @@ import ( "github.com/LFDT-Lineth/zkc/pkg/trace" "github.com/LFDT-Lineth/zkc/pkg/util/collection/iter" "github.com/LFDT-Lineth/zkc/pkg/util/collection/set" + "github.com/LFDT-Lineth/zkc/pkg/util/field" ) // Any converts a concrete schema into a generic view of the schema. -func Any[F any, C Constraint[F]](schema Schema[F, C]) AnySchema[F] { +func Any[F field.Element[F], C Constraint[F]](schema Schema[F, C]) AnySchema[F] { return schema.(Schema[F, Constraint[F]]) } // AnySchema captures a generic view of a schema, which is useful in situations // where exactly details about the schema are not important. -type AnySchema[F any] Schema[F, Constraint[F]] +type AnySchema[F field.Element[F]] Schema[F, Constraint[F]] // ============================================================================ @@ -38,7 +39,7 @@ type AnySchema[F any] Schema[F, Constraint[F]] // in the final trace, whilst constraints are properties which should hold for // any acceptable trace. Finally, assignments represent arbitrary computations // which "assign" values to registers during "trace expansion". -type Schema[F any, C any] interface { +type Schema[F field.Element[F], C any] interface { // Assignments returns an iterator over the assignments of this schema. // That is, the set of computations used to determine values for all // computed columns. @@ -69,7 +70,7 @@ type Schema[F any, C any] interface { // Failure embodies structured information about a failing constraint. // This includes the constraint itself, along with the row -type Failure[F any] interface { +type Failure[F field.Element[F]] interface { // Handle returns the handling of the constraints which caused the failure. Handle() string // Provides a suitable error message diff --git a/pkg/schema/schemas.go b/pkg/schema/schemas.go index b664ddd30..6c7bfdab8 100644 --- a/pkg/schema/schemas.go +++ b/pkg/schema/schemas.go @@ -107,7 +107,7 @@ func processConstraint[F field.Element[F], C Constraint[F]](ith C, trace trace.T // PanicFailure indicates that a panic arose during constraint checking, rather // than an actual constraint failure. The purpose of this is to allow the // testing framework to distinguish panics from actual constraint failures. -type PanicFailure[F any] struct { +type PanicFailure[F field.Element[F]] struct { handle string message string stackTrace []byte diff --git a/pkg/schema/uniform_schema.go b/pkg/schema/uniform_schema.go index 0f67ca9be..dffaae217 100644 --- a/pkg/schema/uniform_schema.go +++ b/pkg/schema/uniform_schema.go @@ -18,16 +18,17 @@ import ( "github.com/LFDT-Lineth/zkc/pkg/schema/module" "github.com/LFDT-Lineth/zkc/pkg/schema/register" "github.com/LFDT-Lineth/zkc/pkg/util/collection/iter" + "github.com/LFDT-Lineth/zkc/pkg/util/field" ) // UniformSchema represents the simplest kind of schema which contains only // modules of the same kind (e.g. all MIR modules). -type UniformSchema[F any, M Module[F]] struct { +type UniformSchema[F field.Element[F], M Module[F]] struct { modules []M } // NewUniformSchema constructs a new schema comprising the given modules. -func NewUniformSchema[F any, M Module[F]](modules []M) UniformSchema[F, M] { +func NewUniformSchema[F field.Element[F], M Module[F]](modules []M) UniformSchema[F, M] { return UniformSchema[F, M]{modules} } @@ -98,7 +99,7 @@ func (p UniformSchema[F, M]) Width() uint { // Extract an iterator over all the constraints in a given array using a // projecting iterator. -func assignmentsOf[F any, M Module[F]](modules []M) iter.Iterator[Assignment[F]] { +func assignmentsOf[F field.Element[F], M Module[F]](modules []M) iter.Iterator[Assignment[F]] { arrIter := iter.NewArrayIterator(modules) // return iter.NewFlattenIterator(arrIter, func(m M) iter.Iterator[Assignment[F]] { @@ -108,7 +109,7 @@ func assignmentsOf[F any, M Module[F]](modules []M) iter.Iterator[Assignment[F]] // Extract an iterator over all the constraints in a given array using a // projecting iterator. -func constraintsOf[F any, M Module[F]](modules []M) iter.Iterator[Constraint[F]] { +func constraintsOf[F field.Element[F], M Module[F]](modules []M) iter.Iterator[Constraint[F]] { arrIter := iter.NewArrayIterator(modules) // return iter.NewFlattenIterator(arrIter, func(m M) iter.Iterator[Constraint[F]] { diff --git a/pkg/trace/array.go b/pkg/trace/array.go deleted file mode 100644 index aa9b5c6cc..000000000 --- a/pkg/trace/array.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright Consensys Software Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -package trace - -import ( - "math" - "strings" - - "github.com/LFDT-Lineth/zkc/pkg/util/collection/iter" -) - -// Array provides an implementation of Trace which stores rows as an array. -type Array[T any, M ModuleBuilder[T, M]] struct { - // Holds the set of modules in this trace. The index of each module in this - // array uniquely identifies it, and is referred to as the "module index". - modules []M -} - -// NewArray constructs a row-major trace from a given set of modules. -func NewArray[T any, M ModuleBuilder[T, M]](modules []M) *Array[T, M] { - return &Array[T, M]{modules} -} - -// HasModule determines whether this trace has a module with the given name and, -// if so, what its module index is. -func (p *Array[T, M]) HasModule(name string) (uint, bool) { - for mid, mod := range p.modules { - if mod.Name() == name { - return uint(mid), true - } - } - // - return math.MaxUint, false -} - -// Module returns a specific module in this trace. -func (p *Array[T, M]) Module(module uint) Module[T] { - return p.modules[module] -} - -// RawModule returns a specific (raw) module in this trace. -func (p *Array[T, M]) RawModule(module uint) M { - return p.modules[module] -} - -// SetRawModule replaces a specific (raw) module in this trace. This is -// necessary, for example, after an operation (such as Pad) which returns a -// new module rather than updating the original in place. -func (p *Array[T, M]) SetRawModule(module uint, m M) { - p.modules[module] = m -} - -// Modules returns an iterator over the modules in this trace. -func (p *Array[T, M]) Modules() iter.Iterator[Module[T]] { - it := iter.NewArrayIterator(p.modules) - // - return iter.NewCastIterator[M, Module[T]](it) -} - -// Width returns the number of modules in this trace. -func (p *Array[T, M]) Width() uint { - return uint(len(p.modules)) -} - -func (p *Array[T, M]) String() string { - var id strings.Builder - - id.WriteString("{") - // - for i, m := range p.modules { - if i != 0 { - id.WriteString(", ") - } - // - id.WriteString(m.String()) - } - // - id.WriteString("}") - // - return id.String() -} diff --git a/pkg/trace/builder.go b/pkg/trace/builder.go new file mode 100644 index 000000000..beffa72cb --- /dev/null +++ b/pkg/trace/builder.go @@ -0,0 +1,119 @@ +// Copyright Consensys Software Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +package trace + +import ( + "math" + "strings" + + "github.com/LFDT-Lineth/zkc/pkg/util/collection/array" + "github.com/LFDT-Lineth/zkc/pkg/util/field" +) + +// ModuleBuilder describes an individual module within a trace, and represents +// each column within that module using an appropriate (compact) encoding. +type ModuleBuilder[F field.Element[F]] struct { + // Recorded height of module + height uint + // Holds the descriptor for this module. + descriptor ModuleDescriptor + // Holds the complete set of columns in this module, with one for each + // descriptor. + columns []array.MutArray[F] +} + +// InitModuleBuilder constructs a new empty module with appropriately allocated +// (but zero-height) columns for each descriptor. +func InitModuleBuilder[F field.Element[F]](descriptor ModuleDescriptor) *ModuleBuilder[F] { + var ( + width = descriptor.Width() + columns = make([]array.MutArray[F], width) + ) + // + for rid := range descriptor.Columns { + var bitwidth = descriptor.Columns[rid].Bitwidth.UnwrapOr(math.MaxUint) + // Allocate compact representation + columns[rid] = array.Alloc[F](bitwidth) + } + // + return &ModuleBuilder[F]{0, descriptor, columns} +} + +// Build constructs a module from this builder. +func (p *ModuleBuilder[F]) Build() Module[F] { + var columns = make([]array.Array[F], len(p.columns)) + // + for i, col := range p.columns { + columns[i] = col.Build() + } + // + return NewModule(p.descriptor, columns...) +} + +// Append implementation for Module interface. +func (p *ModuleBuilder[F]) Append(row ...F) { + if len(row) != len(p.descriptor.Columns) { + panic("mismatched row data") + } + // Append element for each row + for i, v := range row { + p.columns[i] = p.columns[i].Append(v) + } + // Increment height + p.height++ +} + +// Name returns the name of this module. +func (p *ModuleBuilder[F]) Name() string { + return p.descriptor.Name +} + +// Descriptor returns the descriptor of this module. +func (p *ModuleBuilder[F]) Descriptor() ModuleDescriptor { + return p.descriptor +} + +// Height returns the height of this module, meaning the number of assigned +// rows. +func (p *ModuleBuilder[F]) Height() uint { + return p.height +} + +// Width returns the number of columns in this module. +func (p *ModuleBuilder[F]) Width() uint { + return uint(len(p.descriptor.Columns)) +} + +func (p *ModuleBuilder[F]) String() string { + var id strings.Builder + // + if p.descriptor.Name == "" { + id.WriteString("∅") + } else { + id.WriteString(p.descriptor.Name) + } + + id.WriteString("={") + // + for i, c := range p.columns { + if i != 0 { + id.WriteString(", ") + } + // + id.WriteString(c.Build().String()) + } + // + id.WriteString("}") + // Done + return id.String() +} diff --git a/pkg/trace/compact.go b/pkg/trace/compact.go deleted file mode 100644 index 16da1026d..000000000 --- a/pkg/trace/compact.go +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright Consensys Software Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -package trace - -import ( - "fmt" - "math" - "strings" - - "github.com/LFDT-Lineth/zkc/pkg/util/collection/array" - "github.com/LFDT-Lineth/zkc/pkg/util/field" -) - -// CompactModule describes an individual module within a trace, and represents -// each column within that module using an appropriate (compact) encoding. -type CompactModule[F field.Element[F]] struct { - // Recorded height of module - height uint - // Holds the descriptor for this module. - descriptor ModuleDescriptor - // Holds the complete set of columns in this module, with one for each - // descriptor. - columns []array.MutArray[F] -} - -// NewCompactModule constructs a module with the given name, descriptors and rows. -func NewCompactModule[F field.Element[F]](descriptor ModuleDescriptor, data ...array.MutArray[F]) *CompactModule[F] { - var height uint - // Sanity check - if uint(len(data)) != descriptor.Width() { - panic(fmt.Sprintf("incorrect number of data columns for module '%s' (%d vs %d)", - descriptor.Name, len(data), descriptor.Width())) - } - // Determine maximum height - for _, col := range data { - if col != nil { - height = max(height, col.Len()) - } - } - // Check matching heights - for i, col := range data { - if col != nil && col.Len() != height { - panic(fmt.Sprintf("column %s has mismatched height (%d vs %d)", - descriptor.Columns[i].Name, col.Len(), height)) - } - } - // - return &CompactModule[F]{height, descriptor, data} -} - -// InitCompactModule constructs a new empty module with appropriately allocated -// (but zero-height) columns for each descriptor. -func InitCompactModule[F field.Element[F]](descriptor ModuleDescriptor) *CompactModule[F] { - var ( - width = descriptor.Width() - columns = make([]array.MutArray[F], width) - ) - // - for rid := range descriptor.Columns { - var bitwidth = descriptor.Columns[rid].Bitwidth.UnwrapOr(math.MaxUint) - // Allocate compact representation - columns[rid] = array.Alloc[F](bitwidth, 0) - } - // - return &CompactModule[F]{0, descriptor, columns} -} - -// Initialise implementation for ModuleBuilder interface. This constructs a new -// empty module; the receiver is not used. -func (p *CompactModule[F]) Initialise(descriptor ModuleDescriptor) *CompactModule[F] { - return InitCompactModule[F](descriptor) -} - -// Append implementation for Module interface. -func (p *CompactModule[F]) Append(row ...F) { - if len(row) != len(p.descriptor.Columns) { - panic("mismatched row data") - } - // Append element for each row - for i, v := range row { - p.columns[i].Append(v) - } - // Increment height - p.height++ -} - -// Expand a given column in this module -func (p *CompactModule[F]) Expand(col uint, data array.MutArray[F]) { - if p.columns[col] != nil { - panic("cannot expand non-empty column") - } else if p.untouched() { - // This module has no columns assigned yet (e.g. it is an entirely - // computed module, such as a lookup table, which has no natural - // presence in the original trace). In this case, its recorded height - // is just a placeholder rather than an established fact, so the first - // column expanded determines the real height. - p.height = data.Len() - } else if data.Len() != p.height { - panic(fmt.Sprintf("invalid column height (%d vs %d)", data.Len(), p.height)) - } - // - p.columns[col] = data -} - -// untouched determines whether or not any column in this module has been -// assigned data yet. -func (p *CompactModule[F]) untouched() bool { - for _, col := range p.columns { - if col != nil { - return false - } - } - // - return true -} - -// Join a given module into this by appending all rows of each column onto the -// corresponding column in this module. -func (p *CompactModule[F]) Join(m Module[F]) { - if p.Width() != m.Width() { - panic(fmt.Sprintf("cannot join mismatched modules '%s' (%d columns) vs '%s' (%d columns)", - p.descriptor.Name, p.Width(), m.Descriptor().Name, m.Width())) - } - // - for i := range p.Width() { - array.AppendOnto(p.columns[i], m.Column(i)) - } - // Increment height - p.height += m.Height() -} - -// Clone this module, such that mutating the clone (or the original) -// afterwards has no effect on the other. -func (p *CompactModule[F]) Clone() *CompactModule[F] { - columns := make([]array.MutArray[F], len(p.columns)) - // - for i, col := range p.columns { - if col != nil { - columns[i] = col.Clone() - } - } - // - return &CompactModule[F]{p.height, p.descriptor, columns} -} - -// Name returns the name of this module. -func (p *CompactModule[F]) Name() string { - return p.descriptor.Name -} - -// Descriptor returns the descriptor of this module. -func (p *CompactModule[F]) Descriptor() ModuleDescriptor { - return p.descriptor -} - -// Column returns the data for the column at the given index. -func (p *CompactModule[F]) Column(index uint) array.Array[F] { - return p.columns[index] -} - -// MutColumn returns mutable access to the data for the column at the given -// index. -func (p *CompactModule[F]) MutColumn(index uint) array.MutArray[F] { - return p.columns[index] -} - -// Height returns the height of this module, meaning the number of assigned -// rows. -func (p *CompactModule[F]) Height() uint { - return p.height -} - -// Pad returns a copy of this module with the given amount of front/back -// padding added. The receiver itself is left unmodified. -func (p *CompactModule[F]) Pad(front, back uint) *CompactModule[F] { - var ( - zero F - columns = make([]array.MutArray[F], len(p.columns)) - ) - // - for i, col := range p.columns { - if col != nil { - columns[i] = col.Pad(front, back, zero) - } - } - // - return &CompactModule[F]{p.height + front + back, p.descriptor, columns} -} - -// Width returns the number of columns in this module. -func (p *CompactModule[F]) Width() uint { - return uint(len(p.descriptor.Columns)) -} - -func (p *CompactModule[F]) String() string { - var id strings.Builder - // - if p.descriptor.Name == "" { - id.WriteString("∅") - } else { - id.WriteString(p.descriptor.Name) - } - - id.WriteString("={") - // - for i, c := range p.columns { - if i != 0 { - id.WriteString(", ") - } - // - id.WriteString(c.String()) - } - // - id.WriteString("}") - // Done - return id.String() -} diff --git a/pkg/trace/encoding.go b/pkg/trace/encoding.go index 14ff11144..dca00f136 100644 --- a/pkg/trace/encoding.go +++ b/pkg/trace/encoding.go @@ -16,44 +16,79 @@ import ( "bytes" "encoding/binary" "fmt" + "math" "github.com/LFDT-Lineth/zkc/pkg/util" + "github.com/LFDT-Lineth/zkc/pkg/util/collection/array" + "github.com/LFDT-Lineth/zkc/pkg/util/field" ) // MarshalBinary encodes a trace into a binary format. Lengths, counts and // descriptor metadata are encoded as unsigned varints. Register data is // encoded column-major, with each column written using the natural encoding of // its underlying array representation (see array.Array.Encode). -func MarshalBinary[T any, M ModuleBuilder[T, M]](tr Array[T, M]) ([]byte, error) { - var buffer bytes.Buffer +func MarshalBinary[F field.Element[F]](tr Trace[F]) ([]byte, error) { + var ( + buffer bytes.Buffer + nShards = uint(len(tr)) + width = tr[0].Width() + ) // buffer.Write(traceBinaryMagic) - writeUvarint(&buffer, tr.Width()) + // Write number of shards recorded + writeUvarint(&buffer, nShards) + // Write number of modules in each shard + writeUvarint(&buffer, width) // - for mid := uint(0); mid < tr.Width(); mid++ { - marshalModule(&buffer, tr.Module(mid)) + for _, shard := range tr { + if shard.Width() != width { + panic("malformed trace") + } + // + for mid := uint(0); mid < shard.Width(); mid++ { + marshalModule(&buffer, shard.Module(mid)) + } } // return buffer.Bytes(), nil } // UnmarshalBinary decodes a trace encoded by MarshalBinary. -func (p *Array[T, M]) UnmarshalBinary(data []byte) error { - var buffer = bytes.NewBuffer(data) +func (p *Trace[F]) UnmarshalBinary(data []byte) (err error) { + var ( + buffer = bytes.NewBuffer(data) + nShards uint + nModules uint + ) + // Sanity check magic + magic := buffer.Next(len(traceBinaryMagic)) // - modules, err := unmarshalModules[T, M](buffer) - if err != nil { + if len(magic) != len(traceBinaryMagic) || !bytes.Equal(magic, traceBinaryMagic) { + return fmt.Errorf("malformed binary: invalid header") + } else if nShards, err = readUvarint(buffer); err != nil { + return err + } else if nModules, err = readUvarint(buffer); err != nil { return err - } else if buffer.Len() != 0 { - return fmt.Errorf("malformed rtrace binary: %d trailing bytes", buffer.Len()) } // - p.modules = modules + *p = make([]Shard[F], nShards) + // + for shard := range nShards { + var ith = make([]Module[F], nModules) + // + for mid := range nModules { + if ith[mid], err = unmarshalModule[F](buffer); err != nil { + return err + } + } + // Construct ith shard + (*p)[shard] = NewShard(ith) + } // return nil } -func marshalModule[T any](buffer *bytes.Buffer, module Module[T]) { +func marshalModule[F field.Element[F]](buffer *bytes.Buffer, module Module[F]) { var metadata uint // Build metadata if module.Descriptor().Replicated { @@ -66,24 +101,11 @@ func marshalModule[T any](buffer *bytes.Buffer, module Module[T]) { writeUvarint(buffer, module.Height()) // for cid := uint(0); cid < module.Width(); cid++ { - module.Column(cid).Encode(buffer) + array.Encode(module.Column(cid), buffer) } } -func unmarshalModules[T any, M ModuleBuilder[T, M]](buffer *bytes.Buffer) ([]M, error) { - if buffer.Len() < len(traceBinaryMagic) { - return nil, fmt.Errorf("malformed rtrace binary: missing header") - } - // - magic := buffer.Next(len(traceBinaryMagic)) - if !bytes.Equal(magic, traceBinaryMagic) { - return nil, fmt.Errorf("malformed rtrace binary: invalid header") - } - // - return readSlice(buffer, unmarshalModule[T, M]) -} - -func unmarshalModule[T any, M ModuleBuilder[T, M]](buffer *bytes.Buffer) (M, error) { +func unmarshalModule[F field.Element[F]](buffer *bytes.Buffer) (module Module[F], err error) { var ( r = reader{buf: buffer} // @@ -92,22 +114,23 @@ func unmarshalModule[T any, M ModuleBuilder[T, M]](buffer *bytes.Buffer) (M, err metadata = r.uvarint() height = r.uvarint() replicated = metadata != 0 - // - module M + columns = make([]array.Array[F], len(descriptors)) ) // if r.err != nil { return module, r.err } - // Initialise (empty) module, thereby allocating an appropriate array - // representation for each descriptor. - module = module.Initialise(ModuleDescriptor{name, descriptors, replicated}) // Decode each column in place. - for cid := range uint(len(descriptors)) { - if err := module.MutColumn(cid).Decode(height, buffer); err != nil { + for cid, descriptor := range descriptors { + bitwidth := descriptor.Bitwidth.UnwrapOr(math.MaxUint) + // + if columns[cid], err = array.Decode[F](bitwidth, height, buffer); err != nil { return module, err } } + // Initialise (empty) module, thereby allocating an appropriate array + // representation for each descriptor. + module = NewModule(ModuleDescriptor{name, descriptors, replicated}, columns...) // return module, nil } diff --git a/pkg/trace/json/reader.go b/pkg/trace/json/reader.go index 2009efd40..8ffacd91e 100644 --- a/pkg/trace/json/reader.go +++ b/pkg/trace/json/reader.go @@ -78,13 +78,13 @@ func fromBytesLegacy[F field.Element[F]](rawData map[string][]big.Int) (trace.Sh mod, col, error := splitQualifiedColumnName(name) // error check if error != nil { - return nil, error + return trace.Shard[F]{}, error } // Sanity check existing module data if strData[mod] == nil { strData[mod] = make(map[string][]big.Int) } else if _, ok := strData[mod][col]; ok { - return nil, fmt.Errorf("duplicate column %s encountered", trace.QualifiedColumnName(mod, col)) + return trace.Shard[F]{}, fmt.Errorf("duplicate column %s encountered", trace.QualifiedColumnName(mod, col)) } // Assign values strData[mod][col] = rawInts @@ -94,11 +94,11 @@ func fromBytesLegacy[F field.Element[F]](rawData map[string][]big.Int) (trace.Sh } func fromBytesInternal[F field.Element[F]](rawData map[string]map[string][]big.Int) (trace.Shard[F], error) { - var modules []*trace.CompactModule[F] + var modules []trace.Module[F] // for mod, modData := range rawData { var ( - columns []array.MutArray[F] + columns []array.Array[F] descriptors []trace.ColumnDescriptor ) // @@ -106,11 +106,11 @@ func fromBytesInternal[F field.Element[F]](rawData map[string]map[string][]big.I col, bitwidth, error := splitColumnBitwidth(name) // error check if error != nil { - return nil, error + return trace.Shard[F]{}, error } // Validate data array if row := validateBigInts(bitwidth, rawInts); row != math.MaxUint { - return nil, fmt.Errorf("column %s out-of-bounds (row %d, value %s)", + return trace.Shard[F]{}, fmt.Errorf("column %s out-of-bounds (row %d, value %s)", name, row, rawInts[row].String()) } // Construct column @@ -120,26 +120,26 @@ func fromBytesInternal[F field.Element[F]](rawData map[string]map[string][]big.I // construct module descriptor descriptor := trace.NewModuleDescriptor(mod, descriptors) // append new module - modules = append(modules, trace.NewCompactModule[F](descriptor, columns...)) + modules = append(modules, trace.NewModule[F](descriptor, columns...)) } // - return trace.NewArray(modules), nil + return trace.NewShard(modules), nil } -func newArrayFromBigInts[F field.Element[F]](bitwidth util.Option[uint], data []big.Int) array.MutArray[F] { +func newArrayFromBigInts[F field.Element[F]](bitwidth util.Option[uint], data []big.Int) array.Array[F] { // var ( n = uint(len(data)) - arr = array.Alloc[F](bitwidth.UnwrapOr(math.MaxUint), n) + arr = array.Alloc[F](bitwidth.UnwrapOr(math.MaxUint)) ) // for i := range n { var val F // - arr.Set(i, val.SetBytes(data[i].Bytes())) + arr = arr.Append(val.SetBytes(data[i].Bytes())) } // - return arr + return arr.Build() } // SplitQualifiedColumnName splits a qualified column name into its module and diff --git a/pkg/trace/module.go b/pkg/trace/module.go new file mode 100644 index 000000000..0287395b7 --- /dev/null +++ b/pkg/trace/module.go @@ -0,0 +1,163 @@ +// Copyright Consensys Software Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +package trace + +import ( + "fmt" + "slices" + "strings" + + "github.com/LFDT-Lineth/zkc/pkg/util/collection/array" + "github.com/LFDT-Lineth/zkc/pkg/util/field" +) + +// Module describes a module within the trace. Every module is a collection of +// zero or more data columns with the same height. The width of a module is the +// number of such columns it contains. Every column in the module has a +// "descriptor" which provides metadata about the columns, such as its name and +// declared bitwidth, etc. +type Module[F field.Element[F]] struct { + // Recorded height of module + height uint + // Holds the descriptor for this module. + descriptor ModuleDescriptor + // Holds the complete set of columns in this module, with one for each + // descriptor. + columns []array.Array[F] +} + +// NewModule constructs a module with the given name, descriptors and rows. +func NewModule[F field.Element[F]](descriptor ModuleDescriptor, data ...array.Array[F]) Module[F] { + var height uint + // Sanity check + if uint(len(data)) != descriptor.Width() { + panic(fmt.Sprintf("incorrect number of data columns for module '%s' (%d vs %d)", + descriptor.Name, len(data), descriptor.Width())) + } + // Determine maximum height + for _, col := range data { + if col != nil { + height = max(height, col.Len()) + } + } + // Check matching heights + for i, col := range data { + if col != nil && col.Len() != height { + panic(fmt.Sprintf("column %s has mismatched height (%d vs %d)", + descriptor.Columns[i].Name, col.Len(), height)) + } + } + // + return Module[F]{height, descriptor, data} +} + +// Expand a given column in this module +func (p Module[F]) Expand(col uint, data array.Array[F]) Module[F] { + var ( + height = p.height + columns = slices.Clone(p.columns) + ) + // Sanity check + if columns[col] != nil { + panic("cannot expand non-empty column") + } else if p.untouched() { + // This module has no columns assigned yet (e.g. it is an entirely + // computed module, such as a lookup table, which has no natural + // presence in the original trace). In this case, its recorded height + // is just a placeholder rather than an established fact, so the first + // column expanded determines the real height. + height = data.Len() + } else if data.Len() != p.height { + panic(fmt.Sprintf("invalid column height (%d vs %d)", data.Len(), p.height)) + } + // + columns[col] = data + // Done + return Module[F]{height, p.descriptor, columns} +} + +// untouched determines whether or not any column in this module has been +// assigned data yet. +func (p Module[F]) untouched() bool { + for _, col := range p.columns { + if col != nil { + return false + } + } + // + return true +} + +// Name returns the name of this module. +func (p Module[F]) Name() string { + return p.descriptor.Name +} + +// Descriptor returns the descriptor of this module. +func (p Module[F]) Descriptor() ModuleDescriptor { + return p.descriptor +} + +// Column returns the data for the column at the given index. +func (p Module[F]) Column(index uint) array.Array[F] { + return p.columns[index] +} + +// Height returns the height of this module, meaning the number of assigned +// rows. +func (p Module[F]) Height() uint { + return p.height +} + +// Pad returns a copy of this module with the given amount of front/back +// padding added. The receiver itself is left unmodified. +func (p Module[F]) Pad(front uint) Module[F] { + var columns = make([]array.Array[F], len(p.columns)) + // + for i, col := range p.columns { + if col != nil { + columns[i] = col.Pad(front) + } + } + // + return Module[F]{p.height + front, p.descriptor, columns} +} + +// Width returns the number of columns in this module. +func (p Module[F]) Width() uint { + return uint(len(p.descriptor.Columns)) +} + +func (p Module[F]) String() string { + var id strings.Builder + // + if p.descriptor.Name == "" { + id.WriteString("∅") + } else { + id.WriteString(p.descriptor.Name) + } + + id.WriteString("={") + // + for i, c := range p.columns { + if i != 0 { + id.WriteString(", ") + } + // + id.WriteString(c.String()) + } + // + id.WriteString("}") + // Done + return id.String() +} diff --git a/pkg/trace/reduce.go b/pkg/trace/reduce.go deleted file mode 100644 index c27116178..000000000 --- a/pkg/trace/reduce.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Consensys Software Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -package trace - -import ( - "github.com/LFDT-Lineth/zkc/pkg/util/collection/array" - "github.com/LFDT-Lineth/zkc/pkg/util/field" -) - -// Reduce combines a sequence of aligned traces into a single trace. Two traces -// are aligned when they have the same width and modules sharing a name also -// share the same module index and matching register (and hence limb) -// descriptors. For each module, the rows of every input trace are -// concatenated, in trace order, into the corresponding module of the result. -func Reduce[F field.Element[F]](traces []Shard[F]) Shard[F] { - if len(traces) == 0 { - return nil - } - // Aligned traces all share the same width, so the first trace determines - // the module structure of the result. - width := traces[0].Width() - modules := make([]*CompactModule[F], width) - // - for mid := range width { - var descriptor = traces[0].Module(mid).Descriptor() - // - modules[mid] = reduceModule(mid, descriptor, traces) - } - // - return NewArray(modules) -} - -// ParallelReduce behaves exactly like Reduce, combining a sequence of aligned -// traces into a single trace, but reduces each module of the result -// concurrently using a worker pool. -func ParallelReduce[F field.Element[F]](traces []Shard[F]) Shard[F] { - if len(traces) == 0 { - return nil - } - // - width := traces[0].Width() - descriptors := make([]ModuleDescriptor, width) - // - for mid := range width { - descriptors[mid] = traces[0].Module(mid).Descriptor() - } - // - modules := array.ParallelMap(descriptors, func(mid uint, descriptor ModuleDescriptor) *CompactModule[F] { - return reduceModule(mid, descriptor, traces) - }) - // - return NewArray(modules) -} - -// reduceModule concatenates the rows of the module at a given index across all -// traces. The module name and descriptor are taken from the first trace, which -// aligned traces guarantee match those of every other trace. -func reduceModule[F field.Element[F]](mid uint, descriptor ModuleDescriptor, traces []Shard[F]) *CompactModule[F] { - // Check whether replicating - if descriptor.Replicated { - return reduceReplicatedModule(mid, descriptor, traces) - } - // - acc := InitCompactModule[F](descriptor) - // - for i := range traces { - acc.Join(traces[i].Module(mid)) - } - // - return acc -} - -func reduceReplicatedModule[F field.Element[F]](mid uint, descriptor ModuleDescriptor, traces []Shard[F], -) *CompactModule[F] { - var ( - winner Module[F] - height uint - acc = InitCompactModule[F](descriptor) - ) - // Find tallest module - for i := range traces { - ith := traces[i].Module(mid) - // TODO: find a better joining strategy - if height < ith.Height() { - winner = ith - height = ith.Height() - } - } - // Construct new module - acc.Join(winner) - // Done - return acc -} diff --git a/pkg/trace/trace.go b/pkg/trace/trace.go index 705d8b48a..0ed20b5aa 100644 --- a/pkg/trace/trace.go +++ b/pkg/trace/trace.go @@ -13,61 +13,85 @@ package trace import ( - "fmt" + "math" + "strings" "github.com/LFDT-Lineth/zkc/pkg/util" - "github.com/LFDT-Lineth/zkc/pkg/util/collection/array" "github.com/LFDT-Lineth/zkc/pkg/util/collection/iter" + "github.com/LFDT-Lineth/zkc/pkg/util/field" ) // Trace represents a complete (sharded) trace. That is, an array of shards. -type Trace[T any] []Shard[T] - -// Shard forms part of a trace, and describes a set of named modules whose data -// is organised by row. -type Shard[T any] interface { - // Determine whether this trace has a module with the given name and, if so, - // what its module index is. - HasModule(name string) (uint, bool) - // Access a given module in this trace. - Module(uint) Module[T] - // Returns an iterator over the contained modules. - Modules() iter.Iterator[Module[T]] - // Returns the number of modules in this trace. - Width() uint +type Trace[F field.Element[F]] []Shard[F] + +// Shard describes an immutable set of named modules whose data is organised by +// columns. +type Shard[F field.Element[F]] struct { + // Holds the set of modules in this trace. The index of each module in this + // array uniquely identifies it, and is referred to as the "module index". + modules []Module[F] +} + +// NewShard constructs a new shard from a given set of module traces. +func NewShard[F field.Element[F]](modules []Module[F]) Shard[F] { + return Shard[F]{modules} +} + +// IsEmpty determines whether or not this shard is completely empty. +func (p Shard[F]) IsEmpty() bool { + return p.modules == nil +} + +// HasModule determines whether this trace has a module with the given name and, +// if so, what its module index is. +func (p Shard[F]) HasModule(name string) (uint, bool) { + for mid, mod := range p.modules { + if mod.Name() == name { + return uint(mid), true + } + } + // + return math.MaxUint, false } -// Module describes a module within the trace. Every module is a collection of -// zero or more data columns with the same height. The width of a module is the -// number of such columns it contains. Every column in the module has a -// "descriptor" which provides metadata about the columns, such as its name and -// declared bitwidth, etc. -type Module[T any] interface { - fmt.Stringer - // Append a given row onto this module. This will panic if the length of - // this row does not match the width of this module. - Append(...T) - // Module name. - Name() string - // Column returns the data for the column at the given index. - Column(uint) array.Array[T] - // MutColumn returns a mutable reference to the underlying data of the given - // column. - MutColumn(uint) array.MutArray[T] - // Descriptor returns the descriptor of this module. - Descriptor() ModuleDescriptor - // Returns the number of columns in this module. - Width() uint - // Returns the height (i.e. number of rows) of this module. - Height() uint +// Module returns a specific module in this trace. +func (p Shard[F]) Module(module uint) Module[F] { + return p.modules[module] } -// ModuleBuilder describes an extended module which can be used for the purposes -// of constructing new modules. -type ModuleBuilder[T any, M any] interface { - Module[T] - // Initialise a new module from a given set of rows. - Initialise(ModuleDescriptor) M +// RawModule returns a specific (raw) module in this trace. +func (p Shard[F]) RawModule(module uint) Module[F] { + return p.modules[module] +} + +// Modules returns an iterator over the modules in this trace. +func (p Shard[F]) Modules() iter.Iterator[Module[F]] { + it := iter.NewArrayIterator(p.modules) + // + return iter.NewCastIterator[Module[F], Module[F]](it) +} + +// Width returns the number of modules in this trace. +func (p Shard[F]) Width() uint { + return uint(len(p.modules)) +} + +func (p Shard[F]) String() string { + var id strings.Builder + + id.WriteString("{") + // + for i, m := range p.modules { + if i != 0 { + id.WriteString(", ") + } + // + id.WriteString(m.String()) + } + // + id.WriteString("}") + // + return id.String() } // ModuleDescriptor describes an individual module within a trace, including all diff --git a/pkg/util/collection/array/alloc.go b/pkg/util/collection/array/alloc.go index 2f61db1c4..53deeb9a6 100644 --- a/pkg/util/collection/array/alloc.go +++ b/pkg/util/collection/array/alloc.go @@ -13,59 +13,42 @@ package array import ( + "math" + "math/bits" + "github.com/LFDT-Lineth/zkc/pkg/util/word" ) -// Alloc allocates a new array suitable for holding elements upto the given -// bitwidth, and initialises it with default values upto the given height. -func Alloc[F word.Word[F]](bitwidth uint, height uint) MutArray[F] { +// Alloc allocates a new array holding the given elements and which can hold any +// element upto the given bitwidth. The given array maybe consumed by this +// array. +func Alloc[F word.Word[F]](bitwidth uint) MutArray[F] { var zero F // Construct column switch { case bitwidth == 0: - return NewConstantArray(height, 0, zero) + return NewConstantArray(0, 0, zero) case bitwidth == 1: - return NewBitArray[F](height) + return NewBitArray[F](0, false) case bitwidth <= 8: - return NewSmallArray[uint8, F](height, bitwidth) + return NewSmallArray[uint8, F](bitwidth, 0, 0) case bitwidth <= 16: - return NewSmallArray[uint16, F](height, bitwidth) + return NewSmallArray[uint16, F](bitwidth, 0, 0) case bitwidth <= 32: - return NewSmallArray[uint32, F](height, bitwidth) + return NewSmallArray[uint32, F](bitwidth, 0, 0) case bitwidth <= 64: - return NewSmallArray[uint64, F](height, bitwidth) + return NewSmallArray[uint64, F](bitwidth, 0, 0) default: - return NewStaticArray[F](height, bitwidth) + return NewStaticArray[F](bitwidth) } } -// AppendOnto attempts to efficiently append the contents of the right array -// onto the left array. This currently assumes that the type and bitwidth of -// the two arrays matches exactly, and will panic otherwise. -func AppendOnto[F word.Word[F]](left MutArray[F], right Array[F]) { - switch left := left.(type) { - case *ConstantArray[F]: - var right = right.(*ConstantArray[F]) - left.AppendAll(*right) - case *BitArray[F]: - var right = right.(*BitArray[F]) - left.AppendAll(*right) - case *SmallArray[uint8, F]: - var right = right.(*SmallArray[uint8, F]) - left.AppendAll(*right) - case *SmallArray[uint16, F]: - var right = right.(*SmallArray[uint16, F]) - left.AppendAll(*right) - case *SmallArray[uint32, F]: - var right = right.(*SmallArray[uint32, F]) - left.AppendAll(*right) - case *SmallArray[uint64, F]: - var right = right.(*SmallArray[uint64, F]) - left.AppendAll(*right) - case *StaticArray[F]: - var right = right.(*StaticArray[F]) - left.AppendAll(*right) - default: - panic("unknown array") +// bitwidth of returns the (approximate) bitwidth of a given value appropriate +// for determine a suitable column width to use. +func bitwidthOf[F word.Word[F]](val F) uint { + if val.FitsWithin(64) { + return uint(bits.Len64(val.Uint64())) } + // + return math.MaxUint } diff --git a/pkg/util/collection/array/bit_array.go b/pkg/util/collection/array/bit_array.go index 722643b0f..eacd89ba6 100644 --- a/pkg/util/collection/array/bit_array.go +++ b/pkg/util/collection/array/bit_array.go @@ -13,7 +13,6 @@ package array import ( - "bytes" "fmt" "slices" "strings" @@ -25,6 +24,23 @@ import ( // bitOne is the byte-level binary representation of 1. var bitOne = []byte{1} +// NewBitArray constructs a new word array with a given capacity. +func NewBitArray[T word.Word[T]](height uint, value bool) *BitArray[T] { + var ( + bytewidth = word.ByteWidth(height) + elements = make([]byte, bytewidth) + ) + // + if value { + // Fill with 1s + for i := range bytewidth { + elements[i] = 0xff + } + } + // + return &BitArray[T]{elements, height} +} + // ================================================================================= // Implementation // ================================================================================= @@ -39,23 +55,62 @@ type BitArray[T word.Word[T]] struct { height uint } -// NewBitArray constructs a new word array with a given capacity. -func NewBitArray[T word.Word[T]](height uint) *BitArray[T] { - var ( - bytewidth = word.ByteWidth(height) - elements = make([]byte, bytewidth) - ) - // - return &BitArray[T]{elements, height} +// Bytes implementation for Array interface +func (p BitArray[T]) Bytes() uint { + return uint(len(p.data)) } // Len returns the number of elements in this word array. -func (p *BitArray[T]) Len() uint { +func (p BitArray[T]) Len() uint { return p.height } +// BitWidth returns the width (in bits) of elements in this array. +func (p BitArray[T]) BitWidth() uint { + return 1 +} + +// Get returns the field element at the given index in this array. +func (p BitArray[T]) Get(index uint) T { + var b T + // + if bit.LittleEndianRead(p.data, index) { + return b.SetBytes(bitOne) + } + // Default is zero + return b +} + +// Pad returns a copy of this array with n copies of the given padding value +// prepended, and m copies appended. The receiver is left unmodified. +func (p BitArray[T]) Pad(n uint) Array[T] { + return NewPaddedArray(p).Pad(n) +} + +func (p BitArray[T]) String() string { + var sb strings.Builder + + sb.WriteString("[") + + for i := range p.Len() { + if i != 0 { + sb.WriteString(",") + } + + fmt.Fprintf(&sb, "%v", p.Get(i)) + } + + sb.WriteString("]") + + return sb.String() +} + +// ================================================================================= +// MutArray Implementation +// ================================================================================= + // Append new word on this array -func (p *BitArray[T]) Append(val T) { +func (p *BitArray[T]) Append(val T) MutArray[T] { var ( // if byte length is 0, the word represents 0. otherwise, it must be 1. v = !val.IsZero() @@ -70,6 +125,8 @@ func (p *BitArray[T]) Append(val T) { bit.LittleEndianWrite(v, p.data, p.height) // increase height p.height++ + // + return p } // AppendAll elements of the given bit array onto the this array, mutating it @@ -90,53 +147,14 @@ func (p *BitArray[T]) AppendAll(other BitArray[T]) { p.height += other.height } -// BitWidth returns the width (in bits) of elements in this array. -func (p *BitArray[T]) BitWidth() uint { - return 1 -} - -// Encode implementation for Array interface. The natural encoding of a bit -// array is its packed byte representation, where eight bits are packed into -// each byte. -func (p *BitArray[T]) Encode(buffer *bytes.Buffer) { - buffer.Write(p.data) -} - -// Decode implementation for MutArray interface. This reads a packed byte -// representation (as produced by Encode) holding the given number of bits. -func (p *BitArray[T]) Decode(height uint, buffer *bytes.Buffer) error { - bytewidth := word.ByteWidth(height) - // - if uint(buffer.Len()) < bytewidth { - return fmt.Errorf("bit array requires %d bytes, but only %d remain", bytewidth, buffer.Len()) - } - // Observe bytes must be cloned, since the slice returned by Next is only - // valid until the next buffer operation. - p.data = bytes.Clone(buffer.Next(int(bytewidth))) - p.height = height - // - return nil -} - -// Clone makes clones of this array producing an otherwise identical copy. -func (p *BitArray[T]) Clone() MutArray[T] { - // Allocate sufficient memory - ndata := make([]byte, uint(len(p.data))) - // Copy over the data - copy(ndata, p.data) - // - return &BitArray[T]{ndata, p.height} +// Build implementation for MutArray interface +func (p *BitArray[T]) Build() Array[T] { + return p } -// Get returns the field element at the given index in this array. -func (p *BitArray[T]) Get(index uint) T { - var b T - // - if bit.LittleEndianRead(p.data, index) { - return b.SetBytes(bitOne) - } - // Default is zero - return b +// Height implementation of MutArray interface +func (p *BitArray[T]) Height() uint { + return p.Len() } // Set sets the field element at the given index in this array, overwriting the @@ -147,53 +165,3 @@ func (p *BitArray[T]) Set(index uint, word T) { // bit.LittleEndianWrite(val, p.data, index) } - -// Pad returns a copy of this array with n copies of the given padding value -// prepended, and m copies appended. The receiver is left unmodified. -func (p *BitArray[T]) Pad(n uint, m uint, padding T) MutArray[T] { - var ( - height = n + p.height + m - bytewidth = word.ByteWidth(height) - // Allocate exactly, copying existing bits directly into their final - // (shifted) position. - data = make([]byte, bytewidth) - ) - // - bit.LittleEndianCopy(p.data, 0, data, n, p.height) - // - result := &BitArray[T]{data, height} - // Front padding - for i := range n { - result.Set(i, padding) - } - // Back padding - for i := n + p.height; i < height; i++ { - result.Set(i, padding) - } - // - return result -} - -// SetRaw sets a raw bit at the given index in this array, overwriting the -// original value. -func (p *BitArray[T]) SetRaw(index uint, val bool) { - bit.LittleEndianWrite(val, p.data, index) -} - -func (p *BitArray[T]) String() string { - var sb strings.Builder - - sb.WriteString("[") - - for i := range p.Len() { - if i != 0 { - sb.WriteString(",") - } - - fmt.Fprintf(&sb, "%v", p.Get(i)) - } - - sb.WriteString("]") - - return sb.String() -} diff --git a/pkg/util/collection/array/const_array.go b/pkg/util/collection/array/const_array.go index 3401b8771..da41ffc67 100644 --- a/pkg/util/collection/array/const_array.go +++ b/pkg/util/collection/array/const_array.go @@ -13,13 +13,22 @@ package array import ( - "bytes" "fmt" "strings" + "unsafe" "github.com/LFDT-Lineth/zkc/pkg/util/word" ) +// NewConstantArray constructs a new word array with a given capacity. +func NewConstantArray[T word.Word[T]](height uint, bitwidth uint, value T) *ConstantArray[T] { + if !value.FitsWithin(bitwidth) { + panic(fmt.Sprintf("invalid constant value (%s) for u%d", value.String(), bitwidth)) + } + // + return &ConstantArray[T]{height, bitwidth, value} +} + // ================================================================================= // Implementation // ================================================================================= @@ -34,101 +43,41 @@ type ConstantArray[T word.Word[T]] struct { value T } -// NewConstantArray constructs a new word array with a given capacity. -func NewConstantArray[T word.Word[T]](height uint, bitwidth uint, value T) *ConstantArray[T] { - return &ConstantArray[T]{height, bitwidth, value} -} - -// Append new word on this array -func (p *ConstantArray[T]) Append(word T) { - // NOTE: attempting to assign a constant register any value other than the - // given constant cannot change the value stored in the register. This just - // means that a constraint somewhere should fail - p.height++ -} - -// AppendAll elements of the given bit array onto the this array, mutating it -// in place. -func (p *ConstantArray[T]) AppendAll(other ConstantArray[T]) { - if p.value.Cmp(other.value) != 0 { - panic(fmt.Sprintf("cannot append %s onto constant array for %s", p.value.String(), other.value.String())) - } - // NOTE: attempting to assign a constant register any value other than the - // given constant cannot change the value stored in the register. This just - // means that a constraint somewhere should fail - p.height += other.height -} - -// Clone makes clones of this array producing an otherwise identical copy. -func (p *ConstantArray[T]) Clone() MutArray[T] { - return &ConstantArray[T]{p.height, p.bitwidth, p.value} -} - -// Encode implementation for Array interface. The natural encoding of a -// constant array is simply its constant value, written once as a -// length-prefixed sequence of raw bytes. -func (p *ConstantArray[T]) Encode(buffer *bytes.Buffer) { - writeWordBytes(buffer, p.value.Bytes()) -} - -// Decode implementation for MutArray interface. This reads the constant value -// (as produced by Encode), and sets the array to hold the given number of -// copies of it. -func (p *ConstantArray[T]) Decode(height uint, buffer *bytes.Buffer) error { - data, err := readWordBytes(buffer) +// Bytes implementation for Array interface +func (p ConstantArray[T]) Bytes() uint { + var tmp T // - if err != nil { - return err - } - // - p.value = p.value.SetBytes(data) - p.height = height - // - return nil + return uint(unsafe.Sizeof(tmp)) } // Len returns the number of elements in this word array. -func (p *ConstantArray[T]) Len() uint { +func (p ConstantArray[T]) Len() uint { return p.height } // BitWidth returns the width (in bits) of elements in this array. -func (p *ConstantArray[T]) BitWidth() uint { +func (p ConstantArray[T]) BitWidth() uint { return p.bitwidth } -// Build implementation for the array.Builder interface. This simply means that -// a static array is its own builder. -func (p *ConstantArray[T]) Build() Array[T] { - return p -} - // Get returns the field element at the given index in this array. -func (p *ConstantArray[T]) Get(index uint) T { +func (p ConstantArray[T]) Get(index uint) T { return p.value } -// Set sets the field element at the given index in this array, overwriting the -// original value. -func (p *ConstantArray[T]) Set(index uint, word T) { - // NOTE: attempting to assign a constant register any value other than the - // given constant cannot change the value stored in the register. This just - // means that a constraint somewhere should fail -} - // Pad implementation for MutArray interface. The receiver is left // unmodified. -func (p *ConstantArray[T]) Pad(n uint, m uint, padding T) MutArray[T] { - if !padding.Equals(p.value) { - // NOTE: this can be implemented by changing the representation to - // something which can be mutated. - panic("unsupported operation") +func (p ConstantArray[T]) Pad(n uint) Array[T] { + var zero T + // + if !p.value.Equals(zero) { + return NewPaddedArray(p).Pad(n) } // - return &ConstantArray[T]{p.height + n + m, p.bitwidth, p.value} + return &ConstantArray[T]{p.height + n, p.bitwidth, p.value} } -func (p *ConstantArray[T]) String() string { +func (p ConstantArray[T]) String() string { var sb strings.Builder sb.WriteString("[") @@ -145,3 +94,70 @@ func (p *ConstantArray[T]) String() string { return sb.String() } + +// ================================================================================= +// MutArray Implementation +// ================================================================================= + +// Append new word on this array +func (p *ConstantArray[T]) Append(word T) MutArray[T] { + if word.Cmp(p.value) == 0 { + p.height++ + return p + } + // Determine necessary bitwidth + var ( + bitwidth = max(p.bitwidth, bitwidthOf(word)) + q MutArray[T] + ) + // Resize column + switch { + case bitwidth == 1: + q = NewBitArray[T](p.height, p.value.Cmp64(1) == 0) + case bitwidth <= 8: + q = NewSmallArray[uint8, T](bitwidth, p.height, uint8(p.value.Uint64())) + case bitwidth <= 16: + q = NewSmallArray[uint16, T](bitwidth, p.height, uint16(p.value.Uint64())) + case bitwidth <= 32: + q = NewSmallArray[uint32, T](bitwidth, p.height, uint32(p.value.Uint64())) + case bitwidth <= 64: + q = NewSmallArray[uint64, T](bitwidth, p.height, p.value.Uint64()) + default: + var arr = Fill(p.height, p.value) + // + return NewStaticArray[T](bitwidth, append(arr, word)...) + } + // + return q.Append(word) +} + +// AppendAll elements of the given bit array onto the this array, mutating it +// in place. +func (p *ConstantArray[T]) AppendAll(other ConstantArray[T]) { + if p.value.Cmp(other.value) != 0 { + panic(fmt.Sprintf("cannot append %s onto constant array for %s", p.value.String(), other.value.String())) + } + // NOTE: attempting to assign a constant register any value other than the + // given constant cannot change the value stored in the register. This just + // means that a constraint somewhere should fail + p.height += other.height +} + +// Build implementation for the array.Builder interface. This simply means that +// a static array is its own builder. +func (p *ConstantArray[T]) Build() Array[T] { + return p +} + +// Height implementation of MutArray interface +func (p *ConstantArray[T]) Height() uint { + return p.Len() +} + +// Set sets the field element at the given index in this array, overwriting the +// original value. +func (p *ConstantArray[T]) Set(index uint, word T) { + // NOTE: attempting to assign a constant register any value other than the + // given constant cannot change the value stored in the register. This just + // means that a constraint somewhere should fail +} diff --git a/pkg/util/collection/array/encoding.go b/pkg/util/collection/array/encoding.go new file mode 100644 index 000000000..579361399 --- /dev/null +++ b/pkg/util/collection/array/encoding.go @@ -0,0 +1,200 @@ +// Copyright Consensys Software Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +package array + +import ( + "bytes" + "encoding/binary" + "fmt" + + "github.com/LFDT-Lineth/zkc/pkg/util/collection/bit" + "github.com/LFDT-Lineth/zkc/pkg/util/word" +) + +// Encode writes the contents of this array into the given buffer, using +// the natural encoding for this array representation. The encoding is +// self-delimiting given the length of the array (i.e. Decode can determine +// how many bytes to read given the number of encoded elements). +func Encode[F word.Word[F]](array Array[F], buf *bytes.Buffer) { + switch t := array.(type) { + case *ConstantArray[F]: + encodeConstantArray(t, buf) + case *BitArray[F]: + encodeBitArray(t, buf) + case *SmallArray[uint8, F]: + encodeSmallArray(t, buf) + case *SmallArray[uint16, F]: + encodeSmallArray(t, buf) + case *SmallArray[uint32, F]: + encodeSmallArray(t, buf) + case *SmallArray[uint64, F]: + encodeSmallArray(t, buf) + case *StaticArray[F]: + encodeStaticArray(t, buf) + default: + panic("unknown array encountered") + } +} + +// Decode reads a given number of elements from the given buffer into this +// array, replacing any existing contents. The data is expected to be in +// the natural encoding for this array representation (i.e. as produced by +// Encode). +func Decode[F word.Word[F]](bitwidth uint, height uint, buf *bytes.Buffer) (Array[F], error) { + // Construct column + switch { + case bitwidth == 0: + return decodeConstantArray[F](height, bitwidth, buf) + case bitwidth == 1: + return decodeBitArray[F](height, buf) + case bitwidth <= 8: + return decodeSmallArray[uint8, F](height, buf) + case bitwidth <= 16: + return decodeSmallArray[uint16, F](height, buf) + case bitwidth <= 32: + return decodeSmallArray[uint32, F](height, buf) + case bitwidth <= 64: + return decodeSmallArray[uint64, F](height, buf) + default: + return decodeStaticArray[F](height, buf) + } +} + +// ============================================================================ +// Constant Array +// ============================================================================ + +// Encode implementation for Array interface. The natural encoding of a +// constant array is simply its constant value, written once as a +// length-prefixed sequence of raw bytes. +func encodeConstantArray[F word.Word[F]](p *ConstantArray[F], buffer *bytes.Buffer) { + writeWordBytes(buffer, p.value.Bytes()) +} + +// Decode implementation for MutArray interface. This reads the constant value +// (as produced by Encode), and sets the array to hold the given number of +// copies of it. +func decodeConstantArray[F word.Word[F]](height uint, bitwidth uint, buffer *bytes.Buffer) (Array[F], error) { + var ( + data, err = readWordBytes(buffer) + value F + ) + // + if err != nil { + return nil, err + } + // + return NewConstantArray(height, bitwidth, value.SetBytes(data)), nil +} + +// ============================================================================ +// Bit Array +// ============================================================================ + +// Encode implementation for Array interface. The natural encoding of a bit +// array is its packed byte representation, where eight bits are packed into +// each byte. +func encodeBitArray[T word.Word[T]](p *BitArray[T], buffer *bytes.Buffer) { + var ( + bytewidth = word.ByteWidth(p.height) + data = make([]byte, bytewidth) + ) + // + bit.LittleEndianCopy(p.data, 0, data, 0, p.height) + buffer.Write(data) +} + +// Decode implementation for MutArray interface. This reads a packed byte +// representation (as produced by Encode) holding the given number of bits. +func decodeBitArray[F word.Word[F]](height uint, buffer *bytes.Buffer) (Array[F], error) { + var ( + bytewidth = word.ByteWidth(height) + p BitArray[F] + ) + // + if uint(buffer.Len()) < bytewidth { + return nil, fmt.Errorf("bit array requires %d bytes, but only %d remain", bytewidth, buffer.Len()) + } + // Observe bytes must be cloned, since the slice returned by Next is only + // valid until the next buffer operation. + p.data = bytes.Clone(buffer.Next(int(bytewidth))) + p.height = height + // + return &p, nil +} + +// ============================================================================ +// Small Array +// ============================================================================ + +// Encode implementation for Array interface. The natural encoding of a small +// array is its elements written as fixed-width, little endian values. +func encodeSmallArray[K uint8 | uint16 | uint32 | uint64, T word.Word[T]](p *SmallArray[K, T], buffer *bytes.Buffer) { + if err := binary.Write(buffer, binary.LittleEndian, p.data); err != nil { + // Unreachable, since writes to a bytes.Buffer cannot fail. + panic(err) + } +} + +// Decode implementation for MutArray interface. This reads a given number of +// fixed-width, little endian values (as produced by Encode). +func decodeSmallArray[K uint8 | uint16 | uint32 | uint64, T word.Word[T]](height uint, buf *bytes.Buffer, +) (Array[T], error) { + var ( + data = make([]K, height) + p SmallArray[K, T] + ) + // + if err := binary.Read(buf, binary.LittleEndian, data); err != nil { + return nil, err + } + // + p.data = data + // + return &p, nil +} + +// ============================================================================ +// Static Array +// ============================================================================ + +// Encode implementation for Array interface. The natural encoding of a static +// array is its elements written as length-prefixed sequences of raw bytes. +func encodeStaticArray[T word.Word[T]](p *StaticArray[T], buffer *bytes.Buffer) { + for _, w := range p.data { + writeWordBytes(buffer, w.Bytes()) + } +} + +// Decode implementation for MutArray interface. This reads a given number of +// length-prefixed words (as produced by Encode). +func decodeStaticArray[T word.Word[T]](height uint, buffer *bytes.Buffer) (Array[T], error) { + var ( + data = make([]T, height) + p StaticArray[T] + ) + // + for i := range data { + bs, err := readWordBytes(buffer) + // + if err != nil { + return nil, err + } + // + data[i] = data[i].SetBytes(bs) + } + // + p.data = data + // + return &p, nil +} diff --git a/pkg/util/collection/array/interface.go b/pkg/util/collection/array/interface.go index d0e8f976e..058ec9e13 100644 --- a/pkg/util/collection/array/interface.go +++ b/pkg/util/collection/array/interface.go @@ -24,39 +24,29 @@ type Array[T any] interface { fmt.Stringer // Return the number of bits required to store an element of this array. BitWidth() uint - // Clone this array producing a mutable copy - Clone() MutArray[T] - // Encode writes the contents of this array into the given buffer, using - // the natural encoding for this array representation. The encoding is - // self-delimiting given the length of the array (i.e. Decode can determine - // how many bytes to read given the number of encoded elements). - Encode(*bytes.Buffer) + // Bytes returns (approximately) the number of bytes required to store the + // data of this column. + Bytes() uint // Get returns the element at the given index in this array. Get(uint) T // Returns the number of elements in this array. Len() uint - // Pad returns a copy of this array with n copies of the given value - // prepended and m copies appended. The receiver is left unmodified. - Pad(uint, uint, T) MutArray[T] + // Pad returns a copy of this array padding with n zero values prepended. + // The receiver is left unmodified. + Pad(uint) Array[T] } // MutArray provides a generice interface to an array of elements. Typically, we // are interested in arrays of field elements here. type MutArray[T any] interface { - Array[T] + // Build the given array + Build() Array[T] // Append new element onto the end of array producing an updated array. // This updates the array in place, and will panic if the given value is not // representable in the array. - Append(T) - // Decode reads a given number of elements from the given buffer into this - // array, replacing any existing contents. The data is expected to be in - // the natural encoding for this array representation (i.e. as produced by - // Encode). - Decode(uint, *bytes.Buffer) error - // Set the element at the given index in this array, overwriting the - // original value. This updates the array in place, and will panic if the - // given value is not representable in the array. - Set(uint, T) + Append(T) MutArray[T] + // Returns current height of array being built + Height() uint } // writeUvarint writes an unsigned varint into the given buffer. diff --git a/pkg/util/collection/array/padded_array.go b/pkg/util/collection/array/padded_array.go new file mode 100644 index 000000000..3dad178b9 --- /dev/null +++ b/pkg/util/collection/array/padded_array.go @@ -0,0 +1,68 @@ +// Copyright Consensys Software Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +package array + +import ( + "github.com/LFDT-Lineth/zkc/pkg/util/word" +) + +// PaddedArray implements an array of single bit words simply using an underlying +// array of packed bytes. That is, where eight bits are packed into a single +// byte. +type PaddedArray[T word.Word[T], S Array[T]] struct { + data S + // start identifies the first non-padding (i.e. real) row in this array. + // Every row below start is padding and, therefore, implicitly zero. + start uint +} + +// NewPaddedArray constructs a new word array with a given capacity. +func NewPaddedArray[T word.Word[T], S Array[T]](data S) PaddedArray[T, S] { + return PaddedArray[T, S]{data, 0} +} + +// Bytes implementation for Array interface +func (p PaddedArray[W, T]) Bytes() uint { + return p.data.Bytes() +} + +// Len returns the number of elements in this word array. +func (p PaddedArray[T, S]) Len() uint { + return p.data.Len() + p.start +} + +// BitWidth returns the width (in bits) of elements in this array. +func (p PaddedArray[T, S]) BitWidth() uint { + return 1 +} + +// Get returns the field element at the given index in this array. +func (p PaddedArray[T, S]) Get(index uint) T { + var b T + // + if index < p.start { + return b + } + // + return p.data.Get(index - p.start) +} + +// Pad returns a copy of this array with n copies of the given padding value +// prepended, and m copies appended. The receiver is left unmodified. +func (p PaddedArray[T, S]) Pad(n uint) Array[T] { + return PaddedArray[T, S]{p.data, p.start + n} +} + +func (p PaddedArray[T, S]) String() string { + return p.data.String() +} diff --git a/pkg/util/collection/array/small_array.go b/pkg/util/collection/array/small_array.go index e5aa38a22..0642c7b95 100644 --- a/pkg/util/collection/array/small_array.go +++ b/pkg/util/collection/array/small_array.go @@ -13,15 +13,24 @@ package array import ( - "bytes" - "encoding/binary" "fmt" "slices" "strings" + "unsafe" "github.com/LFDT-Lineth/zkc/pkg/util/word" ) +// NewSmallArray constructs a new word array with a given capacity. +func NewSmallArray[K uint8 | uint16 | uint32 | uint64, T word.Word[T]](bitwidth uint, height uint, value K, +) *SmallArray[K, T] { + return &SmallArray[K, T]{Fill(height, value), bitwidth} +} + +// ================================================================================= +// Implementation +// ================================================================================= + // SmallArray implements an array of elements simply using an underlying array. type SmallArray[K uint8 | uint16 | uint32 | uint64, T word.Word[T]] struct { // The data stored in this column (as bytes). @@ -30,129 +39,41 @@ type SmallArray[K uint8 | uint16 | uint32 | uint64, T word.Word[T]] struct { bitwidth uint } -// NewSmallArray constructs a new word array with a given capacity. -func NewSmallArray[K uint8 | uint16 | uint32 | uint64, T word.Word[T]](height uint, bitwidth uint) *SmallArray[K, T] { +// Bytes implementation for Array interface +func (p SmallArray[K, T]) Bytes() uint { var ( - elements = make([]K, height) + tmp K + n = uint(unsafe.Sizeof(tmp)) ) // - return &SmallArray[K, T]{elements, bitwidth} -} - -// Append new word on this array -func (p *SmallArray[K, T]) Append(word T) { - p.data = append(p.data, K(word.Uint64())) -} - -// AppendAll elements of the given array onto the this array, mutating it in -// place. -func (p *SmallArray[K, T]) AppendAll(other SmallArray[K, T]) { - // Determine height of resulting array - var ( - nsize = uint(len(p.data) + len(other.data)) - n = nsize - uint(len(p.data)) - ) - // sanity check - if p.bitwidth != other.bitwidth { - panic(fmt.Sprintf("incompatible array bitwidth (u%d vs u%d)", p.bitwidth, other.bitwidth)) - } - // expand data length - ndata := slices.Grow(p.data, int(n))[:nsize] - // copy data - copy(ndata[len(p.data):], other.data) - // finalisex - p.data = ndata + return n * p.Len() } // Len returns the number of elements in this word array. -func (p *SmallArray[K, T]) Len() uint { +func (p SmallArray[K, T]) Len() uint { // return uint(len(p.data)) } // BitWidth returns the width (in bits) of elements in this array. -func (p *SmallArray[K, T]) BitWidth() uint { +func (p SmallArray[K, T]) BitWidth() uint { return p.bitwidth } -// Encode implementation for Array interface. The natural encoding of a small -// array is its elements written as fixed-width, little endian values. -func (p *SmallArray[K, T]) Encode(buffer *bytes.Buffer) { - if err := binary.Write(buffer, binary.LittleEndian, p.data); err != nil { - // Unreachable, since writes to a bytes.Buffer cannot fail. - panic(err) - } -} - -// Decode implementation for MutArray interface. This reads a given number of -// fixed-width, little endian values (as produced by Encode). -func (p *SmallArray[K, T]) Decode(height uint, buffer *bytes.Buffer) error { - data := make([]K, height) - // - if err := binary.Read(buffer, binary.LittleEndian, data); err != nil { - return err - } - // - p.data = data - // - return nil -} - -// Clone makes clones of this array producing an otherwise identical copy. -func (p *SmallArray[K, T]) Clone() MutArray[T] { - // Allocate sufficient memory - ndata := make([]K, uint(len(p.data))) - // Copy over the data - copy(ndata, p.data) - // - return &SmallArray[K, T]{ndata, p.bitwidth} -} - // Get returns the word at the given index in this array. -func (p *SmallArray[K, T]) Get(index uint) T { +func (p SmallArray[K, T]) Get(index uint) T { var val T // return val.SetUint64(uint64(p.data[index])) } -// Set the word at the given index in this array, overwriting the -// original value. -func (p *SmallArray[K, T]) Set(index uint, word T) { - p.data[index] = K(word.Uint64()) -} - -// SetRaw sets a raw value at the given index in this array, overwriting the -// original value. -func (p *SmallArray[K, T]) SetRaw(index uint, val K) { - p.data[index] = val -} - // Pad returns a copy of this array with n copies of the given padding value // prepended, and m copies appended. The receiver is left unmodified. -func (p *SmallArray[K, T]) Pad(n uint, m uint, padding T) MutArray[T] { - var ( - ol = p.Len() - // Determine new length - l = n + ol + m - val = K(padding.Uint64()) - // Allocate exactly, copying existing data directly into its final - // position. - data = make([]K, l) - ) - // - copy(data[n:], p.data) - // Front padding! - for i := range n { - data[i] = val - } - // Back padding! - for i := l - m; i < l; i++ { - data[i] = val - } - // - return &SmallArray[K, T]{data, p.bitwidth} +func (p SmallArray[K, T]) Pad(n uint) Array[T] { + return NewPaddedArray(p).Pad(n) } -func (p *SmallArray[K, T]) String() string { + +func (p SmallArray[K, T]) String() string { var sb strings.Builder sb.WriteString("[") @@ -169,3 +90,51 @@ func (p *SmallArray[K, T]) String() string { return sb.String() } + +// ================================================================================= +// MutArray Implementation +// ================================================================================= + +// Append new word on this array +func (p *SmallArray[K, T]) Append(word T) MutArray[T] { + p.data = append(p.data, K(word.Uint64())) + // + return p +} + +// AppendAll elements of the given array onto the this array, mutating it in +// place. +func (p *SmallArray[K, T]) AppendAll(other SmallArray[K, T]) { + // Determine height of resulting array + var ( + nsize = uint(len(p.data) + len(other.data)) + n = nsize - uint(len(p.data)) + ) + // sanity check + if p.bitwidth != other.bitwidth { + panic(fmt.Sprintf("incompatible array bitwidth (u%d vs u%d)", p.bitwidth, other.bitwidth)) + } + // expand data length + ndata := slices.Grow(p.data, int(n))[:nsize] + // copy data + copy(ndata[len(p.data):], other.data) + // finalisex + p.data = ndata +} + +// Build implementation for the array.Builder interface. This simply means that +// a static array is its own builder. +func (p *SmallArray[K, T]) Build() Array[T] { + return p +} + +// Height implementation of MutArray interface +func (p *SmallArray[K, T]) Height() uint { + return p.Len() +} + +// Set the word at the given index in this array, overwriting the +// original value. +func (p *SmallArray[K, T]) Set(index uint, word T) { + p.data[index] = K(word.Uint64()) +} diff --git a/pkg/util/collection/array/static_array.go b/pkg/util/collection/array/static_array.go index 2100bcd6d..188f2e650 100644 --- a/pkg/util/collection/array/static_array.go +++ b/pkg/util/collection/array/static_array.go @@ -13,14 +13,24 @@ package array import ( - "bytes" "fmt" "slices" "strings" + "unsafe" "github.com/LFDT-Lineth/zkc/pkg/util/word" ) +// NewStaticArray constructs a new static array from the given elements. +func NewStaticArray[T word.Word[T]](bitwidth uint, elements ...T) *StaticArray[T] { + // + return &StaticArray[T]{elements, bitwidth} +} + +// ================================================================================= +// Implementation +// ================================================================================= + // StaticArray implements an array of elements simply using an underlying array. type StaticArray[T word.Word[T]] struct { // The data stored in this column (as bytes). @@ -29,18 +39,66 @@ type StaticArray[T word.Word[T]] struct { bitwidth uint } -// NewStaticArray constructs a new word array with a given capacity. -func NewStaticArray[T word.Word[T]](height uint, bitwidth uint) *StaticArray[T] { +// Bytes implementation for Array interface +func (p StaticArray[T]) Bytes() uint { var ( - elements = make([]T, height) + tmp T + // NOTE: this works correctly only for fixed-width word types. + n = uint(unsafe.Sizeof(tmp)) ) // - return &StaticArray[T]{elements, bitwidth} + return n * p.Len() +} + +// Len returns the number of elements in this word array. +func (p StaticArray[T]) Len() uint { + // + return uint(len(p.data)) +} + +// BitWidth returns the width (in bits) of elements in this array. +func (p StaticArray[T]) BitWidth() uint { + return p.bitwidth +} + +// Get returns the field element at the given index in this array. +func (p StaticArray[T]) Get(index uint) T { + return p.data[index] +} + +// Pad returns a copy of this array with n copies of the given padding value +// prepended, and m copies appended. The receiver is left unmodified. +func (p StaticArray[T]) Pad(n uint) Array[T] { + return NewPaddedArray(p).Pad(n) } +func (p StaticArray[T]) String() string { + var sb strings.Builder + + sb.WriteString("[") + + for i := range p.Len() { + if i != 0 { + sb.WriteString(",") + } + + fmt.Fprintf(&sb, "%v", p.Get(i)) + } + + sb.WriteString("]") + + return sb.String() +} + +// ================================================================================= +// MutArray Implementation +// ================================================================================= + // Append new word on this array -func (p *StaticArray[T]) Append(word T) { +func (p *StaticArray[T]) Append(word T) MutArray[T] { p.data = append(p.data, word) + // + return p } // AppendAll elements of the given array onto the this array, mutating it in @@ -63,20 +121,15 @@ func (p *StaticArray[T]) AppendAll(other StaticArray[T]) { p.data = ndata } -// Len returns the number of elements in this word array. -func (p *StaticArray[T]) Len() uint { - // - return uint(len(p.data)) -} - -// BitWidth returns the width (in bits) of elements in this array. -func (p *StaticArray[T]) BitWidth() uint { - return p.bitwidth +// Build implementation for the array.Builder interface. This simply means that +// a static array is its own builder. +func (p *StaticArray[T]) Build() Array[T] { + return p } -// Get returns the field element at the given index in this array. -func (p *StaticArray[T]) Get(index uint) T { - return p.data[index] +// Height implementation of MutArray interface +func (p *StaticArray[T]) Height() uint { + return p.Len() } // Set sets the field element at the given index in this array, overwriting the @@ -84,83 +137,3 @@ func (p *StaticArray[T]) Get(index uint) T { func (p *StaticArray[T]) Set(index uint, word T) { p.data[index] = word } - -// Encode implementation for Array interface. The natural encoding of a static -// array is its elements written as length-prefixed sequences of raw bytes. -func (p *StaticArray[T]) Encode(buffer *bytes.Buffer) { - for _, w := range p.data { - writeWordBytes(buffer, w.Bytes()) - } -} - -// Decode implementation for MutArray interface. This reads a given number of -// length-prefixed words (as produced by Encode). -func (p *StaticArray[T]) Decode(height uint, buffer *bytes.Buffer) error { - data := make([]T, height) - // - for i := range data { - bs, err := readWordBytes(buffer) - // - if err != nil { - return err - } - // - data[i] = data[i].SetBytes(bs) - } - // - p.data = data - // - return nil -} - -// Clone makes clones of this array producing an otherwise identical copy. -func (p *StaticArray[T]) Clone() MutArray[T] { - // Allocate sufficient memory - ndata := make([]T, uint(len(p.data))) - // Copy over the data - copy(ndata, p.data) - // - return &StaticArray[T]{ndata, p.bitwidth} -} - -// Pad returns a copy of this array with n copies of the given padding value -// prepended, and m copies appended. The receiver is left unmodified. -func (p *StaticArray[T]) Pad(n uint, m uint, padding T) MutArray[T] { - var ( - ol = p.Len() - // Determine new length - l = n + ol + m - // Allocate exactly, copying existing data directly into its final - // position. - data = make([]T, l) - ) - // - copy(data[n:], p.data) - // Front padding! - for i := range n { - data[i] = padding - } - // Back padding! - for i := l - m; i < l; i++ { - data[i] = padding - } - // - return &StaticArray[T]{data, p.bitwidth} -} -func (p *StaticArray[T]) String() string { - var sb strings.Builder - - sb.WriteString("[") - - for i := range p.Len() { - if i != 0 { - sb.WriteString(",") - } - - fmt.Fprintf(&sb, "%v", p.Get(i)) - } - - sb.WriteString("]") - - return sb.String() -} diff --git a/pkg/util/collection/array/util.go b/pkg/util/collection/array/util.go index 4bebbb526..36dc77342 100644 --- a/pkg/util/collection/array/util.go +++ b/pkg/util/collection/array/util.go @@ -56,6 +56,17 @@ func Expand[T any](data []T, n uint) []T { return data[:nsize] } +// Fill constructs an array of a given size filled with the given item. +func Fill[T any](n uint, item T) []T { + var nslice = make([]T, n) + // Pad out remainder + for i := uint(0); i < n; i++ { + nslice[i] = item + } + // + return nslice +} + // FrontPad pads an array upto a given length n with a given item. // Specifically, new items are inserted at the front of the array. func FrontPad[T any](slice []T, n uint, item T) []T { diff --git a/pkg/util/field/batch_invert.go b/pkg/util/field/batch_invert.go index 2d909fdb7..024ed1366 100644 --- a/pkg/util/field/batch_invert.go +++ b/pkg/util/field/batch_invert.go @@ -13,59 +13,59 @@ package field import ( - "github.com/LFDT-Lineth/zkc/pkg/util/collection/array" "github.com/LFDT-Lineth/zkc/pkg/util/collection/bit" ) // BatchInvert efficiently inverts the list of elements s, in place. -func BatchInvert[T Element[T]](s array.MutArray[T]) { - if s.Len() == 0 { +func BatchInvert[T Element[T]](s []T) { + if len(s) == 0 { return } // var ( zero = Zero[T]() one = One[T]() + last = uint(len(s) - 1) // identifies entries which are zero - isZero = bit.NewSet(s.Len()) + isZero = bit.NewSet(uint(len(s))) - m = make([]T, s.Len()) // m[i] = s[i] * s[i+1] * ... + m = make([]T, len(s)) // m[i] = s[i] * s[i+1] * ... ) // - isZero.Set(s.Len()-1, s.Get(s.Len()-1).IsZero()) + isZero.Set(last, s[last].IsZero()) - if isZero.Get(s.Len() - 1) { - s.Set(s.Len()-1, one) + if isZero.Get(last) { + s[last] = one } - m[s.Len()-1] = s.Get(s.Len() - 1) + m[last] = s[last] - for i := int(s.Len()) - 2; i >= 0; i-- { - isZero.Set(uint(i), s.Get(uint(i)).IsZero()) + for i := len(s) - 2; i >= 0; i-- { + isZero.Set(uint(i), s[uint(i)].IsZero()) if isZero.Get(uint(i)) { - s.Set(uint(i), one) + s[uint(i)] = one } - m[i] = m[i+1].Mul(s.Get(uint(i))) + m[i] = m[i+1].Mul(s[uint(i)]) } inv := m[0].Inverse() // inv = s[0]⁻¹ * s[1]⁻¹ * ... - for i := range s.Len() - 1 { + for i := range len(s) - 1 { // inv = s[i]⁻¹ * s[i+1]⁻¹ * ... - newInv := inv.Mul(s.Get(i)) - s.Set(i, inv.Mul(m[i+1])) + newInv := inv.Mul(s[i]) + s[i] = inv.Mul(m[i+1]) inv = newInv // inv = s[i+1]⁻¹ * s[i+2]⁻¹ * ... - if isZero.Get(i) { - s.Set(i, zero) + if isZero.Get(uint(i)) { + s[i] = zero } } - s.Set(s.Len()-1, inv) + s[last] = inv - if isZero.Get(s.Len() - 1) { - s.Set(s.Len()-1, zero) + if isZero.Get(last) { + s[last] = zero } } diff --git a/pkg/util/field/element_test.go b/pkg/util/field/element_test.go index e0d5e343a..9add88852 100644 --- a/pkg/util/field/element_test.go +++ b/pkg/util/field/element_test.go @@ -15,7 +15,6 @@ package field import ( "bytes" "math/rand" - "slices" "testing" "github.com/LFDT-Lineth/zkc/pkg/util/assert" @@ -58,10 +57,6 @@ func (e elementArray) BitWidth() uint { panic("not implemented") } -func (e elementArray) Clone() array.MutArray[koalabear.Element] { - return slices.Clone(e) -} - func (e elementArray) Get(u uint) koalabear.Element { return e[u] } @@ -86,7 +81,7 @@ func (e elementArray) Set(u uint, t koalabear.Element) { e[u] = t } -func (e elementArray) Pad(u uint, u2 uint, t koalabear.Element) array.MutArray[koalabear.Element] { +func (e elementArray) Pad(u uint, u2 uint, t koalabear.Element) array.Array[koalabear.Element] { panic("not implemented") } diff --git a/pkg/util/termio/table.go b/pkg/util/termio/table.go index 035eb289b..44dbfe48d 100644 --- a/pkg/util/termio/table.go +++ b/pkg/util/termio/table.go @@ -279,14 +279,18 @@ func (p TableSorter) SortColumn(col uint) TableSorter { // SortNumericalColumn adds a sort by the given column to the table sorter. func (p TableSorter) SortNumericalColumn(col uint) TableSorter { return func(lhs []FormattedText, rhs []FormattedText) int { - var l, r string // Try parent sort if c := p(lhs, rhs); c != 0 { return c } // - l = string(lhs[col].text) - r = string(rhs[col].text) + var ( + lv = string(lhs[col].text) + rv = string(rhs[col].text) + ) + // + l := parseNumericColumn(lv) + r := parseNumericColumn(rv) // if len(l) < len(r) { return -1 @@ -297,3 +301,22 @@ func (p TableSorter) SortNumericalColumn(col uint) TableSorter { return strings.Compare(l, r) } } + +func parseNumericColumn(text string) string { + var ( + gtext, giga = strings.CutSuffix(text, "G") + mtext, mega = strings.CutSuffix(text, "M") + ktext, kilo = strings.CutSuffix(text, "K") + ) + // Account for "human-readable" forms. + switch { + case giga: + return fmt.Sprintf("%s000000000", gtext) + case mega: + return fmt.Sprintf("%s000000", mtext) + case kilo: + return fmt.Sprintf("%s000", ktext) + default: + return text + } +} diff --git a/pkg/zkc/constraints/binary_file.go b/pkg/zkc/constraints/binary_file.go index d4cc2aa8b..64380787c 100644 --- a/pkg/zkc/constraints/binary_file.go +++ b/pkg/zkc/constraints/binary_file.go @@ -52,7 +52,7 @@ const BINFILE_MINOR_VERSION uint16 = 1 var ZKC_EXEC [8]byte = [8]byte{'z', 'k', 'c', ' ', 'e', 'x', 'e', 'c'} // Tracer defines the type used for building traces. -type Tracer[F field.Element[F]] = tracer.Builder[vm.Uint32, F, *trace.CompactModule[F]] +type Tracer[F field.Element[F]] = tracer.Builder[vm.Uint32, F] // BinaryFile provides two pieces of functionality: (i) a means for serialising // and deserialising a set of AIR constraints; (ii) a means for generating a diff --git a/pkg/zkc/constraints/trace/builder.go b/pkg/zkc/constraints/trace/builder.go index 9967ac09c..ee4476dcf 100644 --- a/pkg/zkc/constraints/trace/builder.go +++ b/pkg/zkc/constraints/trace/builder.go @@ -16,6 +16,7 @@ import ( "fmt" "github.com/LFDT-Lineth/zkc/pkg/trace" + "github.com/LFDT-Lineth/zkc/pkg/util/collection/array" "github.com/LFDT-Lineth/zkc/pkg/util/field" "github.com/LFDT-Lineth/zkc/pkg/zkc/vm" ) @@ -28,9 +29,7 @@ type ( // Memory provides a useful alias Memory[W Word[W]] = vm.RuntimeMemory[W] // Module provides a useful alias - Module[F any] = trace.Module[F] - // ModuleBuilder provides a useful alias - ModuleBuilder[F any, M trace.Module[F]] = trace.ModuleBuilder[F, M] + Module[F field.Element[F]] = trace.Module[F] ) const ( @@ -112,47 +111,51 @@ func toTraceRegister[W Word[W]](_ uint, reg vm.Register[W]) trace.ColumnDescript // auxiliary registers as required (e.g. for selector bits, etc). For // functions, this means transcribing each state generated for the function // during execution. -type Builder[W Word[W], F Element[F], M trace.ModuleBuilder[F, M]] struct { +type Builder[W Word[W], F Element[F]] struct { descriptors []vm.Module[W] // set of modules actively being traced - modules []M + modules []*trace.ModuleBuilder[F] // scratch memory area, used to avoid memory allocation. scratch []F } // Init initialises a new trace builder from a given program. -func (p Builder[W, F, M]) Init(program vm.Program[W]) Builder[W, F, M] { +func (p Builder[W, F]) Init(program vm.Program[W]) Builder[W, F] { var ( maxWidth uint // - modules = make([]M, len(program.Modules())) + modules = make([]*trace.ModuleBuilder[F], len(program.Modules())) ) // for i, m := range program.Modules() { switch m := m.(type) { case *vm.Function[W]: if m.IsOneLine() { - modules[i] = initOneLineFunction[W, F, M](*m) + modules[i] = initOneLineFunction[W, F](*m) } else { - modules[i] = initMultiLineFunction[W, F, M](*m) + modules[i] = initMultiLineFunction[W, F](*m) } case *vm.Memory[W]: - modules[i] = initialiseMemory[W, F, M](program.Field(), *m) + modules[i] = initialiseMemory[W, F](program.Field(), *m) } // Update maximum width maxWidth = max(maxWidth, modules[i].Width()) } // allocate scratch memory - return Builder[W, F, M]{program.Modules(), modules, make([]F, maxWidth)} + return Builder[W, F]{program.Modules(), modules, make([]F, maxWidth)} } // Build implementation for TraceBuilder interface. -func (p Builder[W, F, M]) Build() trace.Shard[F] { - return trace.NewArray(p.modules) +func (p Builder[W, F]) Build() trace.Shard[F] { + var tr = trace.NewShard(array.Map(p.modules, func(_ uint, b *trace.ModuleBuilder[F]) trace.Module[F] { + return b.Build() + })) + // + return tr } // TraceFunctionLine implementation for the vm.TraceBuilder interface. -func (p Builder[W, F, M]) TraceFunctionLine(state vm.State[W]) { +func (p Builder[W, F]) TraceFunctionLine(state vm.State[W]) { var ( mod = p.modules[state.Fid()] f = p.descriptors[state.Fid()].(*vm.Function[W]) @@ -166,7 +169,7 @@ func (p Builder[W, F, M]) TraceFunctionLine(state vm.State[W]) { } // TraceMemory implementation for the vm.TraceBuilder interface. -func (p Builder[W, F, M]) TraceMemory(mid uint16, m vm.RuntimeMemory[W], field field.Config) { +func (p Builder[W, F]) TraceMemory(mid uint16, m vm.RuntimeMemory[W], field field.Config) { var module = p.modules[mid] // switch m.Descriptor().Kind() { @@ -181,19 +184,18 @@ func (p Builder[W, F, M]) TraceMemory(mid uint16, m vm.RuntimeMemory[W], field f } } -func initialiseMemory[W Word[W], F Element[F], M trace.ModuleBuilder[F, M]](cfg field.Config, memory vm.Memory[W]) M { +func initialiseMemory[W Word[W], F Element[F]](cfg field.Config, memory vm.Memory[W]) *trace.ModuleBuilder[F] { switch memory.Kind() { case vm.PRIVATE_STATIC_MEMORY, vm.PUBLIC_STATIC_MEMORY: - var empty M // ProcessStaticMemory does what is required to represent a static memory within // a trace. Specifically, static memories do exist in the trace, but only to // ensure alignment of module identifiers. Hence, they always have an empty trace. - return empty.Initialise(trace.NewModuleDescriptor(memory.Name(), nil)) + return trace.InitModuleBuilder[F](trace.NewModuleDescriptor(memory.Name(), nil)) case vm.PRIVATE_READ_ONLY_MEMORY, vm.PUBLIC_READ_ONLY_MEMORY: - return initAccessOnceMemory[W, F, M](memory) + return initAccessOnceMemory[W, F](memory) case vm.PRIVATE_WRITE_ONCE_MEMORY, vm.PUBLIC_WRITE_ONCE_MEMORY: - return initAccessOnceMemory[W, F, M](memory) + return initAccessOnceMemory[W, F](memory) default: - return initReadWriteMemory[W, F, M](cfg, memory) + return initReadWriteMemory[W, F](cfg, memory) } } diff --git a/pkg/zkc/constraints/trace/process_aom.go b/pkg/zkc/constraints/trace/process_aom.go index 44ad28707..42cdc0c31 100644 --- a/pkg/zkc/constraints/trace/process_aom.go +++ b/pkg/zkc/constraints/trace/process_aom.go @@ -31,7 +31,7 @@ import ( // // Here, ADDRESS is the set of declared input registers, whilst DATA is the set // of declared output registers. -func initAccessOnceMemory[W Word[W], F Element[F], M ModuleBuilder[F, M]](m vm.Memory[W]) (module M) { +func initAccessOnceMemory[W Word[W], F Element[F]](m vm.Memory[W]) (module *trace.ModuleBuilder[F]) { var ( // Number of address lines nAddressLines = m.NumInputs() @@ -54,12 +54,13 @@ func initAccessOnceMemory[W Word[W], F Element[F], M ModuleBuilder[F, M]](m vm.M descriptor := trace.NewModuleDescriptor(m.Name(), regs). WithReplication(true) // - return module.Initialise(descriptor) + return trace.InitModuleBuilder[F](descriptor) } // traceAccessOnceMemory materialises the trace rows for a read-only (ROM) or // write-once (WOM) memory -func traceAccessOnceMemory[W vm.Word[W], F Element[F]](m vm.RuntimeMemory[W], module Module[F], scratch []F) { +func traceAccessOnceMemory[W vm.Word[W], F Element[F]](m vm.RuntimeMemory[W], module *trace.ModuleBuilder[F], + scratch []F) { var ( one = field.Uint64[F](1) geometry = m.Descriptor() diff --git a/pkg/zkc/constraints/trace/process_function.go b/pkg/zkc/constraints/trace/process_function.go index a51ed1b55..ac2a8f749 100644 --- a/pkg/zkc/constraints/trace/process_function.go +++ b/pkg/zkc/constraints/trace/process_function.go @@ -40,7 +40,7 @@ type Transcriber[W Word[W], F Element[F]] func(state vm.State[W]) []F // function occupies exactly one active row; padding rows are left at 0. // TODO: see https://github.com/LFDT-Lineth/zkc/issues/1975 // OLI won't have a $ret column -func initOneLineFunction[W Word[W], F Element[F], M ModuleBuilder[F, M]](f vm.Function[W]) (module M) { +func initOneLineFunction[W Word[W], F Element[F]](f vm.Function[W]) (module *trace.ModuleBuilder[F]) { var ( // Native functions do not (currently) have return lines hasRet = !f.IsNative() @@ -53,12 +53,12 @@ func initOneLineFunction[W Word[W], F Element[F], M ModuleBuilder[F, M]](f vm.Fu regs = append(regs, trace.NewColumnDescriptor(RET_NAME, util.Some[uint](1))) } // Initialise the module - return module.Initialise(trace.NewModuleDescriptor(f.Name(), regs)) + return trace.InitModuleBuilder[F](trace.NewModuleDescriptor(f.Name(), regs)) } // traceOneLineFunction materialises a trace row for a one-line function. -func traceOneLineFunction[W Word[W], F Element[F]](f vm.Function[W], m Module[F], st vm.State[W], scratch []F, -) { +func traceOneLineFunction[W Word[W], F Element[F]](f vm.Function[W], m *trace.ModuleBuilder[F], st vm.State[W], + scratch []F) { // var ( one = field.Uint64[F](1) @@ -91,7 +91,7 @@ func traceOneLineFunction[W Word[W], F Element[F]](f vm.Function[W], m Module[F] // +----------+-----+----+--------+--------+-----+ // // Here, REGS is the set of registers declared by the given function. -func initMultiLineFunction[W Word[W], F Element[F], M ModuleBuilder[F, M]](f vm.Function[W]) (module M) { +func initMultiLineFunction[W Word[W], F Element[F]](f vm.Function[W]) (module *trace.ModuleBuilder[F]) { var ( nVectors = uint(len(f.Vectors())) // Copy over all address / data lines @@ -111,11 +111,11 @@ func initMultiLineFunction[W Word[W], F Element[F], M ModuleBuilder[F, M]](f vm. regs = append(regs, trace.NewColumnDescriptor(SelectorName(k), u1)) } // Initialise the module - return module.Initialise(trace.NewModuleDescriptor(f.Name(), regs)) + return trace.InitModuleBuilder[F](trace.NewModuleDescriptor(f.Name(), regs)) } // traceMultiLineFunction materialises a trace row a multi-line function -func traceMultiLineFunction[W Word[W], F Element[F]](m Module[F], st vm.State[W], scratch []F) { +func traceMultiLineFunction[W Word[W], F Element[F]](m *trace.ModuleBuilder[F], st vm.State[W], scratch []F) { // var ( one = field.Uint64[F](1) diff --git a/pkg/zkc/constraints/trace/process_ram.go b/pkg/zkc/constraints/trace/process_ram.go index fe6c8ca24..ab62ca3ee 100644 --- a/pkg/zkc/constraints/trace/process_ram.go +++ b/pkg/zkc/constraints/trace/process_ram.go @@ -73,7 +73,7 @@ type ramAccess[W Word[W]] struct { } // InitReadWriteMemory initialises a trace module for a RandomAccessMemory. -func initReadWriteMemory[W Word[W], F Element[F], M ModuleBuilder[F, M]](cfg field.Config, m vm.Memory[W]) (module M) { +func initReadWriteMemory[W Word[W], F Element[F]](cfg field.Config, m vm.Memory[W]) (module *trace.ModuleBuilder[F]) { var ( regs = array.Map(m.Registers(), toTraceRegister) // Timestamp limb widths, most-significant first: the memory's declared @@ -115,8 +115,8 @@ func initReadWriteMemory[W Word[W], F Element[F], M ModuleBuilder[F, M]](cfg fie trace.NewColumnDescriptor(RAM_EXEC_WRITE_NAME, u1), trace.NewColumnDescriptor(RAM_EXEC_READ_NAME, u1), ) - //Done - return module.Initialise(trace.NewModuleDescriptor(m.Name(), regs)) + // Done + return trace.InitModuleBuilder[F](trace.NewModuleDescriptor(m.Name(), regs)) } // traceReadWriteMemory materialises the trace of a read-write (RAM) memory: one @@ -125,8 +125,8 @@ func initReadWriteMemory[W Word[W], F Element[F], M ModuleBuilder[F, M]](cfg fie // constraints.translateReadWriteMemory; the finalization phase (FINL) is left // empty (the rcv/snd consistency bus and finalization rows are a follow-up), so // FINL-guarded columns (ADDRESS_DELTA, ADDR_CARRY) stay zero. -func traceReadWriteMemory[W Word[W], F Element[F]](m vm.RuntimeMemory[W], module Module[F], cfg field.Config, - scratch []F) { +func traceReadWriteMemory[W Word[W], F Element[F]](m vm.RuntimeMemory[W], module *trace.ModuleBuilder[F], + cfg field.Config, scratch []F) { // var ( geometry = m.Descriptor() diff --git a/pkg/zkc/vm/machine.go b/pkg/zkc/vm/machine.go index 562a6c871..5f2827030 100644 --- a/pkg/zkc/vm/machine.go +++ b/pkg/zkc/vm/machine.go @@ -162,7 +162,7 @@ func BootAndCheckpoint[W Word[W]](pr Program[W], in map[string][]byte, strategy // holds when the given execution can be traced (i.e. when no errors in the // latter category arise). func BootAndTrace[W Word[W], F Element[F], T Tracer[W, F, T]](pr Program[W], input map[string][]byte, -) (trace Trace[F], output map[string][]byte, errs []error) { +) (trace Shard[F], output map[string][]byte, errs []error) { // var ( // constracter tracer @@ -176,7 +176,7 @@ func BootAndTrace[W Word[W], F Element[F], T Tracer[W, F, T]](pr Program[W], inp bci := constructTracingInterpreter(pr, tracer) // Execute machine in chunks of 1K steps if output, traceable, errs = BootAndExecute(bci, input, math.MaxUint); !traceable { - return nil, nil, errs + return Shard[F]{}, nil, errs } // var stats = util.NewPerfStats() @@ -198,7 +198,7 @@ func BootAndTrace[W Word[W], F Element[F], T Tracer[W, F, T]](pr Program[W], inp // traceable flag holds when the given execution can be traced (i.e. when no // errors in the latter category arise). func RestoreAndTraceFor[W Word[W], F Element[F], T Tracer[W, F, T]](pr Program[W], cp CheckPoint[W], - fn string, nsteps uint64) (steps uint64, trace Trace[F], errs []error) { + fn string, nsteps uint64) (steps uint64, trace Shard[F], errs []error) { // var ( // constracter tracer @@ -212,13 +212,13 @@ func RestoreAndTraceFor[W Word[W], F Element[F], T Tracer[W, F, T]](pr Program[W bci := constructTraceForInterpreter(pr, fn, nsteps, tracer) // Sanity check error arising construct the interpreter. if bci == nil { - return 0, nil, []error{ + return 0, Shard[F]{}, []error{ fmt.Errorf("unknown function \"%s\"", fn), } } // Execute the given machine if steps, traceable, errs = RestoreAndExecute(bci, cp, math.MaxUint); !traceable { - return steps, nil, errs + return steps, Shard[F]{}, errs } // Apply post processing array.Apply(bci.ExtractMemory(), func(_ uint, p util.Pair[uint16, RuntimeMemory[W]]) { diff --git a/pkg/zkc/vm/trace_builder.go b/pkg/zkc/vm/trace_builder.go index 56be4b4b3..76b4f86e5 100644 --- a/pkg/zkc/vm/trace_builder.go +++ b/pkg/zkc/vm/trace_builder.go @@ -19,8 +19,8 @@ import ( log "github.com/sirupsen/logrus" ) -// Trace defines the type of a general trace -type Trace[F field.Element[F]] = trace.Shard[F] +// Shard defines a component of a trace. +type Shard[F field.Element[F]] = trace.Shard[F] // Element defines the type of field elements type Element[F any] = field.Element[F] @@ -76,13 +76,13 @@ func (p TraceBuilder[W, F, T]) BootAndTrace(inputs map[string][]byte, // Sharded BootAndTrace performs sharding according to the given sharding // strategy. func (p TraceBuilder[W, F, T]) bootAndTraceShards(inputs map[string][]byte, -) ([]Trace[F], map[string][]byte, []error) { +) (trace.Trace[F], map[string][]byte, []error) { var ( strategy = p.config.shardingStrategy.Unwrap() // fast mode execution to generate checkpoints checkpoints, outputs, traceable, errors = BootAndCheckpoint(p.execution, inputs, strategy) // - traces = make([]Trace[F], len(checkpoints)) + shards = make([]Shard[F], len(checkpoints)) ) // Sanity check if traceable { @@ -92,9 +92,9 @@ func (p TraceBuilder[W, F, T]) bootAndTraceShards(inputs map[string][]byte, for i, ith := range results { errors = append(errors, ith.errors...) // Record trace - traces[i] = ith.trace + shards[i] = ith.trace // Record overall traceability - traceable = traceable && traces[i] != nil + traceable = traceable && !shards[i].IsEmpty() } } // @@ -102,7 +102,7 @@ func (p TraceBuilder[W, F, T]) bootAndTraceShards(inputs map[string][]byte, return nil, outputs, errors } // - return traces, outputs, errors + return shards, outputs, errors } func (p TraceBuilder[W, F, T]) traceCheckPoints(checkpoints []CheckPoint[W]) (jobs []traceJob[F]) { @@ -112,7 +112,7 @@ func (p TraceBuilder[W, F, T]) traceCheckPoints(checkpoints []CheckPoint[W]) (jo traceFn = func(i uint, cp CheckPoint[W]) traceJob[F] { var ( steps = strategy.shardSteps - trace Trace[F] + trace Shard[F] errs []error ) // Increment steps for all except first shard to account for the @@ -139,7 +139,7 @@ func (p TraceBuilder[W, F, T]) traceCheckPoints(checkpoints []CheckPoint[W]) (jo } type traceJob[F Element[F]] struct { - trace Trace[F] + trace Shard[F] errors []error }