Add --field-order option, add practical examples to README

Author: flwyd

README.md

@@ -118,6 +118,125 @@ adifmt fix log1.adi \
 creates a file named `minimal.csv` with just the date, time, and callsign from
 each record in the input file `log1.adi`.
 
+## Practical examples
+
+The following examples transform data into a format expected by a particular
+program or ham radio activity.  For details on how they work, see documentation
+for individual commands or formats below.  For contest log examples, see the
+[Cabrillo](#cabrillo) section.  Contributions of useful pipelines are welcome.
+
+### Add station and location to a log
+
+This example uses `edit` to add several fields to a POTA log saved in CSV
+format.  It then uses `fix` to remove the `:` from the time, transform the
+decimal (GPS) latitude and longitude to ADIF sexagesimal format and transforms
+`USA` and `CAN` country abbreviations to DXCC entity numbers.  `flatten` makes
+two copies of each record, one for park `US-0791` and one for park `US-4567`.
+`infer` then sets the band from the frequency, grid square (Maidenhead locator)
+based on the latitude and longitude, `STATION_CALLSIGN` field to the `OPERATOR`
+field, and `SIG` and `SIG_INFO` from the `POTA_REF` field.  (POTA doesn't
+require the country or latitude/longitude fields; they're included for
+illustration.) The input log might look like this:
+
+```csv
+TIME_ON,FREQ,MODE,CALL,STATE,COUNTRY
+12:34,7.012,CW,W1AW,CT,USA
+12:56,14.234,SSB,VA1XYZ,NS,CAN
+```
+
+```sh
+adiifmt edit mylog.csv \
+  --add qso_date=20240704 \
+  --add operator=WT0RJ \
+  --add my_pota_ref=US-0791,US-4567 \
+  --add my_state=DC --add my_country=USA \
+  --add my_lat=38.899736 --add my_lon=-77.063331 \
+| adifmt fix \
+| adifmt flatten --fields pota_ref,my_pota_ref \
+| adifmt infer --fields band,my_gridsquare,station_callsign
+```
+
+### ADIF to SOTA CSV
+
+This example uses `find` to filter out any records which don’t have `SOTA_REF`
+or `MY_SOTA_REF` fields, `edit` to add a `V2` field to each record (required by
+the SOTA uploader), `select` to output only the fields expected by the SOTA
+uploader and in the right order, `validate` to ensure fields are present and
+correctly formatted, and `save --csv-omit-header` to create a file with just
+the records, no file header.  If your log lacks frequencies, replace the `freq`
+field with `band`.  (Note that the SOTA uploader now accepts ADIF files, so you
+could just use the `find` command and upload directly.  This example may be
+useful if the data need to be further transformed or imported by a SOTA data
+analysis program.)
+
+```sh
+SOTA_CSV_ORDER=version,station_callsign,my_sota_ref,qso_date,time_on,freq,mode,call,sota_ref,comment
+adifmt find mylog.adi --if-not 'my_sota_ref=' --or-if-not 'sota_ref=' \
+| adifmt edit --set version=V2 \
+| adifmt select --fields $SOTA_CSV_ORDER \
+| adifmt validate --required-fields station_callsign,qso_date,time_on,freq,mode,call \
+| adifmt save --csv-omit-header --field-order $SOTA_CSV_ORDER sotalog.csv
+```
+
+The variable assignment syntax for `SOTA_CSV_ORDER` works on Mac and Linux.  On
+Windows PowerShell assign the variable as `$SOTA_CSV_ORDER =
+version,station_callsign,...`.  On Windows cmd.exe, assign it as
+`set SOTA_CSV_ORDER = version,station_callsign,...` and reference it as
+`%SOTA_CSV_ORDER%` rather than the `$` prefix.
+
+### Filter WARC bands
+
+This example uses `infer` to set the band from the frequency if the former
+isn’t set.  It then  uses `find` to filter out any contacts made on the
+[WARC bands](https://en.wikipedia.org/wiki/WARC_bands): 12, 17, 30, and 60
+meters.  Contesting is not allowed on those bands, so this is useful when
+preparing a contest submission from a general station log where contacts may
+have been made on bands not part of the contest.
+
+```sh
+adifmt infer --fields band mylog.adi \
+| adifmt find --if-not 'band=60m|30m|17m|12m'
+```
+
+### Set mode from frequency (U.S. band plan)
+
+This example sets the mode and submode based on the frequency, according to the
+U.S. band plan.  It assumes that CW and SSB are the only modes in use (no FM,
+AM, or digital), but can be extended if there are frequency ranges that you use
+exclusively for one mode.  `edit --add` will not overwrite the mode if it
+already has a value (`edit --set` would force the new value).
+
+```sh
+# US HF SSB (overlaps SSTV & AM) 80m: 3.6:4, 40m: 7.125:7.3, 20m:14.15:14.35,
+# 17m:18.11:18.168, 15m:21.2:21.45, 12m:24.93 to 24.99, 10m: 28.3:29
+# 6m 50.1:50.3 is CW/SSB, SSB calling 60.125, assume 60.120+ is SSB
+adifmt edit --if 'freq>3.6' --if 'freq<4' \
+  --or-if 'freq>7.125' --if 'freq<=7.3' \
+  --or-if 'freq>=14.15' --if 'freq<14.35' \
+  --or-if 'freq>=18.11' --if 'freq<18.168' \
+  --or-if 'freq>=21.2' --if 'freq<21.45' \
+  --or-if 'freq>=24.93' --if 'freq<24.99' \
+  --or-if 'freq>=28.3' --if 'freq<29' \
+  --or-if 'freq>=50.12' --if 'freq<50.3' \
+  --add mode=SSB |\
+
+# US HF CW (some digital could occur) low end of the band, below FT8 and friends
+adifmt edit --if 'freq>3.5' --if 'freq<3.7' \
+  --or-if 'freq>7' --if 'freq<7.07' \
+  --or-if 'freq>10.1' --if 'freq<10.3' \
+  --or-if 'freq>14' --if 'freq<14.07' \
+  --or-if 'freq>18.068' --if 'freq<18.1' \
+  --or-if 'freq>21' --if 'freq<21.07' \
+  --or-if 'freq>24.89' --if 'freq<14.91' \
+  --or-if 'freq>28' --if 'freq<28.07' \
+  --or-if 'freq>50.1' --if 'freq<50.12' \
+  --add mode=CW |\
+
+# SSB is usually LSB on 40m and below except 60m, USB on 20m and above
+adifmt edit --if mode=SSB --if 'freq<8' --if-not band=60m --add submode=LSB |\
+adifmt edit --if mode=SSB --if 'freq>=14' --or-if band=60m --add submode=USB
+```
+
 ## Features
 
 ### Input/Output formats
@@ -559,7 +678,7 @@ and a change:
 
 ```sh
 adifmt cat mylog.adi \
-  | adifmt edit --if 'mode=SSB' --if 'band>=20m' --add 'submode=USB' \
+  | adifmt edit --if 'mode=SSB' --if 'band>=20m' --or-if 'band=60m' --add 'submode=USB' \
   | adifmt edit --if 'mode=SSB' --if 'band=40m|80m|160m' --add 'submode=LSB' \
   | adifmt save fixed_sideband.adi
 ```
@@ -613,8 +732,8 @@ contacts on the border of a square as separate:
 
 ```sh
 adifmt flatten --fields VUCC_GRIDS --output tsv \
-  | adifmt select --fields VUCC_GRIDS --output tsv \
-  | tail +2 | sort | uniq -c
+  | adifmt select --fields VUCC_GRIDS --output tsv --tsv-omit-header \
+  | sort | uniq -c
 ```
 
 The `flatten` command will turn
@@ -762,8 +881,8 @@ find duplicate QSOs by date, band, and mode, use
 [uniq](https://man7.org/linux/man-pages/man1/uniq.1.html):
 
 ```sh
-adifmt select --fields call,qso_date,band,mode --output tsv mylog.adi \
-  | tail +2 | sort | uniq -d
+adifmt select --fields call,qso_date,band,mode --output tsv --tsv-omit-header mylog.adi \
+  | sort | uniq -d
 ```
 
 This is similar to a SQL `SELECT` clause, except it cannot (yet?) transform the
@@ -846,6 +965,7 @@ Features I plan to add:
     the same callsign on the same band with the same mode on the same Zulu day
     and the same `MY_SIG_INFO` value.
 *   Option for `save` to append records to an existing ADIF file.
+*   [FLE (fast log entry)](https://df3cb.com/fle/documentation/) format support.
 *   Count the total number of records or the number of distinct values of a
     field.  (The total number of records can currently be counted with
     `--output=tsv --tsv-omit-header` and piping the output to `wc -l`.)  This

adifmt/main.go

@@ -146,6 +146,7 @@ func buildContext(fs *flag.FlagSet, prepare func(l *adif.Logfile)) *cmd.Context
 
 	// General flags
 	fmtopts := "options: " + strings.Join(adif.FormatNames(), ", ")
+	fs.Var(&ctx.FieldOrder, "field-order", "Comma-separated `field` order for output (repeatable)")
 	fs.Var(&ctx.InputFormat, "input",
 		"input `format` when it cannot be inferred from file extension\n"+fmtopts)
 	fs.Var(&ctx.OutputFormat, "output",

adifmt/testdata/sota_csv.txtar

@@ -0,0 +1,33 @@
+# ADI to CSV with field order matching the SOTA uploader expectations.
+# This pipeline is an example in README.md
+exec adifmt find mylog.adi --if-not 'my_sota_ref=' --or-if-not 'sota_ref='
+! stderr .
+stdin stdout
+exec adifmt edit --set version=V2
+! stderr .
+stdin stdout
+exec adifmt select --fields version,station_callsign,my_sota_ref,qso_date,time_on,freq,mode,call,sota_ref,comment
+! stderr .
+stdin stdout
+exec adifmt validate --required-fields station_callsign,qso_date,time_on,freq,mode,call
+! stderr .
+stdin stdout
+exec adifmt save --csv-omit-header --field-order version,station_callsign,my_sota_ref,qso_date,time_on,freq,mode,call,sota_ref,comment sotalog.csv
+cmp sotalog.csv expected.csv
+! stdout .
+stderr 'Wrote 3 records to sotalog.csv'
+
+-- mylog.adi --
+<ADIF_VER:5>3.1.4 <CREATED_TIMESTAMP:15>23450607 080910 <PROGRAMID:6>adifmt <PROGRAMVERSION:7>(devel) <EOH>
+SOTA activator
+<QSO_DATE:8>20200101 <TIME_ON:4>0111 <MODE:2>FM <BAND:2>2m <FREQ:6>146.52 <CALL:3>K1A <STATE:2>CT <STATION_CALLSIGN:4>W1AW <MY_SOTA_REF:9>W1/MB-009 <MY_STATE:2>MA <COMMENT:24>Good signal, clear audio <EOR>
+Summit-to-summit
+<QSO_DATE:8>20210202 <TIME_ON:4>0222 <MODE:2>CW <FREQ:7>21.0123 <BAND:3>15m <CALL:3>W2B <STATE:2>CA <STATION_CALLSIGN:4>W1AW <RST_SENT:3>479 <RST_RCVD:3>559 <SOTA_REF:9>W6/SN-001 <MY_SOTA_REF:10>W4C/CM-009 <MY_STATE:2>NC <EOR>
+Not a SOTA contact
+<QSO_DATE:8>20220303 <TIME_ON:4>0333 <MODE:3>SSB <BAND:3>40m <FREQ:5>7.200 <CALL:3>K3C <STATE:2>PA <STATION_CALLSIGN:4>W1AW <MY_STATE:2>CT <EOR>
+SOTA chaser
+<QSO_DATE:8>20230404 <TIME_ON:4>0444 <MODE:3>FT8 <FREQ:6>14.074 <CALL:6>W4D/9H <STATION_CALLSIGN:4>W1AW <RST_SENT:3>-6 <RST_RCVD:3>-10 <SOTA_REF:9>9H/MA-001 <MY_STATE:2>CT <EOR>
+-- expected.csv --
+V2,W1AW,W1/MB-009,20200101,0111,146.52,FM,K1A,,"Good signal, clear audio"
+V2,W1AW,W4C/CM-009,20210202,0222,21.0123,CW,W2B,W6/SN-001,
+V2,W1AW,,20230404,0444,14.074,FT8,W4D/9H,9H/MA-001,

cmd/cat.go

@@ -14,15 +14,14 @@
 
 package cmd
 
-import (
-	"github.com/flwyd/adif-multitool/adif"
-)
-
 var Cat = Command{Name: "cat", Run: runCat,
 	Description: "Concatenate all input files to standard output"}
 
 func runCat(ctx *Context, args []string) error {
-	acc := accumulator{Out: adif.NewLogfile(), Ctx: ctx}
+	acc, err := newAccumulator(ctx)
+	if err != nil {
+		return err
+	}
 	for _, f := range filesOrStdin(args) {
 		l, err := acc.read(f)
 		if err != nil {

cmd/context.go

@@ -35,6 +35,7 @@ type Context struct {
 	Out                io.Writer
 	Locale             language.Tag
 	CommandCtx         any
+	FieldOrder         FieldList
 	UserdefFields      UserdefFieldList
 	SuppressAppHeaders bool
 	Prepare            func(*adif.Logfile)

cmd/edit.go

@@ -88,18 +88,20 @@ func runEdit(ctx *Context, args []string) error {
 	toTz := cctx.ToZone.Get()
 	adjustTz := fromTz.String() != toTz.String()
 	cond := cctx.Cond.Get()
-	out := adif.NewLogfile()
-	acc := accumulator{Out: out, Ctx: ctx}
+	acc, err := newAccumulator(ctx)
+	if err != nil {
+		return err
+	}
 	for _, f := range filesOrStdin(args) {
 		l, err := acc.read(f)
 		if err != nil {
 			return err
 		}
-		updateFieldOrder(out, l.FieldOrder)
+		updateFieldOrder(acc.Out, l.FieldOrder)
 		for _, r := range l.Records {
 			eval := recordEvalContext{record: r, lang: ctx.Locale}
 			if !cond.Evaluate(eval) {
-				out.AddRecord(r) // edit condition doesn't match, pass through
+				acc.Out.AddRecord(r) // edit condition doesn't match, pass through
 				continue
 			}
 			seen := make(map[string]string)
@@ -155,14 +157,14 @@ func runEdit(ctx *Context, args []string) error {
 						return fmt.Errorf("could not adjust time zone: %w", err)
 					}
 				}
-				out.AddRecord(rec)
+				acc.Out.AddRecord(rec)
 			}
 		}
 	}
 	if err := acc.prepare(); err != nil {
 		return err
 	}
-	return write(ctx, out)
+	return write(ctx, acc.Out)
 }
 
 func adjustTimeZone(r *adif.Record, from, to *time.Location) error {

cmd/find.go

@@ -14,10 +14,6 @@
 
 package cmd
 
-import (
-	"github.com/flwyd/adif-multitool/adif"
-)
-
 var Find = Command{Name: "find", Run: runFind, Help: helpFind,
 	Description: "Include only records matching a condition"}
 
@@ -56,23 +52,25 @@ Use quotes so operators are not treated as special shell characters:
 func runFind(ctx *Context, args []string) error {
 	cctx := ctx.CommandCtx.(*FindContext)
 	cond := cctx.Cond.Get()
-	out := adif.NewLogfile()
-	acc := accumulator{Out: out, Ctx: ctx}
+	acc, err := newAccumulator(ctx)
+	if err != nil {
+		return err
+	}
 	for _, f := range filesOrStdin(args) {
 		l, err := acc.read(f)
 		if err != nil {
 			return err
 		}
-		updateFieldOrder(out, l.FieldOrder)
+		updateFieldOrder(acc.Out, l.FieldOrder)
 		for _, r := range l.Records {
 			eval := recordEvalContext{record: r, lang: ctx.Locale}
 			if cond.Evaluate(eval) {
-				out.AddRecord(r)
+				acc.Out.AddRecord(r)
 			}
 		}
 	}
 	if err := acc.prepare(); err != nil {
 		return err
 	}
-	return write(ctx, out)
+	return write(ctx, acc.Out)
 }

cmd/fix.go

@@ -42,32 +42,33 @@ func helpFix() string {
 }
 
 func runFix(ctx *Context, args []string) error {
-	// TODO add any needed flags
-	out := adif.NewLogfile()
-	acc := accumulator{Out: out, Ctx: ctx}
+	acc, err := newAccumulator(ctx)
+	if err != nil {
+		return err
+	}
 	for _, f := range filesOrStdin(args) {
 		l, err := acc.read(f)
 		if err != nil {
 			return err
 		}
-		updateFieldOrder(out, l.FieldOrder)
+		updateFieldOrder(acc.Out, l.FieldOrder)
 		for _, rec := range l.Records {
-			out.AddRecord(fixRecord(rec, l))
+			acc.Out.AddRecord(fixRecord(rec, l))
 		}
 	}
 	if err := acc.prepare(); err != nil {
 		return err
 	}
 	// fix again in case userdef fields were added
-	for _, r := range out.Records {
+	for _, r := range acc.Out.Records {
 		for _, f := range r.Fields() {
-			ff := fixField(f, r, out)
+			ff := fixField(f, r, acc.Out)
 			if f != ff {
 				r.Set(ff)
 			}
 		}
 	}
-	return write(ctx, out)
+	return write(ctx, acc.Out)
 }
 
 func fixRecord(r *adif.Record, l *adif.Logfile) *adif.Record {

cmd/flatten.go

@@ -60,8 +60,10 @@ func runFlatten(ctx *Context, args []string) error {
 		delims[n] = d
 	}
 
-	out := adif.NewLogfile()
-	acc := accumulator{Out: out, Ctx: ctx}
+	acc, err := newAccumulator(ctx)
+	if err != nil {
+		return err
+	}
 	for _, f := range filesOrStdin(args) {
 		l, err := acc.read(f)
 		if err != nil {
@@ -90,14 +92,14 @@ func runFlatten(ctx *Context, args []string) error {
 				}
 			}
 			for _, e := range expn {
-				out.AddRecord(e)
+				acc.Out.AddRecord(e)
 			}
 		}
 	}
 	if err := acc.prepare(); err != nil {
 		return err
 	}
-	return write(ctx, out)
+	return write(ctx, acc.Out)
 }
 
 var typeDelims = map[spec.DataType]string{