Repository navigation
Expand file tree
/
Copy pathnextflow.config
More file actions
736 lines (690 loc) · 49.1 KB
/
Copy pathnextflow.config
File metadata and controls
736 lines (690 loc) · 49.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
nextflow.enable.dsl=2
params {
config = null // required: path to analysis CSV
data_dir = null // required: directory containing all FASTA files
project = null // optional: defaults to config CSV basename (see below)
outdir = "results"
// Docker / Singularity container image tag.
// Override with --container_version <tag> to use a different image.
container_version = '0.7.0'
run_tool = "phmmer" // phmmer | diamond | blast
evalue = 1e-5 // flat significance cutoff applied to every hit (not
// per-query paralog-derived -- see lib/singleton_presence.py)
diamond_sensitivity = 'very-sensitive' // main pairwise DIAMOND_SEARCH sensitivity mode: 'very-sensitive'
// (default since issue #171), '' (diamond's own fast mode), 'sensitive',
// 'more-sensitive' or 'ultra-sensitive'. Also applies to the
// novelty_discovery singleton searches and CONTEXT_DIAMOND_SEARCH
// (issue #166). DIAMOND_SELF is hardcoded --very-sensitive and is
// NOT affected by this. Benchmark (NovInvenio_Investigations
// notes/diamond-sensitivity/README.md, 3 clades): the share of
// candidates with an other-group TBLASTN hit fell from 29-69%
// (fast mode) to 9-33%, controls unchanged, search cost <=1.7x.
// Changing this changes every study's results on rerun.
parse_evalue = 0.01 // loose noise ceiling for parse_hits; final per-gene
// filtering applies the flat `evalue` cutoff above plus the
// paralog-competition filter in build_presence_matrix
ingroup_min_frac = 0.75 // fraction of ingroup that must contain a hit
outgroup_min_frac = 0.75 // fraction of outgroup that must contain a hit, for LOSS_SEARCH
loss_ingroup_max_frac = 0.0 // max fraction of ingroup a loss candidate may survive in
// (0.0 = strictly absent; raise for "nearly missing" losses)
core_min_frac = 0.95 // presence fraction (all proteomes) for the CORE genes report
paralog_competition_scope = 'target' // filter-2 granularity: 'target' (default -- only
// disqualify a hit the query's paralog beats on the SAME
// target protein; keeps HEX-1-like orthologs whose paralog
// wins on a different gene) or 'proteome' (strict; can
// drop real orthologs, see bin/build_presence_matrix.py)
paralog_rescue_evalue = 1e-20 // floor under filter 2, ON by default (issue #128):
// a hit is never disqualified by paralog-competition if
// the query's own e-value against that target is already
// <= this value. Set null to disable and restore the
// pre-2026-09-20 behaviour (baseline tag
// baseline/pre-paralog-rescue-2026-09-20). Without it,
// filter 2 is purely relative and erases arbitrarily
// strong orthologous hits -- see bin/build_presence_matrix.py's
// module docstring for the A7UWR3-vs-HEX1 case.
paralog_rescue_delta = null // second, OPT-IN rescue arm, OR-ed with the floor above:
// keep a hit the paralog beat by fewer than this many
// orders of magnitude. null (default) = disabled, and NOT
// recommended -- measured non-selective on real data
// (pezizo_set1). Kept so the rule can be re-swept once
// alignment coverage is available (issue #129).
other_coverage_floor_qcov = null // OPT-IN filter 3 (issues #158, #160): treat a pairwise hit to an
// OTHER-group proteome (outgroup for novelties, ingroup for
// losses) as absent when its query coverage < this percent.
// Never applied to query-group cells. null (default) = off;
// 15 is the chosen value when enabled (2026-09-23).
// Requires diamond or blast (wide hit output, issue #129);
// phmmer is rejected at startup. Measured on pezizo_set1: 20
// wrongly rejects 1.18% of BUSCO 1:1 ortholog pairs, 15
// rejects 0.38% -- see docs/superpowers/specs/
// 2026-09-22-coverage-floor-sensitivity-design-handoff.md.
// Writes <matrix>.coverage_floor_rejections.tsv.
use_orthofinder = false // use OrthoFinder instead of mmseqs2 for clustering
// --- Family-profile search pathway (ADR-0002) ---
// cluster_tool selects the novelty-direction presence-matrix producer:
// pairwise (default) — O(N^2) phmmer/diamond/blast SEARCH
// mmseqs — cluster ingroup into families → famsa → hmmbuild → hmmsearch
// novelty_discovery — two-phase targeted novelty pipeline (novelty_discovery → novelty_screen)
cluster_tool = "pairwise" // pairwise | mmseqs | novelty_discovery
// Family definition (mmseqs cluster of the ingroup). Shipped defaults; SWEEP these
// (family_min_seq_id 0.2–0.5, hmm_presence_* below) per ADR-0002 Q3/Q8.
family_sensitivity = 7 // mmseqs -s (cascaded prefilter sensitivity)
family_min_seq_id = 0.3 // mmseqs --min-seq-id
family_cov = 0.8 // mmseqs -c (coverage; guards domain over-merging)
family_cov_mode = 0 // mmseqs --cov-mode (0 = bidirectional)
family_cluster_mode = 0 // mmseqs --cluster-mode (0 = greedy set-cover)
family_min_members = 2 // profile only families with >= this many members
family_max_members = 100 // and drop families with MORE than this many members —
// an ancient multi-copy superfamily (e.g. aldehyde
// dehydrogenase) this large is present across most of
// the tree by construction, so it cannot pass the
// novelty/loss predicate either way; it's also the
// dominant famsa cost outlier (issue #22 follow-up).
// Skipped families are logged to oversized_families.tsv.
family_align_timeout = 600 // per-family safety net (seconds): if famsa/hmmbuild
// for one family exceeds this inside a BUILD_CHUNK task,
// skip just that family (logged) rather than losing the
// whole chunk's other families to a queue timeout.
family_chunk_size = 50 // families per BUILD_CHUNK task (scatter-gather of
// famsa+hmmbuild). Smaller = more parallelism / more
// jobs; sized down (was 200) because BUILD_CHUNK runs
// on the preemptible `preempt` queue and needs to
// finish quickly between preemptions.
// Family HMM search + presence rule (ADR-0002 Q5). Report loosely, filter strictly.
hmm_report_evalue = 10 // hmmsearch -E reporting ceiling (loose on purpose)
hmm_Z = null // hmmsearch -Z (fixed db size for comparable E-values)
hmm_presence_evalue = 1e-3 // presence: full-seq E-value ceiling (swept by
// bin/run_param_sweep.sh)
hmm_presence_cov = 0.3 // presence: min profile coverage fraction. Changed
// 0.5 -> 0.3 (2026-09-22) on an 8-point
// hmm_presence_cov {0.5,0.3,0.2,0.1} x
// hmm_presence_min_residues {0,100} sweep
// (pezizo5; job 28957241 + the original
// 2026-09-03 run): presence_recovery peaks at 0.3
// (0.9952) and does not improve below it (0.991-
// 0.993 at 0.2/0.1) -- 0.3 is the real optimum, not
// the edge of an incomplete grid. See
// todo/validate-hmm-presence-coverage-broader-
// sweep.md and .living/decisions.md for the full
// evidence and the earlier 83%-of-HMMs->600aa
// finding that motivated loosening this floor in
// the first place.
hmm_presence_min_residues = 100 // presence: alternative to hmm_presence_cov -- a
// target qualifies if its aligned span is at least
// this many residues, even below the coverage
// fraction above (0 = disabled, fraction-only).
// Targets the coverage floor's actual purpose
// (reject a hit explained by one small, promiscuous
// shared domain) without penalizing long, multi-
// domain proteins purely for being long. See
// lib/family_presence.py's module docstring.
hmm_presence_domain_evalue = 1e-5 // presence: a domain hit only counts toward
// hmm_presence_cov/hmm_presence_min_residues if ITS
// OWN independent E-value clears this (null =
// disabled, exact prior behaviour). Fixes a real
// false-positive case found 2026-09-10 via
// configs/controls/pezizo_set1.controls.csv
// (ada-1/ham-5): two individually-insignificant
// partial hits to an unrelated promiscuous domain
// (bZIP/WD40) merging past hmm_presence_min_residues
// with no real full-protein homology behind them.
// 1e-5 set as the shipped default 2026-09-12: a
// real --min-domain-evalue sweep against
// pezizo_set1_cluster (.living/decisions.md's
// 2026-09-12 entry) found 1e-5 is the exact
// breakpoint that fixes BOTH ada-1 and ham-5
// (recall 0.400 -> 0.800, fp_rate unchanged at
// 0.000), and it's the least-aggressive setting
// that does so -- tightening further adds no more
// recall on the resolved controls, only unvalidated
// extra candidates. Only one clade's controls so
// far -- see todo/validate-hmm-presence-coverage-
// broader-sweep.md for the still-open second-clade
// check -- and lib/family_presence.py's module
// docstring for the full mechanism.
hmm_search_chunk_size = 5000 // HMMs per FAMILY_HMMSEARCH chunk (scatter-gather of
// the family-db scan, issue #22). One (proteome x
// chunk) job per unit -- at 500, a 130-proteome
// deep_broad_1kfg-scale run needs ~27,000 of these
// jobs, and SLURM's submitRateLimit (50/min, see
// conf/ucr_hpcc_slurm.config) alone would take ~9h just
// to submit them, on top of per-job scheduling overhead
// and preempt-queue contention with everything else.
// Real trace data (sordariales_shallow, 14 proteomes)
// showed a 500-HMM chunk takes only ~1-5 min median --
// far below the ~1-1.5h/job sweet spot for this queue,
// so raised 10x to trade job COUNT for job SIZE without
// changing total compute (~27,000 -> ~2,700 jobs at
// roughly 25 min median each). This value was previously
// 2000, then cut to 500 specifically because
// FAMILY_HMMSEARCH runs on the preemptible `preempt`
// queue and needs to finish quickly between preemptions
// -- 5000 is being tried anyway because the per-family
// timing data above suggests even a 25 min task should
// still comfortably survive to a preemption checkpoint;
// watch actual FAILED/retry rates on the next big sweep
// and dial back if preemption losses get worse, not just
// job count.
// Interactive HTML report — which proteins carry an embedded sequence.
// Sequences dominate report.html's size, so "all" is opt-in.
report_sequences = "novelties" // novelties | all | none
// PDF summary report (view/<project>/summary.pdf). Gated by MAKE_PDF_REPORT's
// `when:` directive — set false to skip the step (e.g. no matplotlib available).
pdf_report = true
// Functional annotation databases (set to null to skip that annotation step)
pfam_hmm = null // path to Pfam-A.hmm (from db/pfam/)
build_uniprot_index = false // run the one-time UniProt library index build instead of an analysis
uniprot_index = null // UniProt library index dir (--build_uniprot_index output); enables UNIPROT_LINK
uniprot_library = null // pre-downloaded UniProt library (dir with data/<prefix>.dat.gz), for --build_uniprot_index
uniprot_library_csv = null // proteome CSV file name inside uniprot_library (proteome_id,tax_id,species_name,file_prefix,...)
uniprot_index_chunk_gb = 8 // compressed .dat.gz per UNIPROT_PARSE_CHUNK job (~48 min at 2.8 MB/s)
swissprot_dmnd = null // path to uniprot_sprot.fasta.dmnd (from db/uniprot/)
// Model organism gene name lookups — YAML config listing reference organisms
// See configs/modelorgs.yaml for format and examples
modelorgs_config = null
// MPI mode for hmmsearch (requires HMMER compiled with MPI, e.g. via conda-forge).
// Default true: benchmarked ~4-8x faster than thread mode for ANNOTATE_MATRIX's Pfam
// search on this cluster (860% CPU / 1m54s vs 94-165% CPU / 7-15min for a 500-protein
// subset vs the full Pfam-A DB), and verified end-to-end via a real SLURM run --
// requires SLURM_CPU_BIND=none (exported in-script; --cpu-bind is srun-only, sbatch
// rejects it) to avoid "CPU binding outside of job step allocation" on this cluster.
// Only affects processes carrying label 'hmmsearch' that actually branch on this flag
// (ANNOTATE_MATRIX, modules/hmmsearch.nf); FAMILY_HMMSEARCH's HMMSEARCH_CHUNK never
// runs --mpi and has its own fixed cpus override in conf/ucr_hpcc_slurm.config so it's
// unaffected by this default.
// When true, runs: mpirun -np <hmm_mpi_tasks> hmmsearch --mpi ...
// When false, runs: hmmsearch --cpu <task.cpus> ...
hmm_mpi = true
hmm_mpi_tasks = null // MPI task count; defaults to max_cpus when hmm_mpi=true
max_cpus = 32
max_memory = '64.GB'
max_time = '24.h'
// --- Pangenome cluster-profiling subworkflow (pangenome.nf / workflows/pangenome_profile.nf) ---
// Ported from a one-off study script chain (NovInvenio_Investigations'
// Afumigatus_pangenome study) into a reusable subworkflow; see that
// study's NEXTFLOW_MIGRATION_NOTES.md for the design record. Entirely
// separate param namespace (pangenome_*) from the novelty/loss params
// above -- this is a structurally different analysis (whole-species-set
// clustering + presence/frequency/co-occurrence), invoked via its own
// entry point (pangenome.nf), not main.nf.
pangenome_samplesheet = null // required: GROUP,Species,Strain,Protein,DNA,GFF3,Short,TaxonGroup CSV
pangenome_data_dir = null // required: directory containing pep/, dna/, gff3/ subdirs
pangenome_gff3_dir = null // optional override; defaults to "${pangenome_data_dir}/gff3"
pangenome_project = null // optional: outdir/storeDir namespace; defaults to the
// samplesheet basename if unset (see pangenome.nf) so two
// different species' runs never collide on the same 'output'
// directory (Helpers.projectName's fallback literal).
help = false // print usage and exit (pangenome.nf; see --help)
// Species tree (phyling + FastTree) in the report; docs/superpowers/specs/2026-10-17-species-tree-design.md
species_tree = true
species_tree_markers = 10 // top N markers by treeness/RCV (phyling filter -n)
species_tree_markerset = 'fungi_odb12' // subdirectory of phyling_markerset_dir (holds hmms/)
species_tree_tip_labels = 'species' // SVG tip text: 'species' (samplesheet Species) or 'short' (Short ID)
species_tree_container_version = '2.5.0' // tag of ghcr.io/stajichlab/phyling used by SPECIES_TREE (not the main image)
phyling_markerset_dir = null // e.g. /bigdata/stajichlab/jstajich/.phyling; null = tree is skipped with a failed status
pangenome_ingroup_label = 'IN' // samplesheet GROUP value for the ingroup
pangenome_outgroup_label = 'OUT' // samplesheet GROUP value for the outgroup
pangenome_outgroup_min_bin_strains = 3 // bin the outgroup (frequency_table *_out columns) only with >= this many representatives (#212)
pangenome_id_sep = '|' // Short-prefix separator in clustering-input FASTA headers
// Tier-1 clustering (the allele/ortholog unit every downstream step relies on).
pangenome_cluster_backend = 'mmseqs' // mmseqs | diamond
pangenome_tier1_min_id = 0.9 // mmseqs --min-seq-id fraction (or diamond --approx-id/100)
pangenome_tier1_cov = 0.8 // mmseqs -c fraction (or diamond --member-cover/100)
// Genome-level tblastn rescue pass (per-strain scatter -- one BLAST database
// per strain, restricted to that strain's own currently-ABSENT family
// queries; see modules/pangenome/rescue.nf's module docstring for why this
// is NOT a shared combined-genome database -- that design had a real,
// documented correctness bug: a shared --max_target_seqs silently
// truncated hits ACROSS strains per query, not per strain).
pangenome_rescue_enable = true
pangenome_rescue_evalue = 1e-10
pangenome_rescue_max_target_seqs = 200 // per-strain DB is tiny; generous headroom over
// any realistic intra-genome copy number
// `rescue_min_pident` should track whatever identity threshold tier-1
// clustering itself used (pangenome_tier1_min_id, as a percentage) --
// otherwise the two stages can disagree about what counts as "the same
// gene". Defaults independently to 90.0 (matching this subworkflow's own
// pangenome_tier1_min_id default of 0.9); override BOTH together if you
// change one.
pangenome_rescue_min_pident = 90.0
pangenome_rescue_min_qcov = 80.0
// Structural rescue filter (issue #133). Identity/coverage thresholds
// alone cannot separate a genuine annotation dropout from the same
// locus counted twice: an exhaustive characterisation of all
// 12,877,646 rescuable cells in a real 529-strain run found 77.6% of
// them land on a predicted gene already assigned to a DIFFERENT
// family in that strain (the query family's own protein model is
// fragmented/redundant with an existing gene, not genuinely missing),
// and the identity/coverage class mix stayed flat (19-23% intergenic)
// across the ENTIRE (min_pident 90-99.9, min_qcov 80-99) grid -- these
// are 100%-identical real DNA, so no threshold combination fixes it.
// Enabled by default alongside pangenome_rescue_enable; only takes
// effect when both are true.
pangenome_rescue_structural_filter = true
// Reject a hit whose query family rep is under this many aa (83.2% of
// the remaining intergenic false-positive rescues came from reps
// under this length; median real family rep is ~201 aa).
pangenome_rescue_min_rep_length = 150
// Reject a hit if its genomic window (this many bp) is also hit by
// pangenome_rescue_hotspot_min_families or more distinct families --
// a repeat/mobile-element region producing many spurious hits, not
// that many independent real gene losses (47.5% of the remaining
// intergenic cells).
pangenome_rescue_hotspot_window = 2000
pangenome_rescue_hotspot_min_families = 3
// Pipeline diagnostics (issue #134). Advisory by default: a triggered
// diagnostic (e.g. rescue_redundancy, issue #133) is surfaced in
// diagnostics.tsv and at the top of report.md/island_synteny.html, but
// never fails the run. Set true to promote any triggered diagnostic to
// a run failure -- e.g. a publication run that wants clean diagnostics
// enforced. The zero-hit rescue guard (issue #126) always fails
// regardless of this flag; that is enforced inside
// pangenome_rescue_pass.py itself, upstream of DIAGNOSTICS.
pangenome_strict = false
pangenome_rescue_redundancy_threshold = 0.5
// Strain dedup + clade assignment (Mash sketch/distance -> PCoA -> k-means).
// See workflows/pangenome_profile.nf's header comment for why this runs as
// two separate MASH_SKETCH invocations (all-strains for dedup, ingroup-only
// for clades) rather than one shared sketch.
pangenome_dereplicate = true // strain-inventory dedup (frequency/co-occurrence
// denominator restricted to is_representative==1)
pangenome_mash_threshold = 0.001 // mash distance below which two strains dedup-group
pangenome_assign_clades = true // fill empty samplesheet TaxonGroup cells from a
// Mash+PCoA+k-means clade label (never overwrites an
// existing value -- see FILL_TAXON_GROUP)
pangenome_stratum_source = 'taxon_group' // 'taxon_group': the co-occurrence null shuffles within TaxonGroup (earlier behaviour).
// 'mash_clade': it shuffles within the Mash clade of each ingroup strain (written to a Stratum column; TaxonGroup is left as the display label, e.g. the species). Needs pangenome_assign_clades. 2026-10-08
pangenome_clades_k_range = '2,10' // silhouette-sweep k range (ignored if k_fixed set)
pangenome_clades_k_fixed = null // skip the silhouette sweep, cluster at exactly this k
pangenome_clades_n_pcoa = 4 // PCoA components fed into k-means
// Frequency binning cutoffs (fraction of ingroup strains carrying a family).
pangenome_core_cutoff = 0.95
pangenome_softcore_cutoff = 0.90
pangenome_shell_cutoff = 0.15
// Co-occurrence screen (Fisher/BH-FDR between shell+cloud families).
pangenome_min_strain_count = 5 // a family needs at least this many carriers to be tested
pangenome_fdr_alpha = 0.05
pangenome_screen_alpha = 0.2 // analytic-prefilter threshold; must stay looser than fdr_alpha
// (pangenome_cooccurrence.py clamps this automatically)
// Outgroup-frequency polarity (direction_a_freq, reported alongside the strict
// all-or-none direction_a). See docs/pangenome-assumptions.md section 5.
pangenome_polarity_loss_min_frac = 0.9 // loss when >= this fraction of outgroup strains carry it
pangenome_polarity_gain_max_frac = 0.1 // gain when <= this fraction do (not calibrated)
// Rooted Newick strain tree (tips = strain Short IDs, ingroup + outgroup) for
// Dollo-parsimony gain/loss polarization in COOCCURRENCE (issue #185; method #140).
// null = off: no tree columns, count-based direction_a only. Must cover every
// matrix strain (hard error otherwise); extra tips are pruned. Nodes with >2
// children are read as soft polytomies. Build it per study from markers.
pangenome_species_tree = null
// Optional captain/mobile-element marker gene (e.g. the fungal Starship
// DUF3435 captain gene) used as physical-linkage evidence in pair
// classification. Deliberately optional and NOT fungal/Starship-specific
// in its mechanics -- omit both to skip captain-gene evidence entirely
// (every physically-linked pair then classifies as unexplained_physical
// rather than starship_explained).
pangenome_captain_hmm = null // path to a single-profile HMM file, used as-is
pangenome_captain_hmm_name = null // OR: hmmfetch this NAME (not accession) out of...
pangenome_pfam_hmm = null // ...this larger Pfam-style HMM database
pangenome_captain_hmm_evalue = 1e-3 // hmmsearch -E for the captain-gene scan
// Optional overlap of PRECOMPUTED Starship element calls and secondary-metabolite
// (SM) cluster calls with the islands (STARSHIP_OVERLAP / SM_OVERLAP,
// modules/pangenome/starship_overlap.nf). null = off; each table runs its part
// alone. TSV (CSV if the name ends in .csv; .gz read). Schemas (lib/starship_tables.py,
// paper/methods_pangenome.md 3.25): elements = element_id, strain (samplesheet
// Short), contig, start, end (1-based, inclusive) [, label, confidence (high = high)],
// or gene-level strain, gene_id, element_id; SM clusters = cluster_id, strain,
// contig, start, end, type [, genes, core_genes (comma-separated protein IDs)].
// Checked at workflow start against the samplesheet. Captain hits (when a captain
// HMM is set) count at full-sequence E <= 1e-5 in these two steps.
pangenome_starship_annotation = null
pangenome_sm_clusters = null
pangenome_starship_permutations = 1000 // element-shift permutations (coord_permutation.tsv, sm_permutation.tsv)
pangenome_starship_seed = 20261017 // RNG seed of those permutations (the NII analysis value)
// Pair classification (physical linkage / captain-gene mechanism between
// co-occurring family pairs). See bin/pangenome_pair_classification.py's
// module docstring for the full label taxonomy and documented
// partial-implementation caveats (rank-window-only linkage, no
// taxon_scheme restriction on min_clades).
pangenome_pair_class_k = 10 // gene-count window for "physically linked"
pangenome_pair_class_physical_threshold = 0.5
pangenome_pair_class_trans_threshold = 0.05
pangenome_pair_class_min_co_carrying = 5
pangenome_pair_class_perm_alpha = 0.05
pangenome_pair_class_min_clades = 2
pangenome_pair_class_perm_correction = 'none' // 'none' | 'bh': gate `trans` on permutation_p (none, the earlier behaviour) or on its Benjamini-Hochberg q over the exact-tested pairs (bh). Default off (2026-10-08).
pangenome_permutation_p_format = 'sci' // 'sci' (4 significant figures) | 'fixed4' (legacy 4 decimals)
pangenome_pair_class_evidence_strains = 'representatives' // 'representatives' | 'all': which strains' positions supply n_co_carrying and linkage_fraction (2026-10-08; 'all' = earlier behaviour)
// Accessory-island construction + Pfam functional enrichment (optional).
// pangenome_island_pfam_hmm is a NEW, distinct param from the
// pre-existing pangenome_pfam_hmm above (used only by the
// captain-by-name branch) -- setting this one gates the whole new
// island+Pfam step; the pre-existing one is untouched.
pangenome_island_pfam_hmm = null // enables the whole island+Pfam step when set (distinct from the existing pangenome_pfam_hmm, used only by the captain-by-name branch)
pangenome_island_min_size = 2
pangenome_pfam_domain_evalue = 1e-3
// Query sequences per FAMILY_PFAM_SCAN chunk (issue #112). As one job,
// that scan measured 1h50m-3h50m on the 529-strain genus_vs_ureesii
// study (22,061 background reps vs 30,134 Pfam-A profiles) and twice
// FAILED at 1h59m against a 2-hour wall-clock cap, losing the whole
// scan. 3000 gives ~8 chunks at that scale, which lands each chunk
// near the ~1-1.5 h per unit of work the HPCC sizing guidance asks for
// -- large enough that per-job scheduling overhead stays negligible,
// small enough to clear the cap. Raise it for a small study rather
// than paying submission overhead on chunks that finish in seconds.
other_signal_qcov = 50 // % query coverage at which an other-group hit is "broad" (issue #208; observed median, not validated)
pangenome_pfam_chunk_size = 3000
pangenome_marker_names = '' // comma list, e.g. 'captain,sm_backbone'
pangenome_marker_hmm_paths = '' // parallel comma list of HMM paths
pangenome_marker_evalue = 1e-5
pangenome_pfam2go = null // path to a local pfam2go mapping file (see
// pangenome.nf --help for where to obtain it);
// GO-term annotation skipped entirely when unset
pangenome_accumulation_permutations = 20
pangenome_accumulation_seed = 0
// Leiden trans-module detection (chained co-occurring gene families) --
// runs unconditionally after PAIR_CLASSIFICATION, no gating param of its
// own; degrades gracefully to empty output for a study with zero `trans`
// pairs. No published pangenome-specific precedent exists for the
// resolution parameter at genome scale (see
// bin/pangenome_detect_trans_modules.py's module docstring) -- the
// default below is not validated for every study; expect to tune it
// empirically per study, e.g. via a small resolution sweep.
pangenome_leiden_resolution = 1.0
pangenome_leiden_seed = 0
pangenome_module_min_size = 2 // MODULE_DOMAINS: minimum module size to summarize
// View B1 (issue #182): per-strain genomic clustering of each trans module's
// members against a within-strain permutation null. 100 kb / 11 genes are the
// spec's provisional defaults; the 2026-09-25 empirical pass on the 529-strain
// Coccidioides run (25/50/100/200 kb x accessory/all null) gave the same module
// calls at every scale, so they stand. min_gene_gap 11 = strictly outside the
// k=10 window that defines `trans` (not swept). The accessory null is the
// default: the all-genes null gave a spurious 1.2-1.3x on non-clustered modules
// at 25 kb. See lib/pangenome_neighborhood.py.
pangenome_neighborhood_max_kb = 100
pangenome_neighborhood_min_gene_gap = 11
pangenome_neighborhood_n_perm = 200
pangenome_neighborhood_seed = 0
pangenome_neighborhood_null_pool = 'accessory' // 'accessory' | 'all'
// Assembly-quality vs pangenome-content QC (issue #130). Runs
// unconditionally (cheap; no gating param) so the confound documented
// there -- fragmented assemblies manufacture spurious accessory/private
// gene families -- is visible on every run rather than requiring a
// one-off manual analysis to rediscover it. Diagnostic only: never
// excludes a strain or alters a presence call.
pangenome_qc_terminus_window_bp = 1000 // "near a contig end" window (bp) for the
// private-family mechanism check
pangenome_qc_rho_warn_threshold = 0.3 // |rho| above this (accessory-family content
// vs assembly quality) triggers a WARNING
// report.md's "Top islands (by size)" table: minimum carrying strains.
// Measured on the 529-strain genus_vs_ureesii run (2026-09-18), 17,218 of
// 27,836 located islands (62%) are present in a single strain, and all 20
// of the largest were single-strain -- so an unfiltered size ranking shows
// only strain-private content (often an assembly/annotation artefact, e.g.
// a 51 kb, 69-gene-model "island" on one contig of one strain). Set to 1
// to disable the filter.
pangenome_top_islands_min_strains = 2
// Islands drawn in the island synteny view (issue #116). Selection is
// top-N by size AFTER the pangenome_top_islands_min_strains filter, so
// the filter never costs slots. 50 keeps the embedded payload small --
// 530 strains x a 56-gene island is ~30 kB raw per island, so ~100
// islands still fits under 1 MB.
pangenome_viz_top_islands = 50
// Island locus view (docs/superpowers/specs/2026-09-24-island-locus-view-design.md,
// ISLAND_LOCI). Defaults are the spec's, taken from the 2026-09-24
// feasibility run and not swept (spec open question 1).
pangenome_locus_flank = 5 // F: flank genes per side
pangenome_locus_flank_min = 3 // F_min: fallback when no carrier has F per side
pangenome_locus_k = 10 // k: gene window of the "in place" test
pangenome_locus_empty_frac = 0.8 // empty site: >= this fraction of locus columns absent
pangenome_island_reference_strains = '' // ordered Short IDs: the display strain of an island or locus is the first listed carrier (names, coordinates, labels); empty = the existing rule
pangenome_locus_min_column_strains = 2 // a locus column classifies strains only if its gene is in place in >= this many strains (1 = earlier behaviour; 2026-10-08)
pangenome_locus_model_diff_min_frac = 0.5 // DNA-present gap = 'model difference' only if a gapped gene is in place in >= this fraction of strains, else 'variable-gene gap' (0 = one class; 2026-10-08)
pangenome_locus_containment = 0.5 // an island joins a larger island's locus at >= this
pangenome_locus_rank = 'presence' // presence (default) | within | species |
// whole_dna | whole_annot | informative
// (= presence) | strains | size
pangenome_top_loci = 100 // loci drawn in island_synteny.html (union of
// per-rank winners + a presence-order fill)
// DNA presence check (spec section 4b; ISLAND_DNA_TARGETS + ISLAND_DNA_CHECK).
// min_id / min_cov are the spec's chosen, not validated, defaults.
pangenome_locus_dna_check = true // blastn the locus DNA at each "empty" site
pangenome_locus_dna_min_id = 90 // HSP identity, percent
pangenome_locus_dna_min_cov = 80 // covered share of an exemplar gene, percent
pangenome_locus_dna_batch = 50 // loci per ISLAND_DNA_CHECK task
pangenome_locus_candidates = 200 // loci whose states are computed before ranking
pangenome_locus_candidate_strata = '' // carrier-fraction cut points, e.g. '0.05,0.2,0.5,0.8': candidates spread over the bins; '' keeps the highest-carrier loci
// Multi-rank locus view (ranks-brief.md, approved 2026-09-27, spec 4b/6
// "Ranking"): five rankings (presence [default], within, species,
// whole_dna, whole_annot) replace the single informative ranking; the
// drawn set is their union, spec docs/superpowers/specs/2026-09-24-island-locus-view-design.md.
pangenome_locus_per_rank = 20 // loci drawn per non-presence rank before the fill
pangenome_locus_poly_min_frac = 0.05 // rank "within": minimum loss fraction in a species
pangenome_locus_poly_max_frac = 0.95 // rank "within": maximum loss fraction in a species
pangenome_locus_poly_min_strains = 20 // rank "within": minimum strains of a species
pangenome_locus_fixed_diff = 0.95 // rank "species": minimum loss-fraction difference
// Clinker synteny panel (spec section 8; ISLAND_GBK_SLICE + ISLAND_CLINKER).
pangenome_clinker = true
pangenome_clinker_max_strains = 12 // strains per clinker figure (<= 66 pairs)
pangenome_clinker_slim = true // drop embedded sequences from the clinker HTML
pangenome_clinker_batch = 50 // loci per ISLAND_CLINKER task
// Gene-free gap (bp) that splits a strain's region into several clinker
// loci of the same cluster, so a huge mostly-empty region is not drawn
// to one unreadable one-bp scale (spec section 8 C1, chosen from real
// Cocci evidence: a 338,667 bp region with a 164 kb gene-free gap).
// 0 = no splitting.
pangenome_clinker_max_gap = 20000
}
process {
cpus = { Math.min(8, params.max_cpus as int) }
memory = '8.GB'
time = '2.h'
// Container image (ghcr.io/stajichlab/novinvenio) for all processes.
// Each process file carries its own `container` directive that references
// this variable, so the image tag can be overridden with --container_version
// without editing any .nf files. A plain string, not a closure: a closure here is
// evaluated at session init, before params are bound, and logs an "undefined
// parameter container_version" WARN (#169). CLI overrides still apply.
container = "ghcr.io/stajichlab/novinvenio:${params.container_version}"
// Non-container fallback: when neither -profile docker nor -profile singularity is
// active, `container` above is set but never used, so tasks need their own
// environment. Activate the pixi-managed conda environment and symlink db/ + configs/
// so relative DB paths (e.g. db/pfam/38.2/Pfam-A.hmm) resolve from the task's work dir.
// Runs before every task script, inside the task's work directory. Skipped entirely
// under -profile docker/singularity, where workflow.containerEngine is set and the
// container image already provides every tool on PATH.
beforeScript = {
workflow.containerEngine ? '' : """
export PATH="\$HOME/.pixi/bin:\$PATH"
eval "\$(pixi shell-hook -s bash --manifest-path ${projectDir}/pixi.toml)"
[ -e db ] || ln -s ${projectDir}/db db
[ -e configs ] || ln -s ${projectDir}/configs configs
""".stripIndent()
}
withLabel: 'high_cpu' {
cpus = { Math.min(8, params.max_cpus as int) }
memory = '16.GB'
}
withLabel: 'med_cpu' {
cpus = { Math.min(8, params.max_cpus as int) }
memory = '8.GB'
}
withLabel: 'hmmsearch' {
// Non-MPI default: multithreaded hmmsearch. Memory scales with task.attempt
// (8/16/24 GB) rather than a fixed value: a HMMSEARCH_CHUNK's memory footprint
// depends on which family HMMs happened to land in that chunk, and an mmseqs
// family-clustering over-merge can occasionally produce a pathologically large
// family HMM (observed LENG up to 13,626 in one chunk during the 2026-09-03
// broader-grid sweep, OOM-killed at a flat 8GB on every attempt since the same
// chunk composition is reused via -resume across grid points) -- retrying with
// more memory, not just retrying, actually recovers from that case. See
// .living/decisions.md #14, #16.
cpus = 4
memory = { 8.GB * task.attempt }
}
withLabel: 'low_cpu' {
cpus = 1
memory = '4.GB'
}
// ASSEMBLY_QUALITY_QC (#130) loads the rescued presence matrix and the
// gene positions whole. At 529 strains it peaked at 5.4 GB MaxRSS (hand
// run, NII job 29044463) and was OOM-killed (exit 137) at low_cpu's 4 GB
// on all 3 attempts in a real run (2026-09-24).
withName: '.*ASSEMBLY_QUALITY_QC' {
memory = { 16.GB * task.attempt }
}
// BUILD_PRESENCE_MATRIX (#189) only had `label 'low_cpu'`'s flat 4 GB, with
// no scaling across retries -- so a study large enough to OOM it (matrix
// construction materialises the whole hits/paralog-cutoff input) hit the
// identical 4 GB wall on every attempt and failed the whole run instead of
// recovering on retry, the same failure mode already fixed for
// ASSEMBLY_QUALITY_QC and HMMSEARCH_CHUNK above. errorStrategy/maxRetries
// (conf/ucr_hpcc_slurm.config: `errorStrategy = { task.attempt < 3 ? 'retry'
// : 'finish' }`, `maxRetries = 2`) already retries unconditionally on any
// non-zero exit, including OOM's exit 137, so scaling memory here is enough
// to let a retry actually succeed instead of repeating the same failure.
withName: '.*BUILD_PRESENCE_MATRIX' {
memory = { 16.GB * task.attempt }
}
// No bespoke withName block for ISLAND_SYNTENY (issue #118 follow-up):
// the 12 GB, then 5 GB, directives that used to live here were sized
// purely to accommodate PresenceMatrix.from_tsv() and
// bin/pangenome_island_synteny.py's load_positions() each materialising
// their WHOLE input regardless of how few islands were actually drawn
// (54,421 families x 530 strains for the presence matrix; 4.5M
// (strain, family, copy) rows for family_positions.tsv, on the real
// 529-strain genus_vs_ureesii study). Both now take the same
// `families=` filter -- the union of `member_families` across the
// islands select_islands() actually selects (298 of 54,421 on that
// study) -- computed once in the CLI and passed to both loaders, so
// neither ever materialises more than what gets drawn. Re-measured on
// the same real study/render (same 50 islands drawn, 17,064 excluded,
// 10,552 truncated -- output unchanged, only memory changed): peak RSS
// dropped to 172 MB, comfortably inside low_cpu's 4 GB (>20x headroom),
// so the process just inherits `label 'low_cpu'` (modules/pangenome/
// island_synteny.nf) like everything else in that tier -- a bespoke
// override here would no longer earn its place.
}
// report/trace/timeline all land under ${outdir}/${project}/nextflow_log/, next to
// this run's actual results -- not ${launchDir}/logs/, which is only the directory
// `nextflow run` happened to be launched from (a throwaway .nf_launch/<study>/ dir
// for NII's own bin/run_study.sh) and has no relationship to where results publish.
// Filenames carry a per-invocation timestamp (params._run_stamp, computed once
// below -- not a user-facing flag) so re-running the same project (e.g. -resume
// after a fix) keeps every past run's report/trace/timeline instead of
// overwriting the last one. A plain `def` can't sit alongside config blocks here
// (stricter than plain Groovy), so the timestamp is threaded through via params.
params {
_run_stamp = new Date().format('yyyyMMdd_HHmmss')
}
report {
enabled = true
file = "${params.outdir}/${params.project ?: 'run'}/nextflow_log/${params.project ?: 'run'}-${params._run_stamp}-report.html"
overwrite = true
}
trace {
enabled = true
file = "${params.outdir}/${params.project ?: 'run'}/nextflow_log/${params.project ?: 'run'}-${params._run_stamp}-trace.txt"
overwrite = true
}
timeline {
enabled = true
file = "${params.outdir}/${params.project ?: 'run'}/nextflow_log/${params.project ?: 'run'}-${params._run_stamp}-timeline.html"
overwrite = true
}
profiles {
// -------------------------------------------------------------------------
// SLURM on the UCR HPCC — see conf/ucr_hpcc_slurm.config for the full
// override set (queue routing, AVX2 node constraints, preempt queue).
//
// Usage (pixi environment on the HPCC):
// nextflow run stajichlab/NovInvenio \
// -profile slurm -c conf/ucr_hpcc_slurm.config \
// --config configs/pezizo4_asco.csv --data_dir /path/to/fastas ...
//
// Fully containerised HPC run:
// nextflow run stajichlab/NovInvenio \
// -profile slurm,docker -c conf/ucr_hpcc_slurm.config \
// --config configs/pezizo4_asco.csv --data_dir /path/to/fastas ...
// -------------------------------------------------------------------------
slurm {
process.executor = 'slurm'
executor {
queueSize = 100
submitRateLimit = '50/1min'
// See conf/ucr_hpcc_slurm.config's executor{} block for why this is
// raised from Nextflow's 270sec default.
exitReadTimeout = '900sec'
// See conf/ucr_hpcc_slurm.config's executor{} block for why this is
// set (cluster-wide queue visibility, not just this pipeline's jobs).
queueGlobalStatus = true
}
}
// -------------------------------------------------------------------------
// Docker container — all tools run inside ghcr.io/stajichlab/novinvenio:<tag>.
// db/, configs/, and the pipeline source (bin/, lib/, modules/) are
// bind-mounted by Nextflow automatically (autoMounts = true).
//
// User-provided paths (--data_dir, --config, --pfam_hmm, --swissprot_dmnd,
// --modelorgs_config) must be absolute host paths so they resolve inside
// the container.
//
// Usage:
// nextflow run stajichlab/NovInvenio \
// -profile docker \
// --config /absolute/path/to/config.csv \
// --data_dir /absolute/path/to/fastas \
// --pfam_hmm /absolute/path/to/Pfam-A.hmm \
// --swissprot_dmnd /absolute/path/to/uniprot_sprot.fasta.dmnd \
// --modelorgs_config /absolute/path/to/modelorgs.yaml
//
// Override the image tag (e.g. to use a local build or :latest):
// nextflow run stajichlab/NovInvenio -profile docker \
// --container_version 0.7.0 ...
//
// Build the image locally:
// docker build -t ghcr.io/stajichlab/novinvenio:0.7.0 .
// docker push ghcr.io/stajichlab/novinvenio:0.7.0
// -------------------------------------------------------------------------
docker {
docker.enabled = true
docker.registry = 'ghcr.io/stajichlab'
docker.autoMounts = true
}
// -------------------------------------------------------------------------
// Singularity / Apptainer — same image as Docker, in SIF format.
//
// Build the SIF:
// singularity build novenio.sif docker://ghcr.io/stajichlab/novinvenio:0.7.0
//
// Usage (standalone — local executor, no SLURM):
// nextflow run stajichlab/NovInvenio \
// -profile singularity \
// --config /absolute/path/to/config.csv \
// --data_dir /absolute/path/to/fastas \
// ...
//
// UCR HPCC: combine the SLURM site config with Singularity:
// nextflow run stajichlab/NovInvenio \
// -profile slurm,singularity \
// -c conf/ucr_hpcc_slurm.config \
// --config configs/pezizo4_asco.csv --data_dir /path/to/fastas ...
//
// The HPCC Docker/Singularity setup must expose bind-mounts for the
// host working directory and project directory. Consult the HPCC
// documentation for the correct --bind / SINGULARITY_BINDPATH settings.
// -------------------------------------------------------------------------
singularity {
singularity.enabled = true
singularity.autoMounts = true
}
local {
process.executor = 'local'
}
test {
params.config = "${projectDir}/tests/data/test.csv"
params.data_dir = "${projectDir}/tests/data"
params.project = "test"
params.run_tool = "phmmer"
process.executor = 'local'
}
}