preserve input star columns during particle export

WarpLib/WorkerWrapper.cs

@@ -463,15 +463,16 @@ public void TomoExportParticleSubtomos(string path, ProcessingOptionsTomoSubReco
                                                     angles));
         }
 
-        public void TomoExportParticleSeries(string path, ProcessingOptionsTomoSubReconstruction options, float3[] coordinates, float3[] angles, string pathsRelativeTo, string pathTableOut)
+        public void TomoExportParticleSeries(string path, ProcessingOptionsTomoSubReconstruction options, float3[] coordinates, float3[] angles, string pathsRelativeTo, string pathTableOut, Dictionary<string, string[]> additionalColumns)
         {
             SendCommand(new NamedSerializableObject("TomoExportParticleSeries",
                                                     path,
                                                     options,
                                                     coordinates,
                                                     angles,
                                                     pathsRelativeTo,
-                                                    pathTableOut));
+                                                    pathTableOut,
+                                                    additionalColumns));
         }
 
         public void MPAPrepareSpecies(string path, string stagingSave)

WarpTools/Commands/Tiltseries/ExportParticlesTiltseries.cs

@@ -102,6 +102,8 @@ class ExportParticlesTiltseriesOptions : DistributedOptions
 
     class ExportParticlesTiltseries : BaseCommand
     {
+        private Dictionary<string, string[]> _additionalColumns;
+
         public override async Task Run(object options)
         {
             await base.Run(options);
@@ -167,6 +169,26 @@ public override async Task Run(object options)
             }
 
             ValidateInputStar(inputStar);
+            _additionalColumns = new Dictionary<string, string[]>();
+            var knownColumns = new HashSet<string>
+            {
+                "rlnCoordinateX", "rlnCoordinateY", "rlnCoordinateZ",
+                "rlnMicrographName", "rlnTomoName",
+                "rlnOriginX", "rlnOriginXAngst",
+                "rlnOriginY", "rlnOriginYAngst",
+                "rlnOriginZ", "rlnOriginZAngst",
+                "rlnPixelSize", "rlnImagePixelSize",
+                "rlnAngleRot", "rlnAngleTilt", "rlnAnglePsi",
+                "rlnOpticsGroup", "rlnOpticsGroupName"
+            };
+            foreach (var columnName in inputStar.GetColumnNames())
+            {
+                if (!knownColumns.Contains(columnName))
+                {
+                    _additionalColumns.Add(columnName, inputStar.GetColumn(columnName));
+                }
+            }
+
             string[] tiltSeriesIDs = inputStar.HasColumn("rlnMicrographName") ? inputStar.GetColumn("rlnMicrographName") : inputStar.GetColumn("rlnTomoName");
             Dictionary<string, List<int>> tiltSeriesIdToParticleIndices =
                 GroupParticles(tiltSeriesIDs);
@@ -236,6 +258,21 @@ public override async Task Run(object options)
                 float3[] tsParticleXyzAngstroms = new float3[tsParticleIdx.Count];
                 float3[] tsParticleRotTiltPsi = new float3[tsParticleIdx.Count];
 
+                Dictionary<string, string[]> tsAdditionalColumns = new Dictionary<string, string[]>();
+                if (_additionalColumns.Count > 0)
+                {
+                    foreach (var col in _additionalColumns)
+                    {
+                        var colData = new string[tsParticleIdx.Count];
+                        for (int i = 0; i < tsParticleIdx.Count; i++)
+                        {
+                            colData[i] = col.Value[tsParticleIdx[i]];
+                        }
+                        tsAdditionalColumns.Add(col.Key, colData);
+                    }
+                }
+
+
                 if (Helper.IsDebug)
                     Console.WriteLine($"{tsParticleIdx.Count} particles for {tiltSeries.Name}");
 
@@ -279,7 +316,8 @@ public override async Task Run(object options)
                                                                     inputHasEulerAngles: inputHasEulerAngles,
                                                                     outputPixelSize: cli.OutputPixelSize,
                                                                     relativeToParticleStarFile: cli.OutputPathsRelativeToStarFile,
-                                                                    particleStarFile: cli.OutputStarFile);
+                                                                    particleStarFile: cli.OutputStarFile,
+                                                                    additionalColumns: tsAdditionalColumns);
                     lock (OutputStarTables)
                         OutputStarTables.Add(tiltSeries.Name, TiltSeriesTable);
                 }
@@ -302,7 +340,8 @@ public override async Task Run(object options)
                                                             pathTableOut: TempTiltSeriesParticleStarPath,
                                                             pathsRelativeTo: cli.OutputPathsRelativeToStarFile ?
                                                                                  Path.GetFullPath(OutputStarPath) :
-                                                                                 Directory.GetCurrentDirectory());
+                                                                                 Directory.GetCurrentDirectory(),
+                                                            additionalColumns: tsAdditionalColumns);
 
                             // generate necessary metadata for particles.star
                             if (Helper.IsDebug)
@@ -697,7 +736,8 @@ private Star ConstructSubvolumeOutputTable(
             bool inputHasEulerAngles,
             float outputPixelSize,
             bool relativeToParticleStarFile, // default is relative to working directory 
-            string? particleStarFile
+            string? particleStarFile,
+            Dictionary<string, string[]> additionalColumns
         )
         {
             int nParticles = xyz.Length;
@@ -807,6 +847,14 @@ private Star ConstructSubvolumeOutputTable(
                 table.RemoveColumn("rlnAnglePsi");
             }
 
+            if (additionalColumns != null)
+            {
+                foreach (var pair in additionalColumns)
+                {
+                    table.AddColumn(pair.Key, pair.Value);
+                }
+            }
+
             return table;
         }
 

WarpWorker/WarpWorker.cs

@@ -1,6 +1,7 @@
 using Microsoft.Extensions.Hosting;
 using System;
 using System.Diagnostics;
+using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
 using System.Linq;
@@ -75,7 +76,7 @@ static async Task Main(string[] args)
                 webBuilder.UseKestrel(options =>
                     {
                         options.ListenAnyIP(Port);
-                        options.Limits.MaxRequestBodySize = 104857600; // 100 MB
+                        options.Limits.MaxRequestBodySize = 500 * 1024 * 1024; // 100 MB
                     })
                     .UseStartup<RESTStartup>()
                     .ConfigureLogging(logging => logging.SetMinimumLevel(LogLevel.Warning));
@@ -858,6 +859,19 @@ public static void EvaluateCommand(NamedSerializableObject Command)
 
                     TiltSeries T = new TiltSeries(Path);
                     T.ReconstructParticleSeries(Options, Coordinates, Angles, PathsRelativeTo, out TableOut);
+
+                    Dictionary<string, string[]> AdditionalColumns = null;
+                    if (Command.Content.Length > 6)
+                        AdditionalColumns = (Dictionary<string, string[]>)Command.Content[6];
+
+                    if (AdditionalColumns != null)
+                    {
+                        foreach (var col in AdditionalColumns)
+                        {
+                            TableOut.AddColumn(col.Key, col.Value);
+                        }
+                    }
+
                     T.SaveMeta();
 
                     if (!string.IsNullOrEmpty(PathTableOut))