Merge pull request #1229 from iceljc/master

Parse csv file

Author: iceljc

BotSharp.sln

@@ -150,6 +150,7 @@ EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.FuzzySharp", "src\Plugins\BotSharp.Plugin.FuzzySharp\BotSharp.Plugin.FuzzySharp.csproj", "{E7C243B9-E751-B3B4-8F16-95C76CA90D31}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.MMPEmbedding", "src\Plugins\BotSharp.Plugin.MMPEmbedding\BotSharp.Plugin.MMPEmbedding.csproj", "{394B858B-9C26-B977-A2DA-8CC7BE5914CB}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.Membase", "src\Plugins\BotSharp.Plugin.Membase\BotSharp.Plugin.Membase.csproj", "{13223C71-9EAC-9835-28ED-5A4833E6F915}"
 EndProject
 Global
@@ -720,7 +721,7 @@ Global
 		{FC63C875-E880-D8BB-B8B5-978AB7B62983} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
 		{242F2D93-FCCE-4982-8075-F3052ECCA92C} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
 		{E7C243B9-E751-B3B4-8F16-95C76CA90D31} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
-		{394B858B-9C26-B977-A2DA-8CC7BE5914CB} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C}
+		{394B858B-9C26-B977-A2DA-8CC7BE5914CB} = {4F346DCE-087F-4368-AF88-EE9C720D0E69}
 		{13223C71-9EAC-9835-28ED-5A4833E6F915} = {53E7CD86-0D19-40D9-A0FA-AB4613837E89}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution

src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/ReadExcelFn.cs

@@ -1,5 +1,7 @@
+using NPOI.HSSF.UserModel;
 using NPOI.SS.UserModel;
 using NPOI.XSSF.UserModel;
+using System.Globalization;
 using System.Linq.Dynamic.Core;
 
 namespace BotSharp.Plugin.ExcelHandler.Functions;
@@ -69,7 +71,7 @@ private void Init()
     {
         if (_excelFileTypes.IsNullOrEmpty())
         {
-            _excelFileTypes = FileUtility.GetMimeFileTypes(["excel", "spreadsheet"]).ToHashSet();
+            _excelFileTypes = FileUtility.GetMimeFileTypes(["excel", "spreadsheet", "csv"]).ToHashSet();
         }
     }
 
@@ -133,7 +135,7 @@ private List<SqlContextOut> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
             }
 
             var binary = _fileStorage.GetFileBytes(file.FileStorageUrl);
-            var workbook = ConvertToWorkBook(binary);
+            var workbook = ConvertToWorkbook(binary, extension);
 
             var currentCommands = _dbService.WriteExcelDataToDB(workbook);
             sqlCommands.AddRange(currentCommands);
@@ -167,11 +169,22 @@ private string GenerateSqlExecutionSummary(List<SqlContextOut> results)
         return stringBuilder.ToString();
     }
 
-    private IWorkbook ConvertToWorkBook(BinaryData binary)
+    private IWorkbook ConvertToWorkbook(BinaryData binary, string extension)
     {
-        using var fileStream = new MemoryStream(binary.ToArray());
-        IWorkbook workbook = new XSSFWorkbook(fileStream);
-        return workbook;
+        var bytes = binary.ToArray();
+
+        if (extension.IsEqualTo(".csv"))
+        {
+            return ExcelHelper.ConvertCsvToWorkbook(bytes);
+        }
+
+        using var fileStream = new MemoryStream(bytes);
+        if (extension.IsEqualTo(".xls"))
+        {
+            return new HSSFWorkbook(fileStream);
+        }
+
+        return new XSSFWorkbook(fileStream);
     }
     #endregion
 }

src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/ExcelHelper.cs

@@ -0,0 +1,82 @@
+using NPOI.SS.UserModel;
+using NPOI.XSSF.UserModel;
+using System.Globalization;
+
+namespace BotSharp.Plugin.ExcelHandler.Helpers;
+
+internal static class ExcelHelper
+{
+    internal static IWorkbook ConvertCsvToWorkbook(byte[] bytes)
+    {
+        IWorkbook workbook = new XSSFWorkbook();
+        ISheet sheet = workbook.CreateSheet("Sheet1");
+
+        using var memoryStream = new MemoryStream(bytes);
+        using var reader = new StreamReader(memoryStream);
+
+        int rowIndex = 0;
+        string? line;
+
+        while ((line = reader.ReadLine()) != null)
+        {
+            IRow row = sheet.CreateRow(rowIndex);
+            var values = ParseCsvLine(line);
+
+            for (int colIndex = 0; colIndex < values.Count; colIndex++)
+            {
+                ICell cell = row.CreateCell(colIndex);
+                var value = values[colIndex];
+
+                if (rowIndex > 0 && double.TryParse(value, NumberStyles.Any, CultureInfo.InvariantCulture, out double numericValue))
+                {
+                    cell.SetCellValue(numericValue);
+                }
+                else
+                {
+                    cell.SetCellValue(value);
+                }
+            }
+
+            rowIndex++;
+        }
+
+        return workbook;
+    }
+
+    private static List<string> ParseCsvLine(string line)
+    {
+        var values = new List<string>();
+        var currentValue = new StringBuilder();
+        bool inQuotes = false;
+
+        for (int i = 0; i < line.Length; i++)
+        {
+            char c = line[i];
+
+            if (c == '"')
+            {
+                if (inQuotes && i + 1 < line.Length && line[i + 1] == '"')
+                {
+                    currentValue.Append('"');
+                    i++;
+                }
+                else
+                {
+                    inQuotes = !inQuotes;
+                }
+            }
+            else if (c == ',' && !inQuotes)
+            {
+                values.Add(currentValue.ToString());
+                currentValue.Clear();
+            }
+            else
+            {
+                currentValue.Append(c);
+            }
+        }
+
+        values.Add(currentValue.ToString());
+        return values;
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs

@@ -162,11 +162,13 @@ private string ParseSheetData(ISheet singleSheet)
     private List<string> ParseSheetColumn(ISheet sheet)
     {
         if (sheet.PhysicalNumberOfRows < 2)
+        {
             throw new Exception("No data found in the excel file");
+        }
 
         _excelRowSize = sheet.PhysicalNumberOfRows - 1;
         var headerRow = sheet.GetRow(0);
-        var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
+        var headerColumn = headerRow.Cells.Where(x => !string.IsNullOrWhiteSpace(x.StringCellValue)).Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
         _excelColumnSize = headerColumn.Count;
         return headerColumn;
     }

src/Plugins/BotSharp.Plugin.ExcelHandler/Using.cs

@@ -27,6 +27,7 @@
 global using BotSharp.Plugin.ExcelHandler.Models;
 global using BotSharp.Plugin.ExcelHandler.Services;
 global using BotSharp.Plugin.ExcelHandler.Settings;
+global using BotSharp.Plugin.ExcelHandler.Helpers;
 
 global using Microsoft.Extensions.Logging;
 global using Microsoft.Extensions.DependencyInjection;