Necessary changes for Linux (gcc & clang), and a few enhancements...

Makefile

@@ -12,6 +12,8 @@ all : engine
 test : enginetest
 	$(ROOT_DIR)/kit/$(PLATFORM)/$(MODE)/bin/iknowenginetest
 
+languagecompiler : base core shell
+	$(MAKE) -f $(ROOT_DIR)/modules/compiler/iKnowLanguageCompiler/languagecompiler.mak
 enginetest : engine base
 	$(MAKE) -f $(ROOT_DIR)/modules/enginetest/enginetest.mak
 engine : base shell core icu
@@ -33,6 +35,7 @@ icu :
 	test -d $(ICUDIR)/include/unicode && (ls $(ICUDIR)/lib/libicu* > /dev/null)
 
 clean :
+	$(MAKE) -f $(ROOT_DIR)/modules/compiler/iKnowLanguageCompiler/languagecompiler.mak clean
 	$(MAKE) -f $(ROOT_DIR)/modules/enginetest/enginetest.mak clean
 	$(MAKE) -f $(ROOT_DIR)/modules/engine/engine.mak clean
 	$(MAKE) -f $(ROOT_DIR)/modules/shell/shell.mak clean

modules/compiler/iKnowLanguageCompiler/CSV_DataGenerator.cpp

@@ -1,3 +1,11 @@
+/*
+** CSV_DataGenerator.cpp
+*/
+
+#ifdef WIN32
+#pragma warning (disable: 4251)
+#endif
+
 #include "CSV_DataGenerator.h"
 #include "Util.h"
 #include "IkStringEncoding.h"
@@ -188,7 +196,8 @@ void CSV_DataGenerator::loadCSVdata(std::string language, bool IsCompiled)
 	kb_labels.clear();
 	cap = kb_labels.capacity();
 	cout << "Reading label data..." << endl;
-	iKnow_KB_Label::ImportFromCSV(csv_path_ + language + "\\" + "labels.csv", *this);
+	if (!iKnow_KB_Label::ImportFromCSV(csv_path_ + language + "\\" + "labels.csv", *this))
+		throw ExceptionFrom<CSV_DataGenerator>("Cannot build a language model without external labels !!!");
 	cout << kb_labels.size() << " label items (reserved=" << cap << ")" << endl;
 
 	if (!IsCompiled) {
@@ -246,7 +255,7 @@ static const size_t kRawSize = 48000000;
 #include "KbMetadata.h"
 
 // static definition:
-const IkLabel::LabelTypeMap IkLabel::label_type_map_;
+// const IkLabel::LabelTypeMap IkLabel::label_type_map_;
 
 using namespace iknow::shell;
 
@@ -641,7 +650,7 @@ void CSV_DataGenerator::generateRAW(void)
 	allocator.generate_image(language_data_path_, GetName());
 #endif
 
-	delete buf_;
+	delete[] buf_;
 }
 
 

modules/compiler/iKnowLanguageCompiler/CSV_DataGenerator.h

@@ -1,4 +1,8 @@
 #pragma once
+#ifdef WIN32
+#pragma warning (disable: 4251)
+#endif
+
 #include "iKnow_KB_Metadata.h"
 #include "iKnow_KB_Acronym.h"
 #include "iKnow_KB_Regex.h"
@@ -88,14 +92,6 @@ namespace iknow {
 			void CompileLexrepDictionaryPhase(/*kb As %iKnow.KB.Knowledgebase,*/ std::string phase, bool phasePredicate /*Predicate *phasePredicate*/, std::string& outputDir_);
 			std::vector<int> CreateLabelsIndexVector(iKnow_KB_Lexrep& lexrep, std::unordered_map<std::string, int>& table);
 		};
-
-		class testje {
-		public:
-			testje(void) {}
-
-
-		};
-
 	}
 }
 

modules/compiler/iKnowLanguageCompiler/GotoFunction.cpp

@@ -2,6 +2,7 @@
 #include "LexrepStateOutputFunc.h"
 #include "IkStringEncoding.h"
 #include "Util.h"
+#include "utlExceptionFrom.h"
 
 #include <fstream>
 #include <map>
@@ -337,7 +338,7 @@ void GotoFunction::ToC(std::string dir)
 					String regexName = String(regex.begin() + 1, regex.end() - 1);  //	Set regexName = $E(regex, 2, *-1) //remove {}
 					String pattern = RegexDictionary->Lookup(regexName); //	Set pattern = ..RegexDictionary.Lookup(regexName)
 
-					// 	If pattern = "" Throw ##class(%Exception.SystemException).%New("Unknown regex specified.")
+					if (pattern.empty()) throw ExceptionFrom<GotoFunction>("Unknown regex specified."); // 	If pattern = "" Throw ##class(%Exception.SystemException).%New("Unknown regex specified.")
 					ofs.o() << "static const Char Regex" << i << "Str[] = {"; // Write "static const Char Regex"_i_"Str[] = {"
 					for (int j = 1; j <= pattern.length(); j++) {	// For j = 1 :1 : $L(pattern) {
 						ofs.o() << static_cast<int>(pattern[j - 1]) << ", "; // W $A(pattern, j)_", "

modules/compiler/iKnowLanguageCompiler/GotoFunction.h

@@ -4,7 +4,7 @@
 #include <map>
 #include <vector>
 
-#include "ikTypes.h"
+#include "IkTypes.h"
 #include "StateOutputFunction.h"
 #include "KnowledgebaseRegexDictionary.h"
 

modules/compiler/iKnowLanguageCompiler/KnowledgebaseRegexDictionary.h

@@ -1,4 +1,8 @@
 #pragma once
+#ifdef WIN32
+#pragma warning (disable: 4251)
+#endif
+
 #include "IkTypes.h"
 
 namespace iknow {

modules/compiler/iKnowLanguageCompiler/iKnowLanguageCompiler.cpp

@@ -19,7 +19,11 @@ int main(int argc, char* argv[])
 	string repo_root("C:/Users/jdenys/source/repos/iknow/");
 
 	string exe_path(argv[0]);
+#ifdef WIN32
 	size_t kit_pos = exe_path.find("\\kit\\");
+#else
+	size_t kit_pos = exe_path.find("/kit/");
+#endif
 	if (kit_pos != string::npos) {
 		repo_root = string(exe_path.begin(), exe_path.begin() + kit_pos + 1);
 	}

modules/compiler/iKnowLanguageCompiler/iKnowLanguageCompiler.vcxproj

@@ -61,14 +61,14 @@
       </PrecompiledHeader>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;_CONSOLE;_DEBUG</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <AdditionalIncludeDirectories>..\..\base\src\headers;..\..\core\src\headers;..\..\shell\src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalDependencies>iKnowBase.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>iKnowBase.lib;iKnowCore.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <AdditionalLibraryDirectories>$(OutDir);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
     </Link>
   </ItemDefinitionGroup>
@@ -80,7 +80,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;_CONSOLE;NDEBUG</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <AdditionalIncludeDirectories>..\..\base\src\headers;..\..\core\src\headers;..\..\shell\src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
@@ -89,7 +89,7 @@
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalDependencies>iKnowBase.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>iKnowBase.lib;iKnowCore.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <AdditionalLibraryDirectories>$(OutDir);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
     </Link>
   </ItemDefinitionGroup>

modules/compiler/iKnowLanguageCompiler/iKnow_KB_Acronym.cpp

@@ -11,7 +11,7 @@ using namespace iknow::csvdata;
 using namespace std;
 
 // ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase)
-void iKnow_KB_Acronym::ImportFromCSV(std::string acro_csv, CSV_DataGenerator& kb)
+bool iKnow_KB_Acronym::ImportFromCSV(std::string acro_csv, CSV_DataGenerator& kb)
 {
 	ifstream ifs = ifstream(acro_csv, ifstream::in);
 	if (ifs.is_open()) {
@@ -33,18 +33,9 @@ void iKnow_KB_Acronym::ImportFromCSV(std::string acro_csv, CSV_DataGenerator& kb
 			kb.kb_acronyms.push_back(acronym); // Set sc = acronym.%Save()
 			// $$$IKModelCheck(sc, stream.Filename, count, line)
 		}
+		ifs.close();
+		return true;
 	}
-	else {
-		cerr << "Error opening file: " << acro_csv << " Language=\"" << kb.GetName() << "\"" << endl;
-	}
-	ifs.close();
-}
-
-iKnow_KB_Acronym::iKnow_KB_Acronym()
-{
-}
-
-
-iKnow_KB_Acronym::~iKnow_KB_Acronym()
-{
+	cerr << "Error opening file: " << acro_csv << " Language=\"" << kb.GetName() << "\"" << endl;
+	return false;
 }

modules/compiler/iKnowLanguageCompiler/iKnow_KB_Acronym.h

@@ -8,7 +8,7 @@ namespace iknow {
 		{
 		public:
 			// ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase)
-			static void ImportFromCSV(std::string acro_csv, CSV_DataGenerator& kb);
+			static bool ImportFromCSV(std::string acro_csv, CSV_DataGenerator& kb);
 
 			std::string Token; // Property Token As %String(MAXLEN = 256, XMLPROJECTION = "ATTRIBUTE")[Required];
 
@@ -18,8 +18,8 @@ namespace iknow {
 
 			// Index KnowledgebaseIndex On Knowledgebase;
 
-			iKnow_KB_Acronym();
-			~iKnow_KB_Acronym();
+			iKnow_KB_Acronym() {}
+			~iKnow_KB_Acronym() {}
 		};
 	}
 }

modules/compiler/iKnowLanguageCompiler/iKnow_KB_Filter.cpp

@@ -7,7 +7,7 @@ using namespace iknow::csvdata;
 using namespace std;
 
 // ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase)
-void iKnow_KB_Filter::ImportFromCSV(string filter_csv, CSV_DataGenerator& kb)
+bool iKnow_KB_Filter::ImportFromCSV(string filter_csv, CSV_DataGenerator& kb)
 {
 	ifstream ifs = ifstream(filter_csv, ifstream::in);
 	if (ifs.is_open()) {
@@ -52,19 +52,9 @@ void iKnow_KB_Filter::ImportFromCSV(string filter_csv, CSV_DataGenerator& kb)
 			kb.kb_filter.push_back(filter); //	Set sc = filter.%Save()
 			//	$$$IKModelCheck(sc, stream.Filename, count, line)
 		}
+		ifs.close();
+		return true;
 	}
-	else {
-		cerr << "Error opening file: " << filter_csv << " Language=\"" << kb.GetName() << "\"" << endl;
-	}
-	ifs.close();
-}
-
-
-iKnow_KB_Filter::iKnow_KB_Filter()
-{
-}
-
-
-iKnow_KB_Filter::~iKnow_KB_Filter()
-{
-}
+	cerr << "Error opening file: " << filter_csv << " Language=\"" << kb.GetName() << "\"" << endl;
+	return false;
+}

modules/compiler/iKnowLanguageCompiler/iKnow_KB_Filter.h

@@ -9,7 +9,7 @@ namespace iknow {
 		{
 		public:
 			// ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase)
-			static void ImportFromCSV(std::string filter_csv, CSV_DataGenerator& kb);
+			static bool ImportFromCSV(std::string filter_csv, CSV_DataGenerator& kb);
 
 			std::string InputToken; // InputToken As %String(MAXLEN = 256, XMLPROJECTION = "ATTRIBUTE")[Required];
 
@@ -27,8 +27,8 @@ namespace iknow {
 
 			// Index KnowledgebaseIndex On Knowledgebase;
 
-			iKnow_KB_Filter();
-			~iKnow_KB_Filter();
+			iKnow_KB_Filter() {}
+			~iKnow_KB_Filter() {}
 		};
 	}
 }

modules/compiler/iKnowLanguageCompiler/iKnow_KB_Label.cpp

@@ -56,7 +56,7 @@ void iKnow_KB_Label::LoadSpecialLabels(CSV_DataGenerator& kb)
 
 }
 
-void iKnow_KB_Label::ImportFromCSV(string labels_csv, CSV_DataGenerator& kb)
+bool iKnow_KB_Label::ImportFromCSV(string labels_csv, CSV_DataGenerator& kb)
 {
 	LoadSpecialLabels(kb); // Do ..LoadSpecialLabels(kb)
 	int count = 0;
@@ -78,11 +78,11 @@ void iKnow_KB_Label::ImportFromCSV(string labels_csv, CSV_DataGenerator& kb)
 			if (isDefault[0]=='1') kb.kb_concept_label = &kb.kb_labels.back(); // If isDefault Set kb.ConceptLabel = label
 			// $$$IKModelCheck(sc,stream.Filename,count,line)
 		}
+		ifs.close();
+		return true;
 	}
-	else {
-		cerr << "Error opening file: " << labels_csv << " Language=\"" << kb.GetName() << "\"" << endl;
-	}
-	ifs.close();
+	cerr << "Error opening file: " << labels_csv << " Language=\"" << kb.GetName() << "\"" << endl;
+	return false;
 }
 
 iKnow_KB_Label iKnow_KB_Label::LabelFromString(vector<string>& row_label, string &isDefault) // ClassMethod LabelFromString(line As %String, ByRef isDefault = "") As Label

modules/compiler/iKnowLanguageCompiler/iKnow_KB_Label.h

@@ -11,7 +11,7 @@ namespace iknow {
 		{
 		public:
 			// ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase)
-			static void ImportFromCSV(std::string label_csv, CSV_DataGenerator& kb);
+			static bool ImportFromCSV(std::string label_csv, CSV_DataGenerator& kb);
 
 			std::string Name; // Name As %String(MAXLEN = 256, XMLPROJECTION = "ATTRIBUTE")[Required];
 

modules/compiler/iKnowLanguageCompiler/iKnow_KB_Lexrep.cpp

@@ -111,7 +111,7 @@ kill:$$$IKISDEVBUILD ^lexreps(kb.Name) // temporary storage for double lexrep ch
 // Set:$$$IKISDEVBUILD hasUpper = ""
 }
 */
-void iKnow_KB_Lexrep::ImportFromCSV(string lexreps_csv, CSV_DataGenerator& kb)
+bool iKnow_KB_Lexrep::ImportFromCSV(string lexreps_csv, CSV_DataGenerator& kb)
 {
 	kb.kb_lexreps.clear();
 	kb.lexrep_index.clear();
@@ -147,14 +147,14 @@ void iKnow_KB_Lexrep::ImportFromCSV(string lexreps_csv, CSV_DataGenerator& kb)
 				lexrep.Token = tokens[i]; // Set lexrep.Token = tokens(i)
 				lexrep.Meta = meta; // Set lexrep.Meta = meta
 				lexrep.Labels = labels; // Set lexrep.Labels = labels
-				kb.lexrep_index[lexrep.Token] = (int) kb.kb_lexreps.size(); // index on Token for fast retrieval
+				kb.lexrep_index[lexrep.Token] = (int)kb.kb_lexreps.size(); // index on Token for fast retrieval
 				kb.kb_lexreps.push_back(lexrep); // Set lexrep.Knowledgebase = kb
 			}
 			if (!(kb.kb_lexreps.size() % 2048)) cout << char(9) << kb.kb_lexreps.size();
 		}
+		ifs.close();
+		return true;
 	}
-	else {
-		cerr << "Error opening file: " << lexreps_csv << " Language=\"" << kb.GetName() << "\"" << endl;
-	}
-	ifs.close();
+	cerr << "Error opening file: " << lexreps_csv << " Language=\"" << kb.GetName() << "\"" << endl;
+	return false;
 }

modules/compiler/iKnowLanguageCompiler/iKnow_KB_Lexrep.h

@@ -12,7 +12,7 @@ namespace iknow {
 		{
 		public:
 			// ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase)
-			static void ImportFromCSV(std::string lexrep_csv, CSV_DataGenerator& kb);
+			static bool ImportFromCSV(std::string lexrep_csv, CSV_DataGenerator& kb);
 
 			iKnow_KB_Lexrep(std::vector<std::string>& row_lexrep) {
 				Meta = row_lexrep[2 - 1];

modules/compiler/iKnowLanguageCompiler/iKnow_KB_Metadata.cpp

@@ -7,7 +7,7 @@
 using namespace iknow::csvdata;
 using namespace std;
 
-void iKnow_KB_Metadata::ImportFromCSV(std::string metadata_csv, CSV_DataGenerator& kb)
+bool iKnow_KB_Metadata::ImportFromCSV(std::string metadata_csv, CSV_DataGenerator& kb)
 {
 	ifstream ifs = ifstream(metadata_csv, ifstream::in);
 	if (ifs.is_open()) {
@@ -29,20 +29,9 @@ void iKnow_KB_Metadata::ImportFromCSV(std::string metadata_csv, CSV_DataGenerato
 			kb.kb_metadata.push_back(metadata); //	Set sc = metadata.%Save()
 			//	$$$IKModelCheck(sc, stream.Filename, count, line)
 		}
+		ifs.close();
+		return true;
 	}
-	else {
-		cerr << "Error opening file: " << metadata_csv << " Language=\"" << kb.GetName() << "\"" << endl;
-	}
-
-	ifs.close();
-
-}
-
-iKnow_KB_Metadata::iKnow_KB_Metadata()
-{
-}
-
-
-iKnow_KB_Metadata::~iKnow_KB_Metadata()
-{
-}
+	cerr << "Error opening file: " << metadata_csv << " Language=\"" << kb.GetName() << "\"" << endl;
+	return false;
+}

modules/compiler/iKnowLanguageCompiler/iKnow_KB_Metadata.h

@@ -9,14 +9,14 @@ namespace iknow {
 		{
 		public:
 			// ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase)
-			static void ImportFromCSV(std::string metadata_csv, CSV_DataGenerator& kb);
+			static bool ImportFromCSV(std::string metadata_csv, CSV_DataGenerator& kb);
 
 			std::string Name; // Property Name As %String(MAXLEN = 128);
 
 			std::string Val; //  Property Val As %String(MAXLEN = 256);
 
-			iKnow_KB_Metadata();
-			~iKnow_KB_Metadata();
+			iKnow_KB_Metadata() {}
+			~iKnow_KB_Metadata() {}
 		};
 	}
 }