From 1b2959dbc6bca11f6156352afb25b445520f64b4 Mon Sep 17 00:00:00 2001 From: jdenys Date: Tue, 21 Apr 2020 15:18:28 +0200 Subject: [PATCH] Necessary changes for Linux (gcc & clang), and a few enhancements... --- Makefile | 3 ++ .../CSV_DataGenerator.cpp | 15 +++++-- .../iKnowLanguageCompiler/CSV_DataGenerator.h | 12 ++--- .../iKnowLanguageCompiler/GotoFunction.cpp | 3 +- .../iKnowLanguageCompiler/GotoFunction.h | 2 +- .../KnowledgebaseRegexDictionary.h | 4 ++ .../iKnowLanguageCompiler.cpp | 4 ++ .../iKnowLanguageCompiler.vcxproj | 8 ++-- .../iKnow_KB_Acronym.cpp | 19 +++----- .../iKnowLanguageCompiler/iKnow_KB_Acronym.h | 6 +-- .../iKnowLanguageCompiler/iKnow_KB_Filter.cpp | 22 +++------- .../iKnowLanguageCompiler/iKnow_KB_Filter.h | 6 +-- .../iKnowLanguageCompiler/iKnow_KB_Label.cpp | 10 ++--- .../iKnowLanguageCompiler/iKnow_KB_Label.h | 2 +- .../iKnowLanguageCompiler/iKnow_KB_Lexrep.cpp | 12 ++--- .../iKnowLanguageCompiler/iKnow_KB_Lexrep.h | 2 +- .../iKnow_KB_Metadata.cpp | 23 +++------- .../iKnowLanguageCompiler/iKnow_KB_Metadata.h | 6 +-- .../iKnow_KB_PreprocessFilter.cpp | 21 +++------ .../iKnow_KB_PreprocessFilter.h | 6 +-- .../iKnowLanguageCompiler/iKnow_KB_Regex.cpp | 22 +++------- .../iKnowLanguageCompiler/iKnow_KB_Regex.h | 6 +-- .../iKnowLanguageCompiler/iKnow_KB_Rule.cpp | 28 ++++-------- .../iKnowLanguageCompiler/iKnow_KB_Rule.h | 6 +-- .../languagecompiler.mak | 26 +++++++++++ modules/core/src/IkLabel.cpp | 44 +++++++++++++++++-- modules/core/src/headers/IkLabel.h | 41 +---------------- modules/shell/src/RawBlock.h | 38 ++++++++-------- 28 files changed, 191 insertions(+), 206 deletions(-) create mode 100644 modules/compiler/iKnowLanguageCompiler/languagecompiler.mak diff --git a/Makefile b/Makefile index 27e56ba1..fdd38d79 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,8 @@ all : engine test : enginetest $(ROOT_DIR)/kit/$(PLATFORM)/$(MODE)/bin/iknowenginetest +languagecompiler : base core shell + $(MAKE) -f $(ROOT_DIR)/modules/compiler/iKnowLanguageCompiler/languagecompiler.mak enginetest : engine base $(MAKE) -f $(ROOT_DIR)/modules/enginetest/enginetest.mak engine : base shell core icu @@ -33,6 +35,7 @@ icu : test -d $(ICUDIR)/include/unicode && (ls $(ICUDIR)/lib/libicu* > /dev/null) clean : + $(MAKE) -f $(ROOT_DIR)/modules/compiler/iKnowLanguageCompiler/languagecompiler.mak clean $(MAKE) -f $(ROOT_DIR)/modules/enginetest/enginetest.mak clean $(MAKE) -f $(ROOT_DIR)/modules/engine/engine.mak clean $(MAKE) -f $(ROOT_DIR)/modules/shell/shell.mak clean diff --git a/modules/compiler/iKnowLanguageCompiler/CSV_DataGenerator.cpp b/modules/compiler/iKnowLanguageCompiler/CSV_DataGenerator.cpp index b86d23e0..20d20151 100644 --- a/modules/compiler/iKnowLanguageCompiler/CSV_DataGenerator.cpp +++ b/modules/compiler/iKnowLanguageCompiler/CSV_DataGenerator.cpp @@ -1,3 +1,11 @@ +/* +** CSV_DataGenerator.cpp +*/ + +#ifdef WIN32 +#pragma warning (disable: 4251) +#endif + #include "CSV_DataGenerator.h" #include "Util.h" #include "IkStringEncoding.h" @@ -188,7 +196,8 @@ void CSV_DataGenerator::loadCSVdata(std::string language, bool IsCompiled) kb_labels.clear(); cap = kb_labels.capacity(); cout << "Reading label data..." << endl; - iKnow_KB_Label::ImportFromCSV(csv_path_ + language + "\\" + "labels.csv", *this); + if (!iKnow_KB_Label::ImportFromCSV(csv_path_ + language + "\\" + "labels.csv", *this)) + throw ExceptionFrom("Cannot build a language model without external labels !!!"); cout << kb_labels.size() << " label items (reserved=" << cap << ")" << endl; if (!IsCompiled) { @@ -246,7 +255,7 @@ static const size_t kRawSize = 48000000; #include "KbMetadata.h" // static definition: -const IkLabel::LabelTypeMap IkLabel::label_type_map_; +// const IkLabel::LabelTypeMap IkLabel::label_type_map_; using namespace iknow::shell; @@ -641,7 +650,7 @@ void CSV_DataGenerator::generateRAW(void) allocator.generate_image(language_data_path_, GetName()); #endif - delete buf_; + delete[] buf_; } diff --git a/modules/compiler/iKnowLanguageCompiler/CSV_DataGenerator.h b/modules/compiler/iKnowLanguageCompiler/CSV_DataGenerator.h index eaf8bf2b..9c68554c 100644 --- a/modules/compiler/iKnowLanguageCompiler/CSV_DataGenerator.h +++ b/modules/compiler/iKnowLanguageCompiler/CSV_DataGenerator.h @@ -1,4 +1,8 @@ #pragma once +#ifdef WIN32 +#pragma warning (disable: 4251) +#endif + #include "iKnow_KB_Metadata.h" #include "iKnow_KB_Acronym.h" #include "iKnow_KB_Regex.h" @@ -88,14 +92,6 @@ namespace iknow { void CompileLexrepDictionaryPhase(/*kb As %iKnow.KB.Knowledgebase,*/ std::string phase, bool phasePredicate /*Predicate *phasePredicate*/, std::string& outputDir_); std::vector CreateLabelsIndexVector(iKnow_KB_Lexrep& lexrep, std::unordered_map& table); }; - - class testje { - public: - testje(void) {} - - - }; - } } diff --git a/modules/compiler/iKnowLanguageCompiler/GotoFunction.cpp b/modules/compiler/iKnowLanguageCompiler/GotoFunction.cpp index 7cf98650..51616a87 100644 --- a/modules/compiler/iKnowLanguageCompiler/GotoFunction.cpp +++ b/modules/compiler/iKnowLanguageCompiler/GotoFunction.cpp @@ -2,6 +2,7 @@ #include "LexrepStateOutputFunc.h" #include "IkStringEncoding.h" #include "Util.h" +#include "utlExceptionFrom.h" #include #include @@ -337,7 +338,7 @@ void GotoFunction::ToC(std::string dir) String regexName = String(regex.begin() + 1, regex.end() - 1); // Set regexName = $E(regex, 2, *-1) //remove {} String pattern = RegexDictionary->Lookup(regexName); // Set pattern = ..RegexDictionary.Lookup(regexName) - // If pattern = "" Throw ##class(%Exception.SystemException).%New("Unknown regex specified.") + if (pattern.empty()) throw ExceptionFrom("Unknown regex specified."); // If pattern = "" Throw ##class(%Exception.SystemException).%New("Unknown regex specified.") ofs.o() << "static const Char Regex" << i << "Str[] = {"; // Write "static const Char Regex"_i_"Str[] = {" for (int j = 1; j <= pattern.length(); j++) { // For j = 1 :1 : $L(pattern) { ofs.o() << static_cast(pattern[j - 1]) << ", "; // W $A(pattern, j)_", " diff --git a/modules/compiler/iKnowLanguageCompiler/GotoFunction.h b/modules/compiler/iKnowLanguageCompiler/GotoFunction.h index bd53fee2..876275a6 100644 --- a/modules/compiler/iKnowLanguageCompiler/GotoFunction.h +++ b/modules/compiler/iKnowLanguageCompiler/GotoFunction.h @@ -4,7 +4,7 @@ #include #include -#include "ikTypes.h" +#include "IkTypes.h" #include "StateOutputFunction.h" #include "KnowledgebaseRegexDictionary.h" diff --git a/modules/compiler/iKnowLanguageCompiler/KnowledgebaseRegexDictionary.h b/modules/compiler/iKnowLanguageCompiler/KnowledgebaseRegexDictionary.h index 27ec77f0..402809dc 100644 --- a/modules/compiler/iKnowLanguageCompiler/KnowledgebaseRegexDictionary.h +++ b/modules/compiler/iKnowLanguageCompiler/KnowledgebaseRegexDictionary.h @@ -1,4 +1,8 @@ #pragma once +#ifdef WIN32 +#pragma warning (disable: 4251) +#endif + #include "IkTypes.h" namespace iknow { diff --git a/modules/compiler/iKnowLanguageCompiler/iKnowLanguageCompiler.cpp b/modules/compiler/iKnowLanguageCompiler/iKnowLanguageCompiler.cpp index 063c61ea..1df45e27 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnowLanguageCompiler.cpp +++ b/modules/compiler/iKnowLanguageCompiler/iKnowLanguageCompiler.cpp @@ -19,7 +19,11 @@ int main(int argc, char* argv[]) string repo_root("C:/Users/jdenys/source/repos/iknow/"); string exe_path(argv[0]); +#ifdef WIN32 size_t kit_pos = exe_path.find("\\kit\\"); +#else + size_t kit_pos = exe_path.find("/kit/"); +#endif if (kit_pos != string::npos) { repo_root = string(exe_path.begin(), exe_path.begin() + kit_pos + 1); } diff --git a/modules/compiler/iKnowLanguageCompiler/iKnowLanguageCompiler.vcxproj b/modules/compiler/iKnowLanguageCompiler/iKnowLanguageCompiler.vcxproj index f8c37ee9..4f424b8f 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnowLanguageCompiler.vcxproj +++ b/modules/compiler/iKnowLanguageCompiler/iKnowLanguageCompiler.vcxproj @@ -61,14 +61,14 @@ Level3 true - _DEBUG;_CONSOLE;%(PreprocessorDefinitions) + WIN32;_CONSOLE;_DEBUG true ..\..\base\src\headers;..\..\core\src\headers;..\..\shell\src;%(AdditionalIncludeDirectories) Console true - iKnowBase.lib;%(AdditionalDependencies) + iKnowBase.lib;iKnowCore.lib;%(AdditionalDependencies) $(OutDir);%(AdditionalLibraryDirectories) @@ -80,7 +80,7 @@ true true true - NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + WIN32;_CONSOLE;NDEBUG true ..\..\base\src\headers;..\..\core\src\headers;..\..\shell\src;%(AdditionalIncludeDirectories) @@ -89,7 +89,7 @@ true true true - iKnowBase.lib;%(AdditionalDependencies) + iKnowBase.lib;iKnowCore.lib;%(AdditionalDependencies) $(OutDir);%(AdditionalLibraryDirectories) diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Acronym.cpp b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Acronym.cpp index d773891a..c166a888 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Acronym.cpp +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Acronym.cpp @@ -11,7 +11,7 @@ using namespace iknow::csvdata; using namespace std; // ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase) -void iKnow_KB_Acronym::ImportFromCSV(std::string acro_csv, CSV_DataGenerator& kb) +bool iKnow_KB_Acronym::ImportFromCSV(std::string acro_csv, CSV_DataGenerator& kb) { ifstream ifs = ifstream(acro_csv, ifstream::in); if (ifs.is_open()) { @@ -33,18 +33,9 @@ void iKnow_KB_Acronym::ImportFromCSV(std::string acro_csv, CSV_DataGenerator& kb kb.kb_acronyms.push_back(acronym); // Set sc = acronym.%Save() // $$$IKModelCheck(sc, stream.Filename, count, line) } + ifs.close(); + return true; } - else { - cerr << "Error opening file: " << acro_csv << " Language=\"" << kb.GetName() << "\"" << endl; - } - ifs.close(); -} - -iKnow_KB_Acronym::iKnow_KB_Acronym() -{ -} - - -iKnow_KB_Acronym::~iKnow_KB_Acronym() -{ + cerr << "Error opening file: " << acro_csv << " Language=\"" << kb.GetName() << "\"" << endl; + return false; } diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Acronym.h b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Acronym.h index 2b5dca44..f777b67b 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Acronym.h +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Acronym.h @@ -8,7 +8,7 @@ namespace iknow { { public: // ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase) - static void ImportFromCSV(std::string acro_csv, CSV_DataGenerator& kb); + static bool ImportFromCSV(std::string acro_csv, CSV_DataGenerator& kb); std::string Token; // Property Token As %String(MAXLEN = 256, XMLPROJECTION = "ATTRIBUTE")[Required]; @@ -18,8 +18,8 @@ namespace iknow { // Index KnowledgebaseIndex On Knowledgebase; - iKnow_KB_Acronym(); - ~iKnow_KB_Acronym(); + iKnow_KB_Acronym() {} + ~iKnow_KB_Acronym() {} }; } } diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Filter.cpp b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Filter.cpp index bf2c4ee7..282507e3 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Filter.cpp +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Filter.cpp @@ -7,7 +7,7 @@ using namespace iknow::csvdata; using namespace std; // ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase) -void iKnow_KB_Filter::ImportFromCSV(string filter_csv, CSV_DataGenerator& kb) +bool iKnow_KB_Filter::ImportFromCSV(string filter_csv, CSV_DataGenerator& kb) { ifstream ifs = ifstream(filter_csv, ifstream::in); if (ifs.is_open()) { @@ -52,19 +52,9 @@ void iKnow_KB_Filter::ImportFromCSV(string filter_csv, CSV_DataGenerator& kb) kb.kb_filter.push_back(filter); // Set sc = filter.%Save() // $$$IKModelCheck(sc, stream.Filename, count, line) } + ifs.close(); + return true; } - else { - cerr << "Error opening file: " << filter_csv << " Language=\"" << kb.GetName() << "\"" << endl; - } - ifs.close(); -} - - -iKnow_KB_Filter::iKnow_KB_Filter() -{ -} - - -iKnow_KB_Filter::~iKnow_KB_Filter() -{ -} + cerr << "Error opening file: " << filter_csv << " Language=\"" << kb.GetName() << "\"" << endl; + return false; +} \ No newline at end of file diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Filter.h b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Filter.h index 550945ee..d29a4260 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Filter.h +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Filter.h @@ -9,7 +9,7 @@ namespace iknow { { public: // ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase) - static void ImportFromCSV(std::string filter_csv, CSV_DataGenerator& kb); + static bool ImportFromCSV(std::string filter_csv, CSV_DataGenerator& kb); std::string InputToken; // InputToken As %String(MAXLEN = 256, XMLPROJECTION = "ATTRIBUTE")[Required]; @@ -27,8 +27,8 @@ namespace iknow { // Index KnowledgebaseIndex On Knowledgebase; - iKnow_KB_Filter(); - ~iKnow_KB_Filter(); + iKnow_KB_Filter() {} + ~iKnow_KB_Filter() {} }; } } diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Label.cpp b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Label.cpp index 81443435..a6a29a85 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Label.cpp +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Label.cpp @@ -56,7 +56,7 @@ void iKnow_KB_Label::LoadSpecialLabels(CSV_DataGenerator& kb) } -void iKnow_KB_Label::ImportFromCSV(string labels_csv, CSV_DataGenerator& kb) +bool iKnow_KB_Label::ImportFromCSV(string labels_csv, CSV_DataGenerator& kb) { LoadSpecialLabels(kb); // Do ..LoadSpecialLabels(kb) int count = 0; @@ -78,11 +78,11 @@ void iKnow_KB_Label::ImportFromCSV(string labels_csv, CSV_DataGenerator& kb) if (isDefault[0]=='1') kb.kb_concept_label = &kb.kb_labels.back(); // If isDefault Set kb.ConceptLabel = label // $$$IKModelCheck(sc,stream.Filename,count,line) } + ifs.close(); + return true; } - else { - cerr << "Error opening file: " << labels_csv << " Language=\"" << kb.GetName() << "\"" << endl; - } - ifs.close(); + cerr << "Error opening file: " << labels_csv << " Language=\"" << kb.GetName() << "\"" << endl; + return false; } iKnow_KB_Label iKnow_KB_Label::LabelFromString(vector& row_label, string &isDefault) // ClassMethod LabelFromString(line As %String, ByRef isDefault = "") As Label diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Label.h b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Label.h index 195f4365..cc0f7b18 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Label.h +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Label.h @@ -11,7 +11,7 @@ namespace iknow { { public: // ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase) - static void ImportFromCSV(std::string label_csv, CSV_DataGenerator& kb); + static bool ImportFromCSV(std::string label_csv, CSV_DataGenerator& kb); std::string Name; // Name As %String(MAXLEN = 256, XMLPROJECTION = "ATTRIBUTE")[Required]; diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Lexrep.cpp b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Lexrep.cpp index e025271a..b34615a2 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Lexrep.cpp +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Lexrep.cpp @@ -111,7 +111,7 @@ kill:$$$IKISDEVBUILD ^lexreps(kb.Name) // temporary storage for double lexrep ch // Set:$$$IKISDEVBUILD hasUpper = "" } */ -void iKnow_KB_Lexrep::ImportFromCSV(string lexreps_csv, CSV_DataGenerator& kb) +bool iKnow_KB_Lexrep::ImportFromCSV(string lexreps_csv, CSV_DataGenerator& kb) { kb.kb_lexreps.clear(); kb.lexrep_index.clear(); @@ -147,14 +147,14 @@ void iKnow_KB_Lexrep::ImportFromCSV(string lexreps_csv, CSV_DataGenerator& kb) lexrep.Token = tokens[i]; // Set lexrep.Token = tokens(i) lexrep.Meta = meta; // Set lexrep.Meta = meta lexrep.Labels = labels; // Set lexrep.Labels = labels - kb.lexrep_index[lexrep.Token] = (int) kb.kb_lexreps.size(); // index on Token for fast retrieval + kb.lexrep_index[lexrep.Token] = (int)kb.kb_lexreps.size(); // index on Token for fast retrieval kb.kb_lexreps.push_back(lexrep); // Set lexrep.Knowledgebase = kb } if (!(kb.kb_lexreps.size() % 2048)) cout << char(9) << kb.kb_lexreps.size(); } + ifs.close(); + return true; } - else { - cerr << "Error opening file: " << lexreps_csv << " Language=\"" << kb.GetName() << "\"" << endl; - } - ifs.close(); + cerr << "Error opening file: " << lexreps_csv << " Language=\"" << kb.GetName() << "\"" << endl; + return false; } \ No newline at end of file diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Lexrep.h b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Lexrep.h index eefd582f..564b870b 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Lexrep.h +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Lexrep.h @@ -12,7 +12,7 @@ namespace iknow { { public: // ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase) - static void ImportFromCSV(std::string lexrep_csv, CSV_DataGenerator& kb); + static bool ImportFromCSV(std::string lexrep_csv, CSV_DataGenerator& kb); iKnow_KB_Lexrep(std::vector& row_lexrep) { Meta = row_lexrep[2 - 1]; diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Metadata.cpp b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Metadata.cpp index 72208ff8..fa76cf97 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Metadata.cpp +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Metadata.cpp @@ -7,7 +7,7 @@ using namespace iknow::csvdata; using namespace std; -void iKnow_KB_Metadata::ImportFromCSV(std::string metadata_csv, CSV_DataGenerator& kb) +bool iKnow_KB_Metadata::ImportFromCSV(std::string metadata_csv, CSV_DataGenerator& kb) { ifstream ifs = ifstream(metadata_csv, ifstream::in); if (ifs.is_open()) { @@ -29,20 +29,9 @@ void iKnow_KB_Metadata::ImportFromCSV(std::string metadata_csv, CSV_DataGenerato kb.kb_metadata.push_back(metadata); // Set sc = metadata.%Save() // $$$IKModelCheck(sc, stream.Filename, count, line) } + ifs.close(); + return true; } - else { - cerr << "Error opening file: " << metadata_csv << " Language=\"" << kb.GetName() << "\"" << endl; - } - - ifs.close(); - -} - -iKnow_KB_Metadata::iKnow_KB_Metadata() -{ -} - - -iKnow_KB_Metadata::~iKnow_KB_Metadata() -{ -} + cerr << "Error opening file: " << metadata_csv << " Language=\"" << kb.GetName() << "\"" << endl; + return false; +} \ No newline at end of file diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Metadata.h b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Metadata.h index 7c6da15d..a16254a1 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Metadata.h +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Metadata.h @@ -9,14 +9,14 @@ namespace iknow { { public: // ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase) - static void ImportFromCSV(std::string metadata_csv, CSV_DataGenerator& kb); + static bool ImportFromCSV(std::string metadata_csv, CSV_DataGenerator& kb); std::string Name; // Property Name As %String(MAXLEN = 128); std::string Val; // Property Val As %String(MAXLEN = 256); - iKnow_KB_Metadata(); - ~iKnow_KB_Metadata(); + iKnow_KB_Metadata() {} + ~iKnow_KB_Metadata() {} }; } } diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_PreprocessFilter.cpp b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_PreprocessFilter.cpp index 15ca2361..07f8f875 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_PreprocessFilter.cpp +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_PreprocessFilter.cpp @@ -7,7 +7,7 @@ using namespace iknow::csvdata; using namespace std; // ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase) -void iKnow_KB_PreprocessFilter::ImportFromCSV(string prepro_csv, CSV_DataGenerator& kb) +bool iKnow_KB_PreprocessFilter::ImportFromCSV(string prepro_csv, CSV_DataGenerator& kb) { ifstream ifs = ifstream(prepro_csv, ifstream::in); if (ifs.is_open()) { @@ -33,18 +33,9 @@ void iKnow_KB_PreprocessFilter::ImportFromCSV(string prepro_csv, CSV_DataGenerat kb.kb_prepro.push_back(filter); // Set sc = filter.%Save() // $$$IKModelCheck(sc, stream.Filename, count, line) } + ifs.close(); + return true; } - else { - cerr << "Error opening file: " << prepro_csv << " Language=\"" << kb.GetName() << "\"" << endl; - } - ifs.close(); -} - -iKnow_KB_PreprocessFilter::iKnow_KB_PreprocessFilter() -{ -} - - -iKnow_KB_PreprocessFilter::~iKnow_KB_PreprocessFilter() -{ -} + cerr << "Error opening file: " << prepro_csv << " Language=\"" << kb.GetName() << "\"" << endl; + return false; +} \ No newline at end of file diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_PreprocessFilter.h b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_PreprocessFilter.h index f2338eb9..6a5e8265 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_PreprocessFilter.h +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_PreprocessFilter.h @@ -9,7 +9,7 @@ namespace iknow { { public: // ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase) - static void ImportFromCSV(std::string prepro_csv, CSV_DataGenerator& kb); + static bool ImportFromCSV(std::string prepro_csv, CSV_DataGenerator& kb); std::string InputToken; // Property InputToken As %String(MAXLEN = 256, XMLPROJECTION = "ATTRIBUTE")[Required]; @@ -21,8 +21,8 @@ namespace iknow { // Index KnowledgebaseIndex On(Knowledgebase, Precedence); - iKnow_KB_PreprocessFilter(); - ~iKnow_KB_PreprocessFilter(); + iKnow_KB_PreprocessFilter() {} + ~iKnow_KB_PreprocessFilter() {} }; } } \ No newline at end of file diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Regex.cpp b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Regex.cpp index 9318d19a..213d752a 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Regex.cpp +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Regex.cpp @@ -6,7 +6,7 @@ using namespace iknow::csvdata; using namespace std; -void iKnow_KB_Regex::ImportFromCSV(string regex_csv, CSV_DataGenerator& kb) +bool iKnow_KB_Regex::ImportFromCSV(string regex_csv, CSV_DataGenerator& kb) { ifstream ifs = ifstream(regex_csv, ifstream::in); if (ifs.is_open()) { @@ -33,20 +33,10 @@ void iKnow_KB_Regex::ImportFromCSV(string regex_csv, CSV_DataGenerator& kb) } Set regex.Pattern = $PIECE(line, ";", 2) */ - } + ifs.close(); + return true; } - else { - cerr << "Error opening file: " << regex_csv << " Language=\"" << kb.GetName() << "\"" << endl; - } - ifs.close(); -} - -iKnow_KB_Regex::iKnow_KB_Regex() -{ -} - - -iKnow_KB_Regex::~iKnow_KB_Regex() -{ -} + cerr << "Error opening file: " << regex_csv << " Language=\"" << kb.GetName() << "\"" << endl; + return false; +} \ No newline at end of file diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Regex.h b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Regex.h index 5df6f9e6..7b52f6ab 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Regex.h +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Regex.h @@ -10,7 +10,7 @@ namespace iknow { { public: // ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase) - static void ImportFromCSV(std::string regex_csv, CSV_DataGenerator& kb); + static bool ImportFromCSV(std::string regex_csv, CSV_DataGenerator& kb); /* Set name = $PIECE(line, ";", 1) @@ -22,8 +22,8 @@ namespace iknow { } std::string name, Pattern; - iKnow_KB_Regex(); - ~iKnow_KB_Regex(); + iKnow_KB_Regex() {} + ~iKnow_KB_Regex() {} }; } } \ No newline at end of file diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Rule.cpp b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Rule.cpp index 05c0e25d..2e4474db 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Rule.cpp +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Rule.cpp @@ -10,7 +10,7 @@ using namespace iknow::csvdata; using namespace std; -void iKnow_KB_Rule::ImportFromCSV(string rules_csv, CSV_DataGenerator& kb) +bool iKnow_KB_Rule::ImportFromCSV(string rules_csv, CSV_DataGenerator& kb) { ifstream ifs = ifstream(rules_csv, ifstream::in); if (ifs.is_open()) { @@ -77,7 +77,7 @@ void iKnow_KB_Rule::ImportFromCSV(string rules_csv, CSV_DataGenerator& kb) SBeginObj.Type = "typeAttribute"; // Set labelObj.Type = "typeAttribute" SBeginObj.Attributes = ""; // Set labelObj.Attributes = "" // Set phaselist = "" - for_each(SBeginPhases.begin(), SBeginPhases.end(), [&SBeginObj](string phase){ SBeginObj.PhaseList += (SBeginObj.PhaseList.size() ? "," + phase : phase); }); // For idxPhases = $listlength(SBeginPhases) :-1 : 1 Set phaselist = $list(SBeginPhases, idxPhases)_$Select($Length(phaselist) : ","_phaselist, 1 : "") + for_each(SBeginPhases.begin(), SBeginPhases.end(), [&SBeginObj](string phase) { SBeginObj.PhaseList += (SBeginObj.PhaseList.size() ? "," + phase : phase); }); // For idxPhases = $listlength(SBeginPhases) :-1 : 1 Set phaselist = $list(SBeginPhases, idxPhases)_$Select($Length(phaselist) : ","_phaselist, 1 : "") // Set labelObj.PhaseList = phaselist kb.kb_labels.push_back(SBeginObj); // Set sc = labelObj.%Save() // If 'sc throw ##class(%Exception.StatusException).CreateFromStatus(sc) @@ -86,14 +86,13 @@ void iKnow_KB_Rule::ImportFromCSV(string rules_csv, CSV_DataGenerator& kb) SEndObj.Name = "SEnd"; SEndObj.Type = "typeAttribute"; SEndObj.Attributes = ""; - for_each(SEndPhases.begin(), SEndPhases.end(), [&SEndObj](string phase){ SEndObj.PhaseList += (SEndObj.PhaseList.size() ? "," + phase : phase); }); + for_each(SEndPhases.begin(), SEndPhases.end(), [&SEndObj](string phase) { SEndObj.PhaseList += (SEndObj.PhaseList.size() ? "," + phase : phase); }); kb.kb_labels.push_back(SEndObj); - - } - else { - cerr << "Error opening file: " << rules_csv << " Language=\"" << kb.GetName() << "\"" << endl; + ifs.close(); + return true; } - ifs.close(); + cerr << "Error opening file: " << rules_csv << " Language=\"" << kb.GetName() << "\"" << endl; + return false; } void str_subsitute(string& text, const string str_find, const string str_replace) @@ -145,7 +144,7 @@ void AddLabelToLexrep(CSV_DataGenerator& kb, string& token, string& label) lexrep.Token = token; // Set lexrep.Token = token lexrep.Labels = label + ";"; // Set lexrep.Labels = label _ ";" // Set lexrep.Knowledgebase = kb - kb.lexrep_index[lexrep.Token] = kb.kb_lexreps.size(); // new index for lexrep + kb.lexrep_index[lexrep.Token] = (int) kb.kb_lexreps.size(); // new index for lexrep kb.kb_lexreps.push_back(lexrep); // Set sc = lexrep.%Save() // If 'sc throw ##class(%Exception.StatusException).CreateFromStatus(sc) } @@ -220,13 +219,4 @@ std::string iKnow_KB_Rule::TransformRulePattern(string& pattern, string& phase, transformed_pattern += *it; } return transformed_pattern; -} - -iKnow_KB_Rule::iKnow_KB_Rule() -{ -} - - -iKnow_KB_Rule::~iKnow_KB_Rule() -{ -} +} \ No newline at end of file diff --git a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Rule.h b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Rule.h index 366403a7..279347aa 100644 --- a/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Rule.h +++ b/modules/compiler/iKnowLanguageCompiler/iKnow_KB_Rule.h @@ -16,7 +16,7 @@ namespace iknow { { public: // ClassMethod ImportFromStream(stream As %CharacterStream, kb As Knowledgebase) - static void ImportFromCSV(std::string rule_csv, CSV_DataGenerator& kb); + static bool ImportFromCSV(std::string rule_csv, CSV_DataGenerator& kb); std::string TransformRulePattern(std::string& csv_rule_input, std::string& csv_phase, CSV_DataGenerator& kb, newLabels_type &newLabels, newLabelsIndex_type &newLabelsIndex, SPhases_type& SBegin, SPhases_type& SEnd); @@ -32,8 +32,8 @@ namespace iknow { int Precedence; std::string Phase; - iKnow_KB_Rule(); - ~iKnow_KB_Rule(); + iKnow_KB_Rule() {} + ~iKnow_KB_Rule() {} }; } } \ No newline at end of file diff --git a/modules/compiler/iKnowLanguageCompiler/languagecompiler.mak b/modules/compiler/iKnowLanguageCompiler/languagecompiler.mak new file mode 100644 index 00000000..43a58b45 --- /dev/null +++ b/modules/compiler/iKnowLanguageCompiler/languagecompiler.mak @@ -0,0 +1,26 @@ +MAKEPATH := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST)))).) +ROOT_DIR = $(realpath $(MAKEPATH)/../../..) + +#TODO: TRW - generalize! +PLATFORM = $(IKNOWPLAT) + +CPP_LANGUAGE = 14 + +INCLUDEDIRS = $(ROOT_DIR)/modules/shell/src $(ROOT_DIR)/modules/shell/src/SDK/headers $(ROOT_DIR)/modules/base/src/headers $(ROOT_DIR)/modules/core/src/headers $(ROOT_DIR)/shared/System/unix $(ROOT_DIR)/shared/System $(ROOT_DIR)/shared/Utility $(ROOT_DIR)/kernel/common/h $(ICUDIR)/include + +SOURCES = $(ROOT_DIR)/modules/compiler/iKnowLanguageCompiler/*.cpp + +LOCATION = bin + +CREATEEXECUTABLE = 1 + +PROJECT = iknowlanguagecompiler + +OBJECTFLAGS = -D_DOUBLEBYTE -DCACHE_COM_DISABLE + +LIBRARIES = iknowbase iknowcore iknowshell + +LIBRARYDIRS = $(ROOT_DIR)/kit/$(PLATFORM)/$(MODE)/bin + +include $(ROOT_DIR)/build/make/master.mak + diff --git a/modules/core/src/IkLabel.cpp b/modules/core/src/IkLabel.cpp index f647fbd5..ec68605e 100644 --- a/modules/core/src/IkLabel.cpp +++ b/modules/core/src/IkLabel.cpp @@ -12,9 +12,6 @@ using namespace iknow::core; using namespace iknow::base; -// static definition: -const IkLabel::LabelTypeMap IkLabel::label_type_map_; - IkLabel::IkLabel() { m_nType = Unknown; @@ -26,3 +23,44 @@ IkLabel::~IkLabel() { } +struct LabelTypeMap { + LabelTypeMap() { + map.insert(TypeMap::value_type("typeNonRelevant", iknow::core::IkLabel::Nonrelevant)); + map.insert(TypeMap::value_type("typeAmbiguous", iknow::core::IkLabel::Ambiguous)); + map.insert(TypeMap::value_type("typeAttribute", iknow::core::IkLabel::Attribute)); + map.insert(TypeMap::value_type("typeConcept", iknow::core::IkLabel::Concept)); + map.insert(TypeMap::value_type("typeRelation", iknow::core::IkLabel::Relation)); + map.insert(TypeMap::value_type("typeBeginConcept", iknow::core::IkLabel::BeginConcept)); + map.insert(TypeMap::value_type("typeEndConcept", iknow::core::IkLabel::EndConcept)); + map.insert(TypeMap::value_type("typeBeginEndConcept", iknow::core::IkLabel::BeginEndConcept)); + map.insert(TypeMap::value_type("typeBeginRelation", iknow::core::IkLabel::BeginRelation)); + map.insert(TypeMap::value_type("typeEndRelation", iknow::core::IkLabel::EndRelation)); + map.insert(TypeMap::value_type("typeBeginEndRelation", iknow::core::IkLabel::BeginEndRelation)); + map.insert(TypeMap::value_type("typeLiteral", iknow::core::IkLabel::Literal)); + map.insert(TypeMap::value_type("typeOther", iknow::core::IkLabel::Other)); + map.insert(TypeMap::value_type("typePathRelevant", iknow::core::IkLabel::PathRelevant)); + } + iknow::core::IkLabel::Type Lookup(const std::string& type_name) const { + TypeMap::const_iterator i = map.find(type_name); + return ((i == map.end()) ? iknow::core::IkLabel::Unknown : i->second); + } + std::string Lookup(const iknow::core::IkLabel::Type label_type) const { + for (TypeMap::const_iterator it = map.begin(); it != map.end(); ++it) { // lookup the label type + if (it->second == label_type) return it->first; + } + return std::string("typeUnknown"); // Unknown type + } + typedef std::map TypeMap; + TypeMap map; +}; + +const static LabelTypeMap label_type_map_; + +iknow::core::IkLabel::Type IkLabel::TypeStringToType(const std::string& typeString) { + return label_type_map_.Lookup(typeString); +} +std::string IkLabel::LabelTypeToString(iknow::core::IkLabel::Type label_type) { + return label_type_map_.Lookup(label_type); +} + + diff --git a/modules/core/src/headers/IkLabel.h b/modules/core/src/headers/IkLabel.h index 0850d1ac..6e17f313 100644 --- a/modules/core/src/headers/IkLabel.h +++ b/modules/core/src/headers/IkLabel.h @@ -90,49 +90,12 @@ namespace iknow if (m_nType > other.m_nType) return false; return m_strName < other.m_strName; } - static iknow::core::IkLabel::Type TypeStringToType(const std::string& typeString) { - const static LabelTypeMap label_type_map; - return label_type_map_.Lookup(typeString); - } - static std::string LabelTypeToString(iknow::core::IkLabel::Type label_type) { - const static LabelTypeMap label_type_map; - return label_type_map_.Lookup(label_type); - } + static iknow::core::IkLabel::Type TypeStringToType(const std::string& typeString); + static std::string LabelTypeToString(iknow::core::IkLabel::Type label_type); private: - struct LabelTypeMap { - LabelTypeMap() { - map.insert(TypeMap::value_type("typeNonRelevant", iknow::core::IkLabel::Nonrelevant)); - map.insert(TypeMap::value_type("typeAmbiguous", iknow::core::IkLabel::Ambiguous)); - map.insert(TypeMap::value_type("typeAttribute", iknow::core::IkLabel::Attribute)); - map.insert(TypeMap::value_type("typeConcept", iknow::core::IkLabel::Concept)); - map.insert(TypeMap::value_type("typeRelation", iknow::core::IkLabel::Relation)); - map.insert(TypeMap::value_type("typeBeginConcept", iknow::core::IkLabel::BeginConcept)); - map.insert(TypeMap::value_type("typeEndConcept", iknow::core::IkLabel::EndConcept)); - map.insert(TypeMap::value_type("typeBeginEndConcept", iknow::core::IkLabel::BeginEndConcept)); - map.insert(TypeMap::value_type("typeBeginRelation", iknow::core::IkLabel::BeginRelation)); - map.insert(TypeMap::value_type("typeEndRelation", iknow::core::IkLabel::EndRelation)); - map.insert(TypeMap::value_type("typeBeginEndRelation", iknow::core::IkLabel::BeginEndRelation)); - map.insert(TypeMap::value_type("typeLiteral", iknow::core::IkLabel::Literal)); - map.insert(TypeMap::value_type("typeOther", iknow::core::IkLabel::Other)); - map.insert(TypeMap::value_type("typePathRelevant", iknow::core::IkLabel::PathRelevant)); - } - iknow::core::IkLabel::Type Lookup(const std::string& type_name) const { - TypeMap::const_iterator i = map.find(type_name); - return ((i == map.end()) ? iknow::core::IkLabel::Unknown : i->second); - } - std::string Lookup(const iknow::core::IkLabel::Type label_type) const { - for (TypeMap::const_iterator it = map.begin(); it != map.end(); ++it) { // lookup the label type - if (it->second == label_type) return it->first; - } - return std::string("typeUnknown"); // Unknown type - } - typedef std::map TypeMap; - TypeMap map; - }; iknow::base::String m_strName; Type m_nType; - const static LabelTypeMap label_type_map_; }; const IkLabel& BeginLabel(); diff --git a/modules/shell/src/RawBlock.h b/modules/shell/src/RawBlock.h index d30122d0..9684b758 100644 --- a/modules/shell/src/RawBlock.h +++ b/modules/shell/src/RawBlock.h @@ -124,21 +124,18 @@ namespace iknow { template size_t SpaceRequiredForString(size_t length) { - typedef CountedString CountedStringT; - if (length == 0) return AlignmentNeeded() + sizeof(CountedStringT); - return AlignmentNeeded() + sizeof(CountedStringT) - (sizeof(CountedStringT().c)) + (sizeof(CountedStringT().c) * length); + typedef CountedString CountedStringT; + if (length == 0) return AlignmentNeeded() + sizeof(CountedStringT); + return AlignmentNeeded() + sizeof(CountedStringT) - (sizeof(CountedStringT().c)) + (sizeof(CountedStringT().c) * length); } - - template size_t SpaceRequired() { - return sizeof(T); + return sizeof(T); } - template size_t AlignmentNeeded() { - size_t alignment = alignof(T); - return allocated_ % alignment ? alignment - allocated_ % alignment : 0; + size_t alignment = alignof(T); + return allocated_ % alignment ? alignment - allocated_ % alignment : 0; } size_t BytesUsed() { return allocated_; } @@ -146,21 +143,24 @@ namespace iknow { #ifdef INCLUDE_GENERATE_IMAGE_CODE void generate_image(std::string& dir_name, std::string kb_name) { // data logging - errno_t err; - FILE *stream; - - std::string file_name = dir_name + "/kb_" + kb_name + "_data.h"; - if ((err = fopen_s(&stream, file_name.c_str(), "wt")) == 0) { + std::string image_file(dir_name + "/kb_" + kb_name + "_data.h"); + std::ofstream ofs = std::ofstream(image_file); // Do ##class(Util).OutputToFile(dir _ "/OneStateMap.inl") + if (ofs.is_open()) { std::string data_name = "kb_" + kb_name + "_data[]"; - fprintf(stream, "const unsigned char %s = { // memory block representing KB data", data_name.c_str()); + ofs << "const unsigned char " << data_name << " = { // memory block representing KB data"; // fprintf(stream, "const unsigned char %s = { // memory block representing KB data", data_name.c_str()); + ofs << hex; // change to hexadecimal format for writing size_t data_size = BytesUsed(); size_t cnt = static_cast(0); for (; cnt < data_size - static_cast(1); ++cnt) { - if (!(cnt % static_cast(8))) fprintf(stream, "\n\t"); - fprintf(stream, "0x%x, ", *(raw_.Begin()+cnt)); + if (!(cnt % static_cast(8))) ofs << "\n\t"; // fprintf(stream, "\n\t"); + ofs << "0x" << (int)(*(raw_.Begin() + cnt)) << ", "; // fprintf(stream, "0x%x, ", *(raw_.Begin() + cnt)); } - fprintf(stream, "0x%x\n};\n", *(raw_.Begin() + cnt)); - fclose(stream); + ofs << "0x" << (int)(*(raw_.Begin() + cnt)) << "\n};\n"; // fprintf(stream, "0x%x\n};\n", *(raw_.Begin() + cnt)); + ofs.close(); + } else { + std::string error_message("Cannot open " + image_file + " for writing !"); + std::cerr << error_message << std::endl; + throw ExceptionFrom(error_message); } } #endif