From 67f363cf3b7925755b70e3ee9985739f3e906cd0 Mon Sep 17 00:00:00 2001 From: psteiwer Date: Tue, 3 Dec 2019 20:24:54 +0100 Subject: [PATCH 01/11] Add GetVersion method --- AnalyzeThis/Utils.cls | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/AnalyzeThis/Utils.cls b/AnalyzeThis/Utils.cls index 8a8a7ec..7ad8dad 100644 --- a/AnalyzeThis/Utils.cls +++ b/AnalyzeThis/Utils.cls @@ -1,6 +1,11 @@ Class AnalyzeThis.Utils { +ClassMethod GetVersion() As %String +{ + Quit "v1.1.3" +} + ClassMethod GetNextRegex(ByRef pMatch,ByRef pResult,pIncludeTerm As %Boolean = 1) As %Boolean { Set pResult="" From d487e77c5c8fb07edde5903ec36fe074a8b8926f Mon Sep 17 00:00:00 2001 From: psteiwer Date: Tue, 3 Dec 2019 20:32:16 +0100 Subject: [PATCH 02/11] Add version to book cover --- AnalyzeThis/Installer.cls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AnalyzeThis/Installer.cls b/AnalyzeThis/Installer.cls index 983ac7e..06c74c6 100644 --- a/AnalyzeThis/Installer.cls +++ b/AnalyzeThis/Installer.cls @@ -16,7 +16,7 @@ ClassMethod RunInstaller(pRootDir) Set tPage="AnalyzeThis.UI.CSVImport.zen" Set tItem.href=$system.CSP.GetPortalApp($namespace,tPage) _ tPage Set tItem.title="Analyze This" - Set tItem.bookCover="{background: {style:'background:white;',src:'covers/AnalyzeThis_Cover.png'},header: {text:'$type',style:'display: none;'},title: {text:'$title',style:'display: none;'},image: {style:'display: none;',src:'deepsee/ds2_globe_44.png',imageStyle:'width:64px;height:64px;'},subtitle: {style:'display: none;'},footer: {text:'$owner',style:'display: none;'}}" + Set tItem.bookCover="{background: {style:'background:white;',src:'covers/AnalyzeThis_Cover.png'},header: {text:'$type',style:'display: none;'},title: {text:'$title',style:'display: none;'},image: {style:'display: none;',src:'deepsee/ds2_globe_44.png',imageStyle:'width:64px;height:64px;'},subtitle: {text:'"_##class(AnalyzeThis.Utils).GetVersion()_"',style:'font-size:9px;top:179px;'},footer: {text:'$owner',style:'display: none;'}}" Set tSC=tItem.%Save() Quit tSC From a02397631a3fe040b050851d3416ba3efaf74615 Mon Sep 17 00:00:00 2001 From: psteiwer Date: Thu, 12 Dec 2019 14:44:28 +0100 Subject: [PATCH 03/11] Don't set chartToggle to chart if type is pivot --- AnalyzeThis/Dashboard/AutoPivot.cls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AnalyzeThis/Dashboard/AutoPivot.cls b/AnalyzeThis/Dashboard/AutoPivot.cls index 1aeb14d..6664b12 100644 --- a/AnalyzeThis/Dashboard/AutoPivot.cls +++ b/AnalyzeThis/Dashboard/AutoPivot.cls @@ -659,7 +659,7 @@ ClassMethod AddWidget(pCubeName As %String, pname As %String, wtype As %String, Kill Properties Set Properties=##class(%ArrayOfDataTypes).%New() Do Properties.SetAt(1,"analyzer") - Do Properties.SetAt("chart","chartToggle") + Do:wtype'="pivot" Properties.SetAt("chart","chartToggle") Do Properties.SetAt(1,"print") If (wtype="time") Do Properties.SetAt(mdx,"MDX") If (wtype="barChart") { From af75b37243e7865ce8205dd6eca4d43ef00975fa Mon Sep 17 00:00:00 2001 From: psteiwer Date: Thu, 12 Mar 2020 15:33:05 -0400 Subject: [PATCH 04/11] Modify SQLFieldName for columns with SQL Reserved Words Fixes #43 --- AnalyzeThis/Generator.cls | 574 +++++++++++++++++++------------------- 1 file changed, 289 insertions(+), 285 deletions(-) diff --git a/AnalyzeThis/Generator.cls b/AnalyzeThis/Generator.cls index f4e1071..aa26475 100644 --- a/AnalyzeThis/Generator.cls +++ b/AnalyzeThis/Generator.cls @@ -1,285 +1,289 @@ -Class AnalyzeThis.Generator -{ - -/// This method will go through the entire generation process. Takes in a CSV file -ClassMethod GenerateAll(pSourceType, pSource, pLineSize, pHasHeaders, pCubeName, pPropertyJSONStreamId, pDataJSONStreamId) As %Status -{ - Try { - // Preparing JSON - Do ..UpdateTracking(1,"Working...") - If pSourceType="CSV" { - Set tSC=##class(AnalyzeThis.UI.Dialog.CSVImport).CSVToJSON(pSource,.tPropertyJSONStreamId,.tDataJSONStreamId,pLineSize,pHasHeaders) - } ElseIf pSourceType="SQL" { - Set tSC=##class(AnalyzeThis.Utils).SQLToJSON(pSource,.tPropertyJSONStreamId,.tDataJSONStreamId,pLineSize) - } ElseIf pSourceType="Class" { - // Get the SQL table name and then pass through SQL method - Set sqlQuery=##class(AnalyzeThis.Utils).ClassToQuery(pSource) - Set tSC=##class(AnalyzeThis.Utils).SQLToJSON(sqlQuery,.tPropertyJSONStreamId,.tDataJSONStreamId,pLineSize) - } - // Probably should provide a way to clean these instead of deleting them here - If pPropertyJSONStreamId'="" { - // Delete this stream if pPropertyJSONStreamId is populated, it may have updates - Do ##class(%Stream.FileBinary).%DeleteId(tPropertyJSONStreamId) - } - If pDataJSONStreamId'="" { - // This may contain "preview mode" data. Delete it - Do ##class(%Stream.FileBinary).%DeleteId(pDataJSONStreamId) - } - Set tPropertyJSONStreamId=pPropertyJSONStreamId - Do ..UpdateTracking(1,tSC) - If $$$ISERR(tSC) $$$ThrowStatus(tSC) - - Set tCubeName=$select($G(pCubeName)'="":pCubeName,1:##class(AnalyzeThis.Generator).GenerateCubeName(pFileName)) - - // Generating Source Class - Do ..UpdateTracking(2,"Working...") - Set tSC=##class(AnalyzeThis.Generator).GenerateSourceClass(tCubeName,tPropertyJSONStreamId) - Do ..UpdateTracking(2,tSC) - If $$$ISERR(tSC) $$$ThrowStatus(tSC) - - // Importing Data - Do ..UpdateTracking(3,"Working...") - Set tSC=##class(AnalyzeThis.Generator).ImportJSONData(tPropertyJSONStreamId, tDataJSONStreamId, tCubeName) - Do ..UpdateTracking(3,tSC) - If $$$ISERR(tSC) $$$ThrowStatus(tSC) - - // Generating Cube - Do ..UpdateTracking(4,"Working...") - Set tSC=##class(AnalyzeThis.Generator).GenerateCube(tCubeName,tPropertyJSONStreamId) - Do ..UpdateTracking(4,tSC) - If $$$ISERR(tSC) $$$ThrowStatus(tSC) - - - // Populating Cube - Do ..UpdateTracking(5,"Working...") - Set tSC=##class(%DeepSee.Utils).%BuildCube(tCubeName,,0) - Do ..UpdateTracking(5,tSC) - If $$$ISERR(tSC) $$$ThrowStatus(tSC) - - // Generating Sample Dashboard - Do ##class(AnalyzeThis.UI.CSVImport).AddDetails(tCubeName,pSourceType,pSource) - - Do ..UpdateTracking(6,"Working...") - Set tSC=##class(AnalyzeThis.Generator).GenerateDash(tCubeName) - Do ..UpdateTracking(6,tSC) - If $$$ISERR(tSC) $$$ThrowStatus(tSC) - } Catch ex { - Set tSC=ex.AsStatus() - } - - // Always clean up streams - If tPropertyJSONStreamId'="" { - Do ##class(%Stream.FileBinary).%DeleteId(tPropertyJSONStreamId) - } - If tDataJSONStreamId'="" { - Do ##class(%Stream.FileBinary).%DeleteId(tDataJSONStreamId) - } - Quit tSC -} - -ClassMethod UpdateTracking(pStep, pMsg) -{ - Set ^AnalyzeThis.GenerateTracking($j,pStep)=pMsg - Quit -} - -/// Given a file name, output the resulting cube name -ClassMethod GenerateCubeName(pFileName) As %String -{ - Set tName=$zstrip($replace($piece(##class(%File).GetFilename(pFileName),".",1),"DEEPSEE",""),"*PCW") - - Quit tName -} - -/// Given a cube name and Property Stream ID, generate a source class -ClassMethod GenerateSourceClass(pCubeName, pPropertyJSONStreamId) As %Status -{ - Set tSC=$$$OK - - Set tSC=##class(AnalyzeThis.Dashboard.Utils).DeleteAll(pCubeName) - - // Take file from request - Set tSC=##class(AnalyzeThis.Generator).JSONToClass(pPropertyJSONStreamId,"AnalyzeThis.Generated."_pCubeName) - - Quit tSC -} - -/// Given a Property Stream ID and a Class Name, generate a Class with the given properties and name -ClassMethod JSONToClass(propertiesJSON As %String, pClassName As %String) As %Status -{ - //Use propertiesJSON to build class - //Populate with dataJSON - If ##class(%Dictionary.ClassDefinition).%ExistsId(pClassName) { - Set tSC=##class(%Dictionary.ClassDefinition).%DeleteId(pClassName) - } - Set class=##class(%Dictionary.ClassDefinition).%New() - Set class.Name=pClassName - Set class.Super="%Persistent" - Set tempStream=##class(%Stream.FileBinary).%OpenId(propertiesJSON) - Set dynamicProperties={}.%FromJSON(tempStream) - Set iter=dynamicProperties.Display.%GetIterator() - While iter.%GetNext(.key,.value) { - If dynamicProperties.Ignore.%Get(key) { - Continue - } - Set prop=##class(%Dictionary.PropertyDefinition).%New() - Set prop.Name=value - Set prop.Type=dynamicProperties.Type.%Get(key) - Set prop.SqlColumnNumber=key+2 - If prop.Type="%String" { - Do prop.Parameters.SetAt("","MAXLEN") - } ElseIf prop.Type="%Integer" { - Set intformat=dynamicProperties.IntFormat.%Get(key) - Set:intformat'="" prop.Type=intformat - - If intformat="%Library.Currency" { - // Add setter method to strip currency signs if included - Set method=##class(%Dictionary.MethodDefinition).%New(class.Name_"||"_prop.Name_"Set") - Set method.Name=prop.Name_"Set" - Set method.FormalSpec="pVal" - Set method.ReturnType="%Status" - - Do method.Implementation.WriteLine(" Try { Set pVal=$zstrip(pVal,""*P"",,"".,"") }") - Do method.Implementation.WriteLine(" Catch ex { }") - Do method.Implementation.WriteLine(" Set i%"_prop.Name_"=pVal") - Do method.Implementation.WriteLine(" Quit $$$OK") - - Do class.Methods.Insert(method) - - } - } ElseIf ((prop.Type="%Date") || (prop.Type="%TimeStamp")) { - Set tempFormat=dynamicProperties.DateFormat.%Get(key) - If tempFormat=5 { - //Cast back as string until we support partial dates - Set prop.Type="%String" - } ElseIf tempFormat=30 { - // No need to convert since this is in $H already - } Else { - //Create setter method to translate to $h - Set method=##class(%Dictionary.MethodDefinition).%New(class.Name_"||"_prop.Name_"Set") - Set method.Name=prop.Name_"Set" - Set method.FormalSpec="pVal" - Set method.ReturnType="%Status" - - If (prop.Type="%Date") { - Do method.Implementation.WriteLine(" Try { Set pVal=$zdh(pVal,"_tempFormat_") }") - } Else { - // This is a %TimeStamp type, convert to %Date since %DeepSee.WizardUtils does not handle %TimeStamp - Set prop.Type="%Date" - Do method.Implementation.WriteLine(" Try { Set pVal=+$zdth(pVal,"_tempFormat_") }") - } - Do method.Implementation.WriteLine(" Catch ex { }") - Do method.Implementation.WriteLine(" Set i%"_prop.Name_"=pVal") - Do method.Implementation.WriteLine(" Quit $$$OK") - - Do class.Methods.Insert(method) - } - } - //If dynamicProperties.Include.%Get(key)="true" { - Do class.Properties.Insert(prop) - //} - } - - // Set global name directly to allow for global mapping of AnalyzeThis.* - Set param=##class(%Dictionary.ParameterDefinition).%New() - Set param.Name="DEFAULTGLOBAL" - // Unique up to 32 characters, only take first 16 chars of class name to avoid confusion - Set param.Default="^AnalyzeThis.G."_$extract($piece(pClassName,".",*),1,16) - Do class.Parameters.Insert(param) - - Set tSC=class.%Save() - Set tSC=$system.OBJ.Compile(pClassName,"fck /displayerror=0 /displaylog=0") - - Quit tSC -} - -/// Given a Property Stream ID, Data Stream ID, and a CubeName - populate the source class with data from the stream -ClassMethod ImportJSONData(propertyJSONStreamId As %String, dataJSONStreamId As %String, pCubeName As %String, pCleanFirst As %Boolean = 1) As %Status -{ - Set tSC=$$$OK - - Set tClassName="AnalyzeThis.Generated."_pCubeName - - If pCleanFirst { - Set tSC=$classmethod(tClassName,"%DeleteExtent") - } - - Set dataJSON=##class(%Stream.FileBinary).%OpenId(dataJSONStreamId) - Set propertyJSON=##class(%Stream.FileBinary).%OpenId(propertyJSONStreamId) - Set dataArray=[].%FromJSON(dataJSON) - Set propertyObj={}.%FromJSON(propertyJSON) - Set iterArray=dataArray.%GetIterator() - While iterArray.%GetNext(.key,.val) { - Set obj=$classmethod(tClassName,"%New") - Set iterObj=dataArray.%Get(key).%GetIterator() - While iterObj.%GetNext(.key2,.val2) { - //Set:propertyObj.Include.%Get(propertyObj.%Get(key2))="true" $property(obj,propertyObj.Display.%Get(propertyObj.%Get(key2)))=val2 - Set $property(obj,propertyObj.Display.%Get(propertyObj.Properties.%Get(key2)))=val2 - } - Set tSC= obj.%Save() - } - - Quit tSC -} - -/// Given a Cube name and a Property Stream ID, generate a cube -ClassMethod GenerateCube(pCubeName, pPropertyJSONStreamId) As %Status -{ - Set tSourceClass="AnalyzeThis.Generated."_pCubeName - Set tCubeClass=tSourceClass_"Cube" - Set tCubeName=pCubeName - - //Generate DeepSee Cube - //Set st=##class(%DeepSee.WizardUtils).%GenerateCubeDefinition("AnalyzeThis.Generated."_tSourceClass_".Record",tCubeName,tCubeClass) - Set st=##class(%DeepSee.WizardUtils).%GenerateCubeDefinition(tSourceClass,tCubeName,tCubeClass) - If $$$ISERR(st) Quit st - - //Compile new cube class - Set st=$System.OBJ.Compile(tCubeClass,"fck /displayerror=0 /displaylog=0") - If $$$ISERR(st) Quit st - - Set tempStream=##class(%Stream.FileBinary).%OpenId(pPropertyJSONStreamId) - Set propertyObj={}.%FromJSON(tempStream) - - //Disable items as needed - Set model=$zobjclassmethod(tCubeClass,"%GetModel") - For i=1:1:model.dimensions.Count() { - Set tempPos=propertyObj.Properties.%Get(model.dimensions.GetAt(i).name) - If propertyObj.Include.%Get(tempPos)'="true" { - Set model.dimensions.GetAt(i).disabled=1 - } - } - For i=1:1:model.measures.Count() { - If propertyObj.Include.%Get(propertyObj.Properties.%Get(model.measures.GetAt(i).name))'="true" { - Set model.measures.GetAt(i).disabled=1 - } - // make measures searchable - Set model.measures.GetAt(i).searchable=1 - Set prop=##class(%Dictionary.PropertyDefinition).%OpenId(tSourceClass_"||"_model.measures.GetAt(i).sourceProperty) - If $isobject(prop) { - If prop.Type="%Library.Currency" { - Set currency=##class(%SYS.NLS.Locale).%New() - Set model.measures.GetAt(i).formatString=currency.Currency_"#.##;-"_currency.Currency_"#.##;;;" - } - } - } - Set st=##class(%DeepSee.Utils).%SaveCubeDefinition(tCubeClass,,model.description,model) - If $$$ISERR(st) Quit st - - Set st=$System.OBJ.Compile(tCubeClass,"fck /displayerror=0 /displaylog=0") - Quit st -} - -/// Given a cube name, generate a Dashboard -ClassMethod GenerateDash(pCubeName As %String, pVerbose As %Boolean = 0) As %Status -{ - Set st=$$$OK - Set dashName="Generated/Samples for "_pCubeName_".dashboard" - If ##class(%DeepSee.Dashboard.Utils).%DashboardExists(dashName) { - Do ##class(AnalyzeThis.Dashboard.Utils).DeleteDashboards(pCubeName) - } - Do ##class(AnalyzeThis.Dashboard.AutoPivot).GenerateSampleDashboards(pCubeName,pVerbose) - Set st=##class(%DeepSee.Dashboard.Utils).%DashboardExists(dashName) - Quit st -} -} \ No newline at end of file +Class AnalyzeThis.Generator +{ + +/// This method will go through the entire generation process. Takes in a CSV file +ClassMethod GenerateAll(pSourceType, pSource, pLineSize, pHasHeaders, pCubeName, pPropertyJSONStreamId, pDataJSONStreamId) As %Status +{ + Try { + // Preparing JSON + Do ..UpdateTracking(1,"Working...") + If pSourceType="CSV" { + Set tSC=##class(AnalyzeThis.UI.Dialog.CSVImport).CSVToJSON(pSource,.tPropertyJSONStreamId,.tDataJSONStreamId,pLineSize,pHasHeaders) + } ElseIf pSourceType="SQL" { + Set tSC=##class(AnalyzeThis.Utils).SQLToJSON(pSource,.tPropertyJSONStreamId,.tDataJSONStreamId,pLineSize) + } ElseIf pSourceType="Class" { + // Get the SQL table name and then pass through SQL method + Set sqlQuery=##class(AnalyzeThis.Utils).ClassToQuery(pSource) + Set tSC=##class(AnalyzeThis.Utils).SQLToJSON(sqlQuery,.tPropertyJSONStreamId,.tDataJSONStreamId,pLineSize) + } + // Probably should provide a way to clean these instead of deleting them here + If pPropertyJSONStreamId'="" { + // Delete this stream if pPropertyJSONStreamId is populated, it may have updates + Do ##class(%Stream.FileBinary).%DeleteId(tPropertyJSONStreamId) + } + If pDataJSONStreamId'="" { + // This may contain "preview mode" data. Delete it + Do ##class(%Stream.FileBinary).%DeleteId(pDataJSONStreamId) + } + Set tPropertyJSONStreamId=pPropertyJSONStreamId + Do ..UpdateTracking(1,tSC) + If $$$ISERR(tSC) $$$ThrowStatus(tSC) + + Set tCubeName=$select($G(pCubeName)'="":pCubeName,1:##class(AnalyzeThis.Generator).GenerateCubeName(pFileName)) + + // Generating Source Class + Do ..UpdateTracking(2,"Working...") + Set tSC=##class(AnalyzeThis.Generator).GenerateSourceClass(tCubeName,tPropertyJSONStreamId) + Do ..UpdateTracking(2,tSC) + If $$$ISERR(tSC) $$$ThrowStatus(tSC) + + // Importing Data + Do ..UpdateTracking(3,"Working...") + Set tSC=##class(AnalyzeThis.Generator).ImportJSONData(tPropertyJSONStreamId, tDataJSONStreamId, tCubeName) + Do ..UpdateTracking(3,tSC) + If $$$ISERR(tSC) $$$ThrowStatus(tSC) + + // Generating Cube + Do ..UpdateTracking(4,"Working...") + Set tSC=##class(AnalyzeThis.Generator).GenerateCube(tCubeName,tPropertyJSONStreamId) + Do ..UpdateTracking(4,tSC) + If $$$ISERR(tSC) $$$ThrowStatus(tSC) + + + // Populating Cube + Do ..UpdateTracking(5,"Working...") + Set tSC=##class(%DeepSee.Utils).%BuildCube(tCubeName,,0) + Do ..UpdateTracking(5,tSC) + If $$$ISERR(tSC) $$$ThrowStatus(tSC) + + // Generating Sample Dashboard + Do ##class(AnalyzeThis.UI.CSVImport).AddDetails(tCubeName,pSourceType,pSource) + + Do ..UpdateTracking(6,"Working...") + Set tSC=##class(AnalyzeThis.Generator).GenerateDash(tCubeName) + Do ..UpdateTracking(6,tSC) + If $$$ISERR(tSC) $$$ThrowStatus(tSC) + } Catch ex { + Set tSC=ex.AsStatus() + } + + // Always clean up streams + If tPropertyJSONStreamId'="" { + Do ##class(%Stream.FileBinary).%DeleteId(tPropertyJSONStreamId) + } + If tDataJSONStreamId'="" { + Do ##class(%Stream.FileBinary).%DeleteId(tDataJSONStreamId) + } + Quit tSC +} + +ClassMethod UpdateTracking(pStep, pMsg) +{ + Set ^AnalyzeThis.GenerateTracking($j,pStep)=pMsg + Quit +} + +/// Given a file name, output the resulting cube name +ClassMethod GenerateCubeName(pFileName) As %String +{ + Set tName=$zstrip($replace($piece(##class(%File).GetFilename(pFileName),".",1),"DEEPSEE",""),"*PCW") + + Quit tName +} + +/// Given a cube name and Property Stream ID, generate a source class +ClassMethod GenerateSourceClass(pCubeName, pPropertyJSONStreamId) As %Status +{ + Set tSC=$$$OK + + Set tSC=##class(AnalyzeThis.Dashboard.Utils).DeleteAll(pCubeName) + + // Take file from request + Set tSC=##class(AnalyzeThis.Generator).JSONToClass(pPropertyJSONStreamId,"AnalyzeThis.Generated."_pCubeName) + + Quit tSC +} + +/// Given a Property Stream ID and a Class Name, generate a Class with the given properties and name +ClassMethod JSONToClass(propertiesJSON As %String, pClassName As %String) As %Status +{ + //Use propertiesJSON to build class + //Populate with dataJSON + If ##class(%Dictionary.ClassDefinition).%ExistsId(pClassName) { + Set tSC=##class(%Dictionary.ClassDefinition).%DeleteId(pClassName) + } + Set class=##class(%Dictionary.ClassDefinition).%New() + Set class.Name=pClassName + Set class.Super="%Persistent" + Set tempStream=##class(%Stream.FileBinary).%OpenId(propertiesJSON) + Set dynamicProperties={}.%FromJSON(tempStream) + Set iter=dynamicProperties.Display.%GetIterator() + While iter.%GetNext(.key,.value) { + If dynamicProperties.Ignore.%Get(key) { + Continue + } + Set prop=##class(%Dictionary.PropertyDefinition).%New() + Set prop.Name=value + If $SYSTEM.SQL.IsReservedWord(value) { + Set prop.SqlFieldName=value_"_SQLSafe" + } + Set prop.Type=dynamicProperties.Type.%Get(key) + Set prop.SqlColumnNumber=key+2 + If prop.Type="%String" { + Do prop.Parameters.SetAt("","MAXLEN") + } ElseIf prop.Type="%Integer" { + Set intformat=dynamicProperties.IntFormat.%Get(key) + Set:intformat'="" prop.Type=intformat + + If intformat="%Library.Currency" { + // Add setter method to strip currency signs if included + Set method=##class(%Dictionary.MethodDefinition).%New(class.Name_"||"_prop.Name_"Set") + Set method.Name=prop.Name_"Set" + Set method.FormalSpec="pVal" + Set method.ReturnType="%Status" + + Do method.Implementation.WriteLine(" Try { Set pVal=$zstrip(pVal,""*P"",,"".,"") }") + Do method.Implementation.WriteLine(" Catch ex { }") + Do method.Implementation.WriteLine(" Set i%"_prop.Name_"=pVal") + Do method.Implementation.WriteLine(" Quit $$$OK") + + Do class.Methods.Insert(method) + + } + } ElseIf ((prop.Type="%Date") || (prop.Type="%TimeStamp")) { + Set tempFormat=dynamicProperties.DateFormat.%Get(key) + If tempFormat=5 { + //Cast back as string until we support partial dates + Set prop.Type="%String" + } ElseIf tempFormat=30 { + // No need to convert since this is in $H already + } Else { + //Create setter method to translate to $h + Set method=##class(%Dictionary.MethodDefinition).%New(class.Name_"||"_prop.Name_"Set") + Set method.Name=prop.Name_"Set" + Set method.FormalSpec="pVal" + Set method.ReturnType="%Status" + + If (prop.Type="%Date") { + Do method.Implementation.WriteLine(" Try { Set pVal=$zdh(pVal,"_tempFormat_") }") + } Else { + // This is a %TimeStamp type, convert to %Date since %DeepSee.WizardUtils does not handle %TimeStamp + Set prop.Type="%Date" + Do method.Implementation.WriteLine(" Try { Set pVal=+$zdth(pVal,"_tempFormat_") }") + } + Do method.Implementation.WriteLine(" Catch ex { }") + Do method.Implementation.WriteLine(" Set i%"_prop.Name_"=pVal") + Do method.Implementation.WriteLine(" Quit $$$OK") + + Do class.Methods.Insert(method) + } + } + //If dynamicProperties.Include.%Get(key)="true" { + Do class.Properties.Insert(prop) + //} + } + + // Set global name directly to allow for global mapping of AnalyzeThis.* + Set param=##class(%Dictionary.ParameterDefinition).%New() + Set param.Name="DEFAULTGLOBAL" + // Unique up to 32 characters, only take first 16 chars of class name to avoid confusion + Set param.Default="^AnalyzeThis.G."_$extract($piece(pClassName,".",*),1,16) + Do class.Parameters.Insert(param) + + Set tSC=class.%Save() + Set tSC=$system.OBJ.Compile(pClassName,"fck /displayerror=0 /displaylog=0") + + Quit tSC +} + +/// Given a Property Stream ID, Data Stream ID, and a CubeName - populate the source class with data from the stream +ClassMethod ImportJSONData(propertyJSONStreamId As %String, dataJSONStreamId As %String, pCubeName As %String, pCleanFirst As %Boolean = 1) As %Status +{ + Set tSC=$$$OK + + Set tClassName="AnalyzeThis.Generated."_pCubeName + + If pCleanFirst { + Set tSC=$classmethod(tClassName,"%DeleteExtent") + } + + Set dataJSON=##class(%Stream.FileBinary).%OpenId(dataJSONStreamId) + Set propertyJSON=##class(%Stream.FileBinary).%OpenId(propertyJSONStreamId) + Set dataArray=[].%FromJSON(dataJSON) + Set propertyObj={}.%FromJSON(propertyJSON) + Set iterArray=dataArray.%GetIterator() + While iterArray.%GetNext(.key,.val) { + Set obj=$classmethod(tClassName,"%New") + Set iterObj=dataArray.%Get(key).%GetIterator() + While iterObj.%GetNext(.key2,.val2) { + //Set:propertyObj.Include.%Get(propertyObj.%Get(key2))="true" $property(obj,propertyObj.Display.%Get(propertyObj.%Get(key2)))=val2 + Set $property(obj,propertyObj.Display.%Get(propertyObj.Properties.%Get(key2)))=val2 + } + Set tSC= obj.%Save() + } + + Quit tSC +} + +/// Given a Cube name and a Property Stream ID, generate a cube +ClassMethod GenerateCube(pCubeName, pPropertyJSONStreamId) As %Status +{ + Set tSourceClass="AnalyzeThis.Generated."_pCubeName + Set tCubeClass=tSourceClass_"Cube" + Set tCubeName=pCubeName + + //Generate DeepSee Cube + //Set st=##class(%DeepSee.WizardUtils).%GenerateCubeDefinition("AnalyzeThis.Generated."_tSourceClass_".Record",tCubeName,tCubeClass) + Set st=##class(%DeepSee.WizardUtils).%GenerateCubeDefinition(tSourceClass,tCubeName,tCubeClass) + If $$$ISERR(st) Quit st + + //Compile new cube class + Set st=$System.OBJ.Compile(tCubeClass,"fck /displayerror=0 /displaylog=0") + If $$$ISERR(st) Quit st + + Set tempStream=##class(%Stream.FileBinary).%OpenId(pPropertyJSONStreamId) + Set propertyObj={}.%FromJSON(tempStream) + + //Disable items as needed + Set model=$zobjclassmethod(tCubeClass,"%GetModel") + For i=1:1:model.dimensions.Count() { + Set tempPos=propertyObj.Properties.%Get(model.dimensions.GetAt(i).name) + If propertyObj.Include.%Get(tempPos)'="true" { + Set model.dimensions.GetAt(i).disabled=1 + } + } + For i=1:1:model.measures.Count() { + If propertyObj.Include.%Get(propertyObj.Properties.%Get(model.measures.GetAt(i).name))'="true" { + Set model.measures.GetAt(i).disabled=1 + } + // make measures searchable + Set model.measures.GetAt(i).searchable=1 + Set prop=##class(%Dictionary.PropertyDefinition).%OpenId(tSourceClass_"||"_model.measures.GetAt(i).sourceProperty) + If $isobject(prop) { + If prop.Type="%Library.Currency" { + Set currency=##class(%SYS.NLS.Locale).%New() + Set model.measures.GetAt(i).formatString=currency.Currency_"#.##;-"_currency.Currency_"#.##;;;" + } + } + } + Set st=##class(%DeepSee.Utils).%SaveCubeDefinition(tCubeClass,,model.description,model) + If $$$ISERR(st) Quit st + + Set st=$System.OBJ.Compile(tCubeClass,"fck /displayerror=0 /displaylog=0") + Quit st +} + +/// Given a cube name, generate a Dashboard +ClassMethod GenerateDash(pCubeName As %String, pVerbose As %Boolean = 0) As %Status +{ + Set st=$$$OK + Set dashName="Generated/Samples for "_pCubeName_".dashboard" + If ##class(%DeepSee.Dashboard.Utils).%DashboardExists(dashName) { + Do ##class(AnalyzeThis.Dashboard.Utils).DeleteDashboards(pCubeName) + } + Do ##class(AnalyzeThis.Dashboard.AutoPivot).GenerateSampleDashboards(pCubeName,pVerbose) + Set st=##class(%DeepSee.Dashboard.Utils).%DashboardExists(dashName) + Quit st +} + +} From c2fae54553744d659aec4f81ffde661f388c5b51 Mon Sep 17 00:00:00 2001 From: psteiwer Date: Thu, 12 Mar 2020 17:32:12 -0400 Subject: [PATCH 05/11] Convert $zobjclassmethod to $ClassMethod --- AnalyzeThis/Dashboard/MetaDataAnalyze.cls | 408 +++++++++++----------- AnalyzeThis/Generator.cls | 2 +- 2 files changed, 205 insertions(+), 205 deletions(-) diff --git a/AnalyzeThis/Dashboard/MetaDataAnalyze.cls b/AnalyzeThis/Dashboard/MetaDataAnalyze.cls index 2dc5212..df76847 100644 --- a/AnalyzeThis/Dashboard/MetaDataAnalyze.cls +++ b/AnalyzeThis/Dashboard/MetaDataAnalyze.cls @@ -1,204 +1,204 @@ -Class AnalyzeThis.Dashboard.MetaDataAnalyze Extends %DeepSee.Utils -{ - -ClassMethod EvaluateCube(pCube As %String) As %Status -{ - Set tSC=$$$OK - Try { - If (pCube="") { - Quit - } - Set tCube=$$$UPPER(pCube) - Set tFactClass=$$$UPPER(..%GetCubeFactClass(tCube)) - If (tFactClass="") { - Quit - } - - //Delete Existing MetaData for the Cube - Do ##class(AnalyzeThis.Dashboard.MetaData).Delete(tCube) - - Set tSC=..%GetCubeLevels(tCube,.tLevels,1,1) - If $$$ISERR(tSC) Quit - - Set tFactCount=..%GetCubeFactCount(tCube) - - If (tFactCount>0) { - Set n=$O(tLevels("")) - While (n'="") { - Kill tMeta - Set tMeta=##class(AnalyzeThis.Dashboard.MetaData).%New() - Set tMeta.CubeName=tCube - Set tMDXnull="", tMDXavg="", tMDXstd="" - Set tSpec="" - Set tWord=0 - Set tGender=0 - Set ttName=$$$UPPER($LG(tLevels(n),3)) - - Set tType=$LG(tLevels(n),1) - - //Dimensions - If (tType="l") { - Set tSpec="["_$LG(tLevels(n),2)_"].["_$LG(tLevels(n),3)_"].["_$LG(tLevels(n),4)_"]" - Set tMDXnull="SELECT "_tSpec_".&[] ON ROWS FROM ["_tCube_"]" - - Set ttName=$$$UPPER($LG(tLevels(n),4)) - - //Word Recognition - If (ttName["MEASURE") { - Set tWord=1 - } - - If ((ttName["SEX")||(ttName["GENDER")) { - Set tWord=tWord+1 - Set tMeta.DataType="Gender" - } - - If (ttName["STATE") { - Set tWord=tWord+1 - Set tMeta.DataType="State" - } - } - //Measures - ElseIf (tType="m") { - If ($LG(tLevels(n),3)'="%COUNT") { - Set tSpec="["_$LG(tLevels(n),2)_"].["_$LG(tLevels(n),3)_"]" - Set tMDXnull="SELECT FROM ["_tCube_"] WHERE %SEARCH.&["_tSpec_" IS NULL]" - Set tMDXavg="SELECT "_tSpec_".AVG ON 0 FROM ["_tCube_"]" - Do ##class(%DeepSee.Utils).%GetDimensionInfo(tCube,tSpec,.tDim,.tHier,.tLevel) - Do ##class(%DeepSee.Utils).%GetDimensionFact(tCube,tDim,tHier,tLevel,.sqlname) - Set tablename=##class(%DeepSee.Utils).%GetCubeFactTable(tCube) - - Kill sql, statement,st,rs - Set sql="SELECT STDDEV("_sqlname_") As SD,MAX("_sqlname_") As MX ,MIN("_sqlname_") As MN FROM "_tablename - Set statement=##class(%SQL.Statement).%New() - Set st=statement.%Prepare(sql) - If (st) { - Set rs=statement.%Execute() - While (rs.%Next()'=0) { - Set tMeta.STDDEV=rs.%Get("SD") - Set tMeta.Maximum=rs.%Get("MX") - Set tMeta.Minimum=rs.%Get("MN") - } - } - - Set ttName=$$$UPPER($LG(tLevels(n),3)) - //Word Recognition - If (ttName["SCORE") { - Set tWord=1 - } - If (ttName["AVERAGE") { - Set tWord=tWord+1 - } - If (ttName["TOTAL") { - Set tWord=tWord+1 - } - If (ttName["REVENUE") { - Set tWord=tWord+1 - } - } - } - //Relations - ElseIf (tType="r") { - // find null ref for relation - Set tRelation=$LG(tLevels(n),2) - Set tNullRef=$G($$$DeepSeeMetaGLVN("cubes",tCube,"relations",$$$UPPER(tRelation),"nullReplacement")) - If (tNullRef'="") { - Set tSpec="["_tRelation_"]" - Set tMDXnull="SELECT FROM ["_tCube_"] WHERE "_tSpec_".&["_tNullRef_"]" - } - } - - //Get Cardinality - Set tCard=0 - If tSpec'="" { - Do ##class(%DeepSee.Utils).%GetDimensionInfo(tCube,tSpec,.tDim,.tHier,.tLevel) - If tDim=0 { - Set tIndex=$G($$$DeepSeeMetaGLVN("cubes",tCube,"msr#",$LG($$$DeepSeeMetaGLVN("cubes",tCube,"mbr#",tDim,tHier,tLevel),5)))_"Search" - } Else { - Set tIndex=$LG($$$DeepSeeMetaGLVN("cubes",tCube,"mbr#",tDim,tHier,tLevel),5) - } - - Set tValue=$O($$$DeepSeeIndexGLVN(tCube,tIndex,"")) - While (tValue '="") { - Set tCard=tCard + 1 - Set tValue=$O($$$DeepSeeIndexGLVN(tCube,tIndex,tValue)) - } - Set tMeta.Cardinality=tCard - } - - //Execute - Set tMeta.LevelName=$Case(tType, "l":$LG(tLevels(n),4),"m":$LG(tLevels(n),3),:tSpec) - Set tMeta.DimName=$LG(tLevels(n),2) - Set tMeta.LevelType=$Case(tType,"l":"Dimension","m":"Measure","r":"Relation",:tType) - Set tMeta.WordRecognition=tWord - Set listVals=$lb(tMDXnull,tMDXavg,tMDXstd) - For i=1:1:$listlength(listVals) { - Set tMDX=$listget(listVals,i) - If tMDX'="" { - Set tRS=##class(%DeepSee.ResultSet).%New() - Set tSC=tRS.%PrepareMDX(tMDX) - If $$$ISERR(tSC) Quit - Set tSC=tRS.%Execute() - If $$$ISERR(tSC) Quit - Set tVal=tRS.%GetOrdinalValue() - If i=1 { - Set tMeta.Null=(tVal/tFactCount) - } ElseIf i=2 { - Set tMeta.Mean=tVal - } ElseIf i=3 { - Set tMeta.STDDEV=tVal - } - } - } - - Set tSC=tMeta.%Save() - If $$$ISERR(tSC) Quit - - Set n=$O(tLevels(n)) - } - } - - Set namespace=$NAMESPACE - Set tCubeClass=##class(%DeepSee.Utils).%GetCubeClass(tCube) - Set tSC=$zobjclassmethod(tCubeClass,"%GetSourceInfo",.tSourceInfo) - Set tSourceClass=tSourceInfo("sourceClass") - Set class=##class(%Dictionary.ClassDefinition).%OpenId(tSourceClass,,.st) - - If (st) { - For i=1:1:class.Properties.Count() { - Set prop=class.Properties.GetAt(i) - Set tType=prop.Type - If (tType="%Library.Date") { - Kill propName,sql,statement,st,rs - Set propName="%"_prop.Name_"%" - Set sql="UPDATE AnalyzeThis_Dashboard.MetaData (DataType) VALUES ('Date') WHERE LevelName LIKE ?" - Set statement=##class(%SQL.Statement).%New("deffered") - Set st=statement.%Prepare(sql) - If (st) { - Set rs=statement.%Execute(propName) - } - } - If (($$$UPPER(prop.Name)[$$$UPPER("year"))&&(tType'="%Library.Date")) { - Kill propName,sql,statement,st,rs - Set propName="%"_prop.Name_"%" - Set sql="UPDATE AnalyzeThis_Dashboard.MetaData (DataType) VALUES ('Year') WHERE LevelName LIKE ?" - Set statement=##class(%SQL.Statement).%New("deffered") - Set st=statement.%Prepare(sql) - If (st) { - Set rs=statement.%Execute(propName) - } - } - } - } - } Catch(ex) { - Set tSC=ex.AsStatus() - } - - If $$$ISERR(tSC) { - Do $System.Status.DisplayError(tSC) - } - - Quit tSC -} - -} +Class AnalyzeThis.Dashboard.MetaDataAnalyze Extends %DeepSee.Utils +{ + +ClassMethod EvaluateCube(pCube As %String) As %Status +{ + Set tSC=$$$OK + Try { + If (pCube="") { + Quit + } + Set tCube=$$$UPPER(pCube) + Set tFactClass=$$$UPPER(..%GetCubeFactClass(tCube)) + If (tFactClass="") { + Quit + } + + //Delete Existing MetaData for the Cube + Do ##class(AnalyzeThis.Dashboard.MetaData).Delete(tCube) + + Set tSC=..%GetCubeLevels(tCube,.tLevels,1,1) + If $$$ISERR(tSC) Quit + + Set tFactCount=..%GetCubeFactCount(tCube) + + If (tFactCount>0) { + Set n=$O(tLevels("")) + While (n'="") { + Kill tMeta + Set tMeta=##class(AnalyzeThis.Dashboard.MetaData).%New() + Set tMeta.CubeName=tCube + Set tMDXnull="", tMDXavg="", tMDXstd="" + Set tSpec="" + Set tWord=0 + Set tGender=0 + Set ttName=$$$UPPER($LG(tLevels(n),3)) + + Set tType=$LG(tLevels(n),1) + + //Dimensions + If (tType="l") { + Set tSpec="["_$LG(tLevels(n),2)_"].["_$LG(tLevels(n),3)_"].["_$LG(tLevels(n),4)_"]" + Set tMDXnull="SELECT "_tSpec_".&[] ON ROWS FROM ["_tCube_"]" + + Set ttName=$$$UPPER($LG(tLevels(n),4)) + + //Word Recognition + If (ttName["MEASURE") { + Set tWord=1 + } + + If ((ttName["SEX")||(ttName["GENDER")) { + Set tWord=tWord+1 + Set tMeta.DataType="Gender" + } + + If (ttName["STATE") { + Set tWord=tWord+1 + Set tMeta.DataType="State" + } + } + //Measures + ElseIf (tType="m") { + If ($LG(tLevels(n),3)'="%COUNT") { + Set tSpec="["_$LG(tLevels(n),2)_"].["_$LG(tLevels(n),3)_"]" + Set tMDXnull="SELECT FROM ["_tCube_"] WHERE %SEARCH.&["_tSpec_" IS NULL]" + Set tMDXavg="SELECT "_tSpec_".AVG ON 0 FROM ["_tCube_"]" + Do ##class(%DeepSee.Utils).%GetDimensionInfo(tCube,tSpec,.tDim,.tHier,.tLevel) + Do ##class(%DeepSee.Utils).%GetDimensionFact(tCube,tDim,tHier,tLevel,.sqlname) + Set tablename=##class(%DeepSee.Utils).%GetCubeFactTable(tCube) + + Kill sql, statement,st,rs + Set sql="SELECT STDDEV("_sqlname_") As SD,MAX("_sqlname_") As MX ,MIN("_sqlname_") As MN FROM "_tablename + Set statement=##class(%SQL.Statement).%New() + Set st=statement.%Prepare(sql) + If (st) { + Set rs=statement.%Execute() + While (rs.%Next()'=0) { + Set tMeta.STDDEV=rs.%Get("SD") + Set tMeta.Maximum=rs.%Get("MX") + Set tMeta.Minimum=rs.%Get("MN") + } + } + + Set ttName=$$$UPPER($LG(tLevels(n),3)) + //Word Recognition + If (ttName["SCORE") { + Set tWord=1 + } + If (ttName["AVERAGE") { + Set tWord=tWord+1 + } + If (ttName["TOTAL") { + Set tWord=tWord+1 + } + If (ttName["REVENUE") { + Set tWord=tWord+1 + } + } + } + //Relations + ElseIf (tType="r") { + // find null ref for relation + Set tRelation=$LG(tLevels(n),2) + Set tNullRef=$G($$$DeepSeeMetaGLVN("cubes",tCube,"relations",$$$UPPER(tRelation),"nullReplacement")) + If (tNullRef'="") { + Set tSpec="["_tRelation_"]" + Set tMDXnull="SELECT FROM ["_tCube_"] WHERE "_tSpec_".&["_tNullRef_"]" + } + } + + //Get Cardinality + Set tCard=0 + If tSpec'="" { + Do ##class(%DeepSee.Utils).%GetDimensionInfo(tCube,tSpec,.tDim,.tHier,.tLevel) + If tDim=0 { + Set tIndex=$G($$$DeepSeeMetaGLVN("cubes",tCube,"msr#",$LG($$$DeepSeeMetaGLVN("cubes",tCube,"mbr#",tDim,tHier,tLevel),5)))_"Search" + } Else { + Set tIndex=$LG($$$DeepSeeMetaGLVN("cubes",tCube,"mbr#",tDim,tHier,tLevel),5) + } + + Set tValue=$O($$$DeepSeeIndexGLVN(tCube,tIndex,"")) + While (tValue '="") { + Set tCard=tCard + 1 + Set tValue=$O($$$DeepSeeIndexGLVN(tCube,tIndex,tValue)) + } + Set tMeta.Cardinality=tCard + } + + //Execute + Set tMeta.LevelName=$Case(tType, "l":$LG(tLevels(n),4),"m":$LG(tLevels(n),3),:tSpec) + Set tMeta.DimName=$LG(tLevels(n),2) + Set tMeta.LevelType=$Case(tType,"l":"Dimension","m":"Measure","r":"Relation",:tType) + Set tMeta.WordRecognition=tWord + Set listVals=$lb(tMDXnull,tMDXavg,tMDXstd) + For i=1:1:$listlength(listVals) { + Set tMDX=$listget(listVals,i) + If tMDX'="" { + Set tRS=##class(%DeepSee.ResultSet).%New() + Set tSC=tRS.%PrepareMDX(tMDX) + If $$$ISERR(tSC) Quit + Set tSC=tRS.%Execute() + If $$$ISERR(tSC) Quit + Set tVal=tRS.%GetOrdinalValue() + If i=1 { + Set tMeta.Null=(tVal/tFactCount) + } ElseIf i=2 { + Set tMeta.Mean=tVal + } ElseIf i=3 { + Set tMeta.STDDEV=tVal + } + } + } + + Set tSC=tMeta.%Save() + If $$$ISERR(tSC) Quit + + Set n=$O(tLevels(n)) + } + } + + Set namespace=$NAMESPACE + Set tCubeClass=##class(%DeepSee.Utils).%GetCubeClass(tCube) + Set tSC=$ClassMethod(tCubeClass,"%GetSourceInfo",.tSourceInfo) + Set tSourceClass=tSourceInfo("sourceClass") + Set class=##class(%Dictionary.ClassDefinition).%OpenId(tSourceClass,,.st) + + If (st) { + For i=1:1:class.Properties.Count() { + Set prop=class.Properties.GetAt(i) + Set tType=prop.Type + If (tType="%Library.Date") { + Kill propName,sql,statement,st,rs + Set propName="%"_prop.Name_"%" + Set sql="UPDATE AnalyzeThis_Dashboard.MetaData (DataType) VALUES ('Date') WHERE LevelName LIKE ?" + Set statement=##class(%SQL.Statement).%New("deffered") + Set st=statement.%Prepare(sql) + If (st) { + Set rs=statement.%Execute(propName) + } + } + If (($$$UPPER(prop.Name)[$$$UPPER("year"))&&(tType'="%Library.Date")) { + Kill propName,sql,statement,st,rs + Set propName="%"_prop.Name_"%" + Set sql="UPDATE AnalyzeThis_Dashboard.MetaData (DataType) VALUES ('Year') WHERE LevelName LIKE ?" + Set statement=##class(%SQL.Statement).%New("deffered") + Set st=statement.%Prepare(sql) + If (st) { + Set rs=statement.%Execute(propName) + } + } + } + } + } Catch(ex) { + Set tSC=ex.AsStatus() + } + + If $$$ISERR(tSC) { + Do $System.Status.DisplayError(tSC) + } + + Quit tSC +} + +} diff --git a/AnalyzeThis/Generator.cls b/AnalyzeThis/Generator.cls index aa26475..0c78aeb 100644 --- a/AnalyzeThis/Generator.cls +++ b/AnalyzeThis/Generator.cls @@ -245,7 +245,7 @@ ClassMethod GenerateCube(pCubeName, pPropertyJSONStreamId) As %Status Set propertyObj={}.%FromJSON(tempStream) //Disable items as needed - Set model=$zobjclassmethod(tCubeClass,"%GetModel") + Set model=$ClassMethod(tCubeClass,"%GetModel") For i=1:1:model.dimensions.Count() { Set tempPos=propertyObj.Properties.%Get(model.dimensions.GetAt(i).name) If propertyObj.Include.%Get(tempPos)'="true" { From 09618599e03fff32124116cfb744b069470118f3 Mon Sep 17 00:00:00 2001 From: psteiwer Date: Mon, 30 Mar 2020 14:11:11 -0400 Subject: [PATCH 06/11] Visual Studio Syntax changes --- AnalyzeThis/UI/Dialog/CSVImport.cls | 2826 ++++++++++++++------------- AnalyzeThis/Utils.cls | 444 ++--- 2 files changed, 1636 insertions(+), 1634 deletions(-) diff --git a/AnalyzeThis/UI/Dialog/CSVImport.cls b/AnalyzeThis/UI/Dialog/CSVImport.cls index 3b5a75c..97fbd26 100644 --- a/AnalyzeThis/UI/Dialog/CSVImport.cls +++ b/AnalyzeThis/UI/Dialog/CSVImport.cls @@ -1,1412 +1,1414 @@ -/// Created using the page template: Default -Class AnalyzeThis.UI.Dialog.CSVImport Extends %ZEN.Dialog.standardDialog [ System = 4 ] -{ -Parameter DOMAIN = "AnalyzeThis"; - -Parameter Version = 1; - -Property INPUTTYPE As %ZEN.Datatype.string(ZENURL = "INPUTTYPE"); - -/// This is the temporary file name we saved on remote server, only saved when Input Type is LOCAL. -Property LOCALFILENAME As %ZEN.Datatype.string; - -Property LineSize As %Integer [ InitialExpression = 10000 ]; - -Property hasHeaders As %ZEN.Datatype.string [ InitialExpression = 1 ]; - -Property Source As %String; - -Property SourceClass As %String; - -Property CubeName As %String; - -Property propertyJSONStreamID As %ZEN.Datatype.string; - -Property dataJSONStreamID As %ZEN.Datatype.string; - -Property SourceType As %String; - -/// Id used to track progress. -Property trackingId As %ZEN.Datatype.string; - -/// This Style block contains page-specific CSS style definitions. -XData Style -{ - -} - -XData Contents [ XMLNamespace = "http://www.intersystems.com/zen" ] -{ - - -
- - - -
-} - -/// This XML block defines the contents of this pane. -XData dialogBody [ XMLNamespace = "http://www.intersystems.com/zen" ] -{ - - -} - -/// Update the LineSize property -Method UpdateLineSize(value) [ Internal, ZenMethod ] -{ - Set ..LineSize=value -} - -/// Update the hasHeaders property -Method UpdateHasHeaders(value) [ Internal, ZenMethod ] -{ - Set ..hasHeaders=value -} - -/// Update the Source property -Method UpdateSource(value) [ Internal, ZenMethod ] -{ - Set ..Source=value - Do ..GenerateCubeName() -} - -/// Update the LOCALFILENAME property -Method UpdateLocalFile(value) [ Internal, ZenMethod ] -{ - Set ..LOCALFILENAME=value - - Set button=..%GetComponentById("btnSubmit") - Set button.disabled=(value="") - If value'="" { - Set button.controlClass="commandButton" - } Else { - Set button.controlClass="commandButtonDisabled" - } - - Do ..GenerateCubeName() -} - -/// Fire popup action to navigate to dashboard -Method seeDash() [ Internal, ZenMethod ] -{ - Set pcube=..CubeName - &JS -} - -/// Fire popup action to navigate to scorecard -Method seeScoreCard() [ Internal, ZenMethod ] -{ - Set pcube=..CubeName - &JS -} - -/// User changed to pick file between Remote and Local -ClientMethod doLocationChange(flag) [ Internal, Language = javascript ] -{ - if (flag == "LOCAL") { - zen("idRemoteFile").setHidden(true); - zen("LocalFile").setHidden(false); - // for local we cannot allow for Directory - // show Submit button - zen("btnNext").setHidden(true); - zen("btnSubmit").setHidden(false); - } else { - zen("idRemoteFile").setHidden(false); - zen("LocalFile").setHidden(true); - zen("btnNext").setHidden(false); - zen("btnSubmit").setHidden(true); - var Source=zen("FileName").getValue() - if (zen("FileName").getValue() != "") { - zen("btnNext").setProperty('disabled',Source==""); - zen("btnNext").setProperty('controlClass', (Source!="" ? 'commandButton' :'commandButtonDisabled')); - } - } -} - -/// Start the Import -ClientMethod doImportJSON() [ Language = javascript ] -{ - //Before processing, make sure all property names are unique - var count=1 - var ok=1 - var test=[]; - while (document.getElementById("propName"+count)!=null) { - document.getElementById("propName"+count).style="" - if (test[document.getElementById("propName"+count).value]!=null) { - document.getElementById("propName"+test[document.getElementById("propName"+count).value]).style.border="2px solid #FF0000" - document.getElementById("propName"+count).style.border="2px solid #FF0000" - ok=0 - } - test[document.getElementById("propName"+count).value]=count - count+=1 - } - - if (!ok) { - alert("Property names not unique."); - return - } - - zenPage.nextPage(); - zen("btnDone").setHidden(false); - zen("btnCancel").setHidden(true); - zenPage.updateState(); -} - -/// Update the state of the template buttons. -/// Subclasses should call this method when they need to -/// update the state of the footer buttons. -ClientMethod updateState() [ Language = javascript ] -{ - var btnBack=zen('btnBack'); - var btnNext=zen('btnNext'); - var btnFinish=zen('btnFinish'); - var btnGoStats=zen('btnGoStats'); - var btnHelp=zen('btnHelp'); - var btnSubmit=zen('btnSubmit'); - var multi=this.hasMultiplePages(); - - if (0) { - btnBack.setProperty('hidden',!this.canGoBack()); - btnBack.setProperty('disabled',!this.canGoBack()); - btnBack.setProperty('controlClass', (this.canGoBack() ? 'commandButton' :'commandButtonDisabled')); - } - if (btnNext) { - btnNext.setProperty('hidden',!this.canGoNext()); - btnNext.setProperty('disabled',!this.canGoNext()); - btnNext.setProperty('controlClass', (this.canGoNext() ? 'commandButton' :'commandButtonDisabled')); - } - // special for Import on this page. Hide regular Next and show Submit Next for LOCAL. - if (zen("InputType").getValue()=="LOCAL") { - btnSubmit.setProperty('disabled',true); - btnSubmit.setProperty('controlClass', 'commandButtonDisabled'); - } - if (btnFinish) { - btnFinish.setProperty('disabled',!this.canFinish()); - btnFinish.setProperty('hidden',!this.canFinish()); - btnFinish.setProperty('controlClass', (this.canFinish() ? 'commandButton' :'commandButtonDisabled')); - } -} - -/// Kicks off GenerateAll and starts the update timer -ClientMethod updateProgress() [ Language = javascript] -{ - var file=this.Source - if (this.INPUTTYPE=='LOCAL') { - file=this.LOCALFILENAME - } - this.trackingId=this.GenerateAll(this.SourceType,file,this.LineSize,this.hasHeaders,this.CubeName,this.propertyJSONStreamID,this.dataJSONStreamID); - if (this.trackingId!='') { - zen('timer').timeout=500; - zen('timer').startTimer(); - } -} - -/// Update the status area. -ClientMethod updateStatus() [ Language = javascript ] -{ - var status=this.CheckStatus(this.trackingId); - var html=zen('htmlContent3'); - if (status != '') { - html.setContent(status); - zen('timer').timeout=250; - zen('timer').startTimer(); - } - else { - this.trackingId=''; - - if (zenPage.DashboardExists()) { - zen("btnGoDash").setProperty('disabled',0); - zen("btnGoDash").setProperty('hidden',0); - zen("btnGoDash").setProperty('controlClass','commandButton'); - - // If the dashboard was generated, we know it is safe to proceed to stats tab - zen("btnGoStats").setProperty('disabled',!this.canStat()); - zen("btnGoStats").setProperty('hidden',!this.canStat()); - zen("btnGoStats").setProperty('controlClass', (this.canStat() ? 'commandButton' :'commandButtonDisabled')); - } - - if (zenPage.ScoreCardExists()) { - zen("btnScoreDash").setProperty('disabled',0); - zen("btnScoreDash").setProperty('hidden',0); - zen("btnScoreDash").setProperty('controlClass','commandButton'); - } - } - return; -} - -ClientMethod showMsg(msg) [ Language = javascript ] -{ - zen("idRespond").setValue(msg); - zen("idRespond").setHidden(false); -} - -/// Get the current status of the background task. -ClassMethod CheckStatus(pTrackingId As %String) As %String [ ZenMethod ] -{ - Set tOutput="" - Set tIsComplete=0 - Set tIsErr=0 - - Merge tTracking=^AnalyzeThis.GenerateTracking(pTrackingId) - - Set currStep=$O(tTracking(""),-1) - If currStep="" Quit tOutput - For tStep=1:1:currStep { - - Set tPhase=$Case(tStep, - 1:"Preparing JSON", - 2:"Generating Source Class", - 3:"Importing Data", - 4:"Generating Cube", - 5:"Populating Cube", - :"Generating Sample Dashboard") - - Set tStatus=tTracking(tStep) - - If tStatus="Working..." { - // Leave message as "Working..." - } ElseIf $$$ISERR(tStatus) { - Set tStatus=$System.Status.GetErrorText(tStatus) - Set tIsErr=1 - } Else { - Set tStatus="Complete" - If tStep=6 { - Set tIsComplete=1 - } - } - - Set tOutput=tOutput _ "" - Set tOutput=tOutput _ ""_tPhase_"" - Set tOutput=tOutput _ ""_tStatus_"" - Set tOutput=tOutput _ "" - } - For tStep=currStep+1:1:6 { - //Set tOutput=tOutput _ "" - Set tOutput=tOutput _ "" - - Set tPhase=$Case(tStep, - 1:"Preparing JSON", - 2:"Generating Source Class", - 3:"Importing Data", - 4:"Generating Cube", - 5:"Populating Cube", - :"Generating Sample Dashboard") - - Set tOutput=tOutput _ ""_tPhase_"" - Set tOutput=tOutput _ "Not Yet Started" - Set tOutput=tOutput _ "" - } - - If (tOutput'="") { - Set tOutput="
"_$$$Text("Status")_ "
" _ tOutput _ "
" - } - - If (tIsComplete || tIsErr) { - Kill ^AnalyzeThis.GenerateTracking(pTrackingId) - } - Quit tOutput -} - -/// Start the build cube process in the background. -/// Return the tracking id. -ClassMethod GenerateAll(pSourceType,pSource, pLineSize, pHasHeaders, pCubeName, pPropertyJSONStreamId, pDataJSONStreamId) As %String [ ZenMethod ] -{ - Set tTrackingId="" - Set tJobTimeOut=10 - - Job ##class(AnalyzeThis.Generator).GenerateAll(pSourceType,pSource, pLineSize, pHasHeaders, pCubeName, pPropertyJSONStreamId, pDataJSONStreamId)::tJobTimeOut - If '$Test { - &js - Set tTrackingId="" - } - Else { - Set tTrackingId=$ZChild - } - Quit tTrackingId -} - -/// Using the current cube name, check if a dashboard was generated for this cube -Method DashboardExists() As %Status [ ZenMethod ] -{ - Set dashName="Generated/Samples for "_..CubeName_".dashboard" - Set st=##class(%DeepSee.Dashboard.Utils).%DashboardExists(dashName) - Quit st -} - -/// Using the current cube name, check if a scorecard was generated for this cube -Method ScoreCardExists() As %Status [ ZenMethod ] -{ - Set dashName="Generated/Sample ScoreCard for "_..CubeName_".dashboard" - Set st=##class(%DeepSee.Dashboard.Utils).%DashboardExists(dashName) - Quit st -} - -/// Display cube stats in the status area -ClientMethod drawStats() [ Language = javascript ] -{ - document.getElementById("statsTable").innerHTML="

Generating Stats...

" - document.getElementById("statsTable").innerHTML=zenPage.GetStats() - zenPage.updateState(); -} - -/// Generate stats display -Method GetStats() As %String [ ZenMethod ] -{ - Set st=$$$OK - - Set mdx="SELECT [Measures].[%COUNT] on 1 from ["_..CubeName_"]" - Set rs=##class(%DeepSee.ResultSet).%ExecuteDirect(mdx) - Set totalmembers=rs.%GetOrdinalValue(1) - Set html="

"_totalmembers_" facts have been built for Cube "_..CubeName_"

" - Set st=##class(%DeepSee.Utils).%GetDimensionList(..CubeName,.info) - Set d=$order(info("")) - While d'="" { - Set h=$order(info(d,"")) - While h'="" { - Set l=$order(info(d,h,"")) - While l'="" { - If $lg(info(d,h,l),1)="l" { - Set spec="["_$lg(info(d,h,l),2)_"].["_$lg(info(d,h,l),3)_"].["_$lg(info(d,h,l),4)_"]" - Set mdx="SELECT COUNT("_spec_".Members) on 1 from ["_..CubeName_"]" - Set rs=##class(%DeepSee.ResultSet).%ExecuteDirect(mdx) - Set members=rs.%GetOrdinalValue(1) - - Set mdx="SELECT ISNULL("_spec_".&[],0)/[Measures].[%COUNT] on 1 from ["_..CubeName_"]" - Set rs=##class(%DeepSee.ResultSet).%ExecuteDirect(mdx) - Set nullpercent=(((rs.%GetOrdinalValue(1)*10000)\1)/100) - Set html=html_"" - } - Set l=$order(info(d,h,l)) - } - Set h=$order(info(d,h)) - } - Set d=$order(info(d)) - } - Set html=html_"
Level NameNumber of MembersPercentage of Null Values
"_spec_""_members_""_nullpercent_"%
" - Quit html -} - -/// Using file properties, create the cube name -Method GenerateCubeName() [ ZenMethod ] -{ - If ..SourceType="CSV" { - Set file="" - If ..%GetValueById("InputType")="LOCAL" { - Set file=..LOCALFILENAME - } Else { - Set file=..Source - } - If $length(file,"\")>$length(file,"/") { - Set tName=$zstrip($replace($piece($piece(file,"\",*),".",1),"DEEPSEE",""),"*PCW") - } Else { - Set tName=$zstrip($replace($piece($piece(file,"/",*),".",1),"DEEPSEE",""),"*PCW") - } - } ElseIf ..SourceType="Class" { - Set tName=..Source - // Class should end in .cls, extract package + .cls extension to get class name - // If class does not end in .cls, handle appropriately - Set tCls=1 - If $zconvert($extract(tName,*-3,*),"U")'=".CLS" { - Set tCls=0 - } - Set tName=$piece(tName,".",*-tCls) - } Else { - // SouceType not supported to auto generate class name - Quit - } - - Set ..CubeName=tName - Set ..SourceClass="AnalyzeThis.Generated."_tName - Do ..%SetValueById("CubeName",tName) -} - -/// Validates and updates the cube name property -Method UpdateCubeName(tName) [ ZenMethod ] -{ - If ($$$UPPER($ZStrip(tName," - } - Set ..CubeName=tName - Set ..SourceClass="AnalyzeThis.Generated."_tName -} - -/// This method is fired up after onDrawContent is finished. Hide progress message and display error if exists. -ClientMethod drawDone(tab) [ Language = javascript ] -{ - - var result=this.GetStatus(tab); - var id="idProgress"+tab; - if (result.Status == "Done") { - if (result.Error != "") { - this.showMsg(result.Error); - zen("btnFinish").setProperty('disabled',true); - zen("btnFinish").setProperty('controlClass','commandButtonDisabled'); - } - } else { - zen("btnNext").setProperty('disabled',true); - zen("btnNext").setProperty('controlClass','commandButtonDisabled'); - zen("btnFinish").setProperty('disabled',true); - zen("btnFinish").setProperty('controlClass','commandButtonDisabled'); - } - zen(id).setHidden(true); -} - -/// Return true if this template can go to the next page (i.e., enable -/// the Next button).
-ClientMethod canGoNext() [ Language = javascript ] -{ - var tabGroup=zen('tabGroup'); - var tabNo=tabGroup.getCurrTabNo(); - var flag=false; - switch(tabNo) { - case 1: - flag=true; - if (this.INPUTTYPE == "LOCAL") flag=false; - break; - case 2: - break; - case 3: - break; - case 4: - break; - } - return flag; -} - -/// Return true if this template can go to the stats page (i.e., enable -/// the stats button).
-ClientMethod canStat() [ Language = javascript ] -{ - var tabGroup=zen('tabGroup'); - var tabNo=tabGroup.getCurrTabNo(); - var flag=false; - switch(tabNo) { - case 1: - break; - case 2: - break; - case 3: - flag=true; - break; - case 4: - break; - } - return flag; -} - -/// Return true if this template can go to the previous page (i.e., enable -/// the Back button).
-/// This is implemented by subclasses. -ClientMethod canGoBack() [ Language = javascript ] -{ - var tabGroup=zen('tabGroup'); - var tabNo=tabGroup.getCurrTabNo(); - var flag=true; - switch(tabNo) { - case 1: - flag=false; - break; - case 2: - break; - case 3: - break; - case 4: - break; - } - return flag -} - -/// Return true if this template can Finish (i.e., enable -/// the Finish button).
-ClientMethod canFinish() [ Language = javascript ] -{ - var tabGroup=zen('tabGroup'); - var tabNo=tabGroup.getCurrTabNo(); - var flag=false; - switch(tabNo) { - case 1: - break; - case 2: - flag=true; - break; - case 3: - break; - case 4: - break; - } - - return flag; -} - -ClientMethod hasMultiplePages() [ Language = javascript ] -{ - return true; -} - -/// Go to the next page of the template (if there is one).
-ClientMethod nextPage() [ Language = javascript ] -{ - var tabGroup=zen('tabGroup'); - var tabNo=tabGroup.getCurrTabNo(); - var html=zen("htmlContent"); - this.showMsg("",1); - switch(tabNo) { - case 1: - var SourceType=zen("SourceType").getValue(); - - if (SourceType=="CSV") { - var Source=zen("FileName").getValue(); - var inputtype="REMOTE" - if (this.LOCALFILENAME != "") { - Source=this.LOCALFILENAME; - inputtype="LOCAL"; - } - var ok=this.ValidateFile(Source); - if (ok == 0) { - this.showMsg($$$Text("File required."),1); - zen("FileName").focus(); - break; - } - if (ok == -1) { - this.showMsg($$$Text("You have entered a directory path without a file name."),1); - zen("FileName").focus(); - break; - } - if (ok == -2) { - this.showMsg($$$Text("File does not exist!"),1); - zen("FileName").focus(); - break; - } - if (ok == -3) { - this.showMsg($$$Text("File Name cannot start with a number!"),1); - zen("FileName").focus(); - break; - } - html.setProperty('seed',Source); - } else if (SourceType=="SQL") { - var sql=zen("SQLText").getValue(); - if (sql=="") { - this.showMsg($$$Text("SQL Query required."),1); - zen("SQLText").focus(); - break; - } - html.setProperty('seed',sql); - } else if (SourceType=="Class") { - var sourceClass=zen("sourceClass").getValue(); - if (sourceClass=="") { - this.showMsg($$$Text("Source Class required."),1); - zen("sourceClass").focus(); - break; - } - html.setProperty('seed',sourceClass); - } - - if (zen("CubeName").getValue()=="") { - this.showMsg($$$Text("Cube Name required."),1); - zen("CubeName").focus(); - break; - } - - var uniquename=zenPage.UniqueName(zen("CubeName").getValue()); - if (uniquename!="") { - this.showMsg($$$Text(uniquename),1); - zen("CubeName").focus(); - break; - } - - tabGroup.showNextTab(); - break; - case 2: - var html=zen("htmlContent3"); - html.setProperty('seed',1); - tabGroup.showNextTab(); - break; - case 3: - var html=zen("htmlContent4"); - html.setProperty('seed',1); - tabGroup.showNextTab(); - break; - case 4: - break; - } - zenPage.updateState(); -} - -/// Go to the previous page of the template (if there is one).
-/// This is implemented by subclasses. -ClientMethod previousPage() [ Language = javascript ] -{ - var tabGroup=zen('tabGroup'); - var tabNo=tabGroup.getCurrTabNo(); - switch(tabNo) { - case 1: - break; - case 2: - tabGroup.showPreviousTab(); - zenPage.updateState(); - if (zen("InputType").getValue()=="LOCAL") { - zen('btnNext').setProperty('hidden',true); - } - break; - case 3: - tabGroup.showPreviousTab(); - zenPage.updateState(); - break; - case 4: - tabGroup.showPreviousTab(); - zenPage.updateState(); - break; - } -} - -/// This client event, if present, is fired when the page is loaded. -ClientMethod onloadHandler() [ Internal, Language = javascript ] -{ - this.invokeSuper('onloadHandler',arguments); - if (this.InvalidParam) { - zen("btnNext").setHidden(true); - } - this.onstartHandler(); -} - -ClientMethod getDialogValue() [ Internal, Language = javascript ] -{ - return "" -} - -/// User clicked the Browse button. -ClientMethod browseSelect(name) [ Internal, Language = javascript ] -{ - var Dir=zen(name).getValue(); - zenLaunchPopupWindow('%ZEN.Dialog.fileSelect.zen?Dir='+encodeURIComponent(Dir)+'&wildcard=*.csv&showdirectoryonly=0','FileSelect','resizable,width=600,height=700'); -} - -/// Returning from file select OR qualifers dialog and setting the value into the appropriate field. -/// The id of the field that needs to be updated is saved in "dialogClicked". -ClientMethod onPopupAction(popupName, action, value) [ Internal, Language = javascript ] -{ - if (action=="ok") { - if (popupName=="sourceClass") { - zenPage.getComponentById(popupName).setValue(value); - zenPage.UpdateSource(value); - zenPage.updateState(); - } else if (popupName=="FileSelect") { - zen("FileName").setValue(value); - zenPage.UpdateSource(value); - zenPage.GenerateCubeName(); - zenPage.updateState(); - } - } -} - -/// This is called when the template is first displayed; -/// This provides a chance to load the last filetype, etc. -ClientMethod onstartHandler() [ Internal, Language = javascript ] -{ - this.onresizeHandler(); - // if this is from the submit of the Local file, load the content of the file on tab 2. - if (this.INPUTTYPE == "LOCAL") { - // set the first tab for LOCAL correctly - this.doLocationChange("LOCAL"); - if (this.LOCALFILENAME != "") { - var html=zen("htmlContent"); - html.setProperty('seed',this.LOCALFILENAME); - var tabGroup=zen('tabGroup'); - var tabNo=tabGroup.getCurrTabNo(); - if (tabNo == 1) { - tabGroup.showNextTab(); - } - zenPage.updateState(); - } - zen("btnNext").setHidden(true); - } -} - -/// Get the (localized) title string for the dialog. -/// This should be implemented in a subclass. -Method %OnGetTitle() As %String [ Internal ] -{ - Quit $$$TextHTML("Import Data") -} - -/// Get the (localized) subtitle string for the dialog. -/// This should be implemented in a subclass. -Method %OnGetSubtitle() As %String [ Internal ] -{ - Quit "" -} - -/// This callback is called after the server-side page -/// object and all of its children are created.
-/// Subclasses can override this to add, remove, or modify -/// items within the page object model, or to provide values -/// for controls. -Method %OnAfterCreatePage() As %Status [ Internal ] -{ - Set tSC=##super() - If $$$ISERR(tSC) Quit tSC - - // try and determine if user is on *same* machine as server - // if so, do not provide local upload option - Set tTCPAddr=$SYSTEM.TCPDevice.PeerAddr(0) - Set tClientAddr=$G(%request.CgiEnvs("REMOTE_ADDR")) - Set tIsLocal=((tClientAddr="127.0.0.1") ! (tClientAddr="::1")) & ((tTCPAddr="127.0.0.1") ! (tTCPAddr="::1")) - If tIsLocal { - Set ..%GetComponentById("InputType").hidden=1 - Set ..%GetComponentById("LocalFile").hidden=1 - } - Set ..SourceType=..%GetComponentById("SourceType").value - Set Source="" - Set ..%GetComponentById("FileName").value=Source - #; Set for Remote input types choice: ServerName or local machine - Set ..%GetComponentById("InputType").displayList=$zu(110)_","_$$$Text("My Local Machine") - If $G(%request.Data("INPUTTYPE",1))="LOCAL" { - Set %page.LOCALFILENAME=$G(%session.Data($Username,"Import","LOCALFileName")) - Set ..%GetComponentById("InputType").value="LOCAL" - } - If $G(%request.Data("LineSize",1)) '= "" { - Set %page.LineSize=$G(%request.Data("LineSize",1)) - Set ..%GetComponentById("LineSize").value=$G(%request.Data("LineSize",1)) - Set ..LineSize=$G(%request.Data("LineSize",1)) - } - If $G(%request.Data("CubeName",1)) '= "" { - Set %page.CubeName=$G(%request("CubeName",1)) - Set ..%GetComponentById("CubeName").value=$G(%request.Data("CubeName",1)) - Set ..CubeName=$G(%request.Data("CubeName",1)) - } - Quit $$$OK -} - -ClassMethod DrawSortPageTitle(pSeed As %String) As %Status [ Internal ] -{ - Set tPageTitle="Please sort each column as Dimension, Date, or Measure. "_ - "
A Measure might be something like revenue, or total discharges.
If it is a measure, please specify if it is a number or currency."_ - "
A Dimension would be things like zip code, state, or measure description
" - &html<
#(tPageTitle)# -

> - Quit 1 -} - -ClassMethod DrawPageTitle(pSeed As %String) As %Status [ Internal ] -{ - Set tPageTitle="Import Data" - &html<
#(tPageTitle)# -

> - Quit 1 -} - -/// Draw preview content of the CSV file for user to confirm properties -Method PropertyCheckJSON(pSeed) As %Status [ ZenMethod ] -{ - Set tSC=$$$OK - - Quit:pSeed="" tSC - - // Get JSON from CSV - Set file="" - If ..INPUTTYPE="LOCAL" { - Set file=..LOCALFILENAME - } Else { - Set file=..Source - } - If ..SourceType="SQL" { - Set tSC=##class(AnalyzeThis.Utils).SQLToJSON(pSeed,.propertiesJSONStreamId,.dataJSONStreamId,10) - } ElseIf ..SourceType="Class" { - // Get the SQL table name and then pass through SQL method - Set sqlQuery=##class(AnalyzeThis.Utils).ClassToQuery(pSeed) - Set tSC=##class(AnalyzeThis.Utils).SQLToJSON(sqlQuery,.propertiesJSONStreamId,.dataJSONStreamId,10) - } Else { - Set tSC=##class(AnalyzeThis.UI.Dialog.CSVImport).CSVToJSON(file,.propertiesJSONStreamId,.dataJSONStreamId,10,..hasHeaders) - } - - Quit:$$$ISERR(tSC) tSC - - Set ..propertyJSONStreamID=propertiesJSONStreamId - Set ..dataJSONStreamID=dataJSONStreamId - - Set properties=##class(%Stream.FileBinary).%OpenId(propertiesJSONStreamId) - Set propertyObj={}.%FromJSON(properties) - - Set data=##class(%Stream.FileBinary).%OpenId(dataJSONStreamId) - Set dataArray=[].%FromJSON(data) - - Quit:$$$ISERR(tSC) tSC - - &html<> - &html<> - Set iter=propertyObj.Display.%GetIterator() - While iter.%GetNext(.key,.val) { - If propertyObj.Ignore.%Get(key) { - Continue - } - Set format=0 - &html<> - } - &html<> - &html<> - Set iter=propertyObj.Display.%GetIterator() - While iter.%GetNext(.key,.val) { - If propertyObj.Ignore.%Get(key) { - Continue - } - &html<> - } - &html<> - Set iter=dataArray.%GetIterator() - While ((iter.%GetNext(.key,.val)) && (key<10)) { - If propertyObj.Ignore.%Get(key) { - Continue - } - &html<> - Set tPropNum=-1 - Set iter2=val.%GetIterator() - While iter2.%GetNext(.key2,.val2) { - &html<> - } - &html<> - } - &html<
Include?

#(val2)#
> - Set %session.Data($Username,"Import","ContentStatus") = "Done" - Quit tSC -} - -/// Changes formatting options based on the type -ClientMethod swapFormat(propID, value) [ Internal, Language = javascript ] -{ - if ((value=='%Date') || (value=='%TimeStamp')) { - document.getElementById("dateFormat"+propID).style.display="" - document.getElementById("intFormat"+propID).style.display="none" - - zenPage.autoDateFormat(propID); - } else if (value=='%Integer') { - document.getElementById("intFormat"+propID).style.display="" - document.getElementById("dateFormat"+propID).style.display="none" - } else { - document.getElementById("dateFormat"+propID).style.display="none" - document.getElementById("intFormat"+propID).style.display="none" - } -} - -/// Try and automatically select the date format -ClientMethod autoDateFormat(propID) [ Internal, Language = javascript ] -{ - var dates=[]; - for (var i=0;i<10;i++) { - var prop=document.getElementById("p"+propID+"r"+i); - if (prop==null) { - return - } - var temp=parseInt(zenPage.findDateType(prop.innerHTML)); - if (typeof dates[temp]=="undefined") { - dates[temp]=1; - } else { - dates[temp]+=1; - } - } - - var typemaxval=0; - var typemaxpos=0; - for (var i=0;itypemaxval) { - typemaxpos=i; - typemaxval=dates[i]; - } - } - - // Can enhance this later to keep type 0 and prevent user from going to next page if type=0 - if (typemaxpos==0) { - typemaxpos=1; - } - - document.getElementById("dateFormat"+propID).value=typemaxpos; - document.getElementById("dateFormat"+propID).onchange(); -} - -/// As changes are made to the properties, update the JSON stream -Method updateProp(key, which, value, stream) [ ZenMethod ] -{ - Set propertyJSON=##class(%Stream.FileBinary).%OpenId(stream) - Set propertyObj={}.%FromJSON(propertyJSON) - - Do propertyObj.%Get(which).%Set(key,value) - - Do propertyObj.%ToJSON(.propertyJSON) - Do propertyJSON.%Save() -} - -Method findDateType(pValue) As %Integer [ ZenMethod ] -{ - Quit ##class(AnalyzeThis.UI.Dialog.CSVImport).FindType(pValue) -} - -/// Given a value, find the best guess of type -ClassMethod FindType(value) As %Integer -{ - Set type=0 - - If (value?1.2N1"/"1.2N1"/"2.4N) { - // [N]N/[N]N/[NN]NN - Set type=1 - } ElseIf (value?2N1" "1(1(1"J",1"j")1(1"A",1"a")1(1"N",1"n"),1(1"F",1"f")1(1"E",1"e")1(1"B",1"b"),1(1"M",1"m")1(1"A",1"a")1(1"R",1"r"),1(1"A",1"a")1(1"P",1"p")1(1"R",1"r"),1(1"M",1"m")1(1"A",1"a")1(1"Y",1"y"),1(1"J",1"j")1(1"U",1"u")1(1"N",1"n"),1(1"J",1"j")1(1"U",1"u")1(1"L",1"l"),1(1"A",1"a")1(1"U",1"u")1(1"G",1"g"),1(1"S",1"s")1(1"E",1"e")1(1"P",1"p"),1(1"O",1"o")1(1"C",1"c")1(1"T",1"t"),1(1"N",1"n")1(1"O",1"o")1(1"V",1"v"),1(1"D",1"d")1(1"E",1"e")1(1"C",1"c"))1" "2.4N) { - // NN (Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec) [NN]NN - Set type=2 - } ElseIf (value?4N1"-"1.2N1"-"1.2N) { - // NNNN-[N]N-[N]N - Set type=3 - } ElseIf (value?2N1"/"2N1"/"2.4N) { - // NN/NN/[NN]NN - Set type=4 - } ElseIf ((value?1"18"2N)||(value?1"19"2N)||(value?1"20"2N)||(value?1"21"2N)) { - // 18NN, 19NN, 20NN, 21NN - Set type=5 - } ElseIf (value?5N) { - // NNNNN, $H format - Set type=30 - } - - Quit type -} - -/// tab = 2: Content - when DrawContent is finished, Status is "Done". -ClassMethod GetStatus(tab) As %ZEN.proxyObject [ Internal, ZenMethod ] -{ - Set proxy=##class(%ZEN.proxyObject).%New() - Set proxy.Status=$G(%session.Data($Username,"Import","ContentStatus")) - Set proxy.Error=$G(%session.Data($Username,"Import","ContentError")) - Quit proxy -} - -/// Validate file name. -ClassMethod ValidateFile(FILE) As %Integer [ Internal, ZenMethod ] -{ - If $ZStrip(FILE,"<>W")="" Quit 0 - #; Entered directory only! - If ##class(%File).DirectoryExists(FILE) Quit -1 - #; File does not exist!" - If '##class(%File).Exists(FILE) Quit -2 - Set tFILE=$PIECE(FILE,"\",*) - If ($$$UPPER($ZStrip(tFILE," -/// The necessary parameters are passed in URL. The temp file name is saved in %session and to be used in loading content. -ClassMethod %OnSubmit(pSubmit As %ZEN.Submit) As %Status [ Internal ] -{ - Set tStream=pSubmit.%GetStream("LocalFile") - Set dir=##class(%File).SubDirectoryName($zu(12),"Temp") - #; If temp directory does not exist, create it now - If ##class(%File).DirectoryExists(dir)=0 { - Do ##class(%File).CreateDirectory(dir) - } - #; Get proper delimeter since SubDirectoryName does not include delimeter - Set tDelim="\" - If $$$isUNIX Set tDelim="/" - If $$$isVMS Set tDelim="" - #; Give it a name so it won't be deleted automatically - Set DirFileName=dir_tDelim_pSubmit.%Data("CubeName")_".stream" - Set file=##class(%Stream.FileBinary).%New() - Set tSC=file.LinkToFile(DirFileName) - If tStream '= "" { - #; Copy the stream from local server - Set tSC=file.CopyFrom(tStream) - #; Save it to the file stream on the remote server - If tSC Set tSC=file.%Save() - } - Set %response.Context("INPUTTYPE")="LOCAL" - Set %response.Context("LineSize")=pSubmit.%Data("LineSize") - Set %response.Context("CubeName")=pSubmit.%Data("CubeName") - // remember popup info - Set %response.Context("$ZEN_POPUP")=1 - Set %response.Context("$ZEN_POPUPPARENT")=+$G(%request.Data("$ZEN_POPUPPARENT",1)) - Set %response.Context("$ZEN_SOFTMODAL")=..%OnUseSoftModals() - // save the temp local file name to be used in later to load content - Set %session.Data($Username,"Import","LOCALFileName")=DirFileName - - Quit tSC -} - -/// This client event, if present, is fired when the page is resized. -ClientMethod onresizeHandler() [ Internal, Language = javascript ] -{ - zenbody=document.getElementById("zenBody") - body=document.getElementById("body") - body.offsetHeight=body.parentNode.offsetHeight+'px' - body.style.height=body.parentNode.offsetHeight+'px' - body.offsetWidth=(zenbody.offsetWidth-20)+'px' - body.style.width=(zenbody.offsetWidth-20)+'px' -} - -Method SwitchSourceType() As %Status [ZenMethod] -{ - Set st=$$$OK - - Set sourceType=..%GetComponentById("SourceType") - Set sourceCSV=..%GetComponentById("SourceCSV") - Set sourceSQL=..%GetComponentById("SourceSQL") - Set sourceClass=..%GetComponentById("SourceClass") - Set sourceClassFile=..%GetComponentById("sourceClass") - - Set ..SourceType=sourceType.value - If sourceType.value="CSV" { - // Unhide CSV - Set sourceCSV.hidden=0 - - // Hide others - Set sourceSQL.hidden=1 - Set sourceClass.hidden=1 - - Set sourceClassFile.required=0 - } ElseIf sourceType.value="SQL" { - // Unhide SQL - Set sourceSQL.hidden=0 - - // Hide others - Set sourceCSV.hidden=1 - Set sourceClass.hidden=1 - - Set sourceClassFile.required=0 - } ElseIf sourceType.value="Class" { - // Unhide SQL - Set sourceClass.hidden=0 - - // Hide others - Set sourceCSV.hidden=1 - Set sourceSQL.hidden=1 - - Set sourceClassFile.required=1 - } - - Quit st -} - -/// Takes a CSV file and returns two stream IDs containing relating to a JSON object for the Properties and a JSON object for the Data -ClassMethod CSVToJSON(pFileName As %String = "C:\Users\psteiwer\Documents\simplecsv.csv", ByRef propertyJSONStreamId, ByRef dataJSONStreamId, pMaxLines As %Integer = 0, pHasHeaders As %Boolean = 1, pNewLine As %String = {$c(13,10)}) As %Status -{ - Set st=$$$OK - - Set stream=##class(%Stream.FileBinary).%New() - Do stream.LinkToFile(pFileName) - Set s=##class(AnalyzeThis.Utils).ReadStream(.stream) - Set pNewLine=##class(AnalyzeThis.Utils).DetermineNewLine(s) - Set s=$replace(s,$c(13),"\r") - Set s=$replace(s,$c(10),"\n") - Set pNewLine=$zconvert(pNewLine,"O","JS") - Set m=##class(%Regex.Matcher).%New(##class(AnalyzeThis.Utils).GetRegexLine(pNewLine)) - Set m.Text=s - Set m2=##class(%Regex.Matcher).%New(##class(AnalyzeThis.Utils).GetRegexProp(pNewLine)) - Set properties={} - Set propposition={} - Set displayarray=[] - Set typearray=[] - Set dateformatarray=[] - Set includearray=[] - Set intformatarray=[] - // ignorearray is an array that says if a column has an empty header - Set ignorearray=[] - Set dynamicArray=[] - Set pos=0 - - Try { - If ##class(AnalyzeThis.Utils).GetNextRegex(.m,.line) { - Set m2.Text=line - Set propcount=-1 - While ##class(AnalyzeThis.Utils).GetNextRegex(.m2,.prop,0) { - Set propcount=$i(propcount) - If pHasHeaders { - Set prop=$zstrip($zstrip(prop,">",","),">C") - Set:$e(prop,1)="""" prop=$e(prop,2,*) - Set:$e(prop,*)="""" prop=$e(prop,1,*-1) - Set prop=$zstrip(prop,"*P") - } Else { - Set prop="Property"_propcount - } - Do displayarray.%Push(prop) - Do typearray.%Push("%String") - Do dateformatarray.%Push("") - Do intformatarray.%Push("") - Do includearray.%Push("true") - Do ignorearray.%Push(prop="") - Do propposition.%Set(prop,propcount) - } - Do properties.%Set("Properties",propposition) - Do properties.%Set("Display",displayarray) - Do properties.%Set("Type",typearray) - Do properties.%Set("DateFormat",dateformatarray) - Do properties.%Set("IntFormat",intformatarray) - Do properties.%Set("Include",includearray) - Do properties.%Set("Ignore",ignorearray) - - If pHasHeaders { - Set pos=m.End-1 - } Else { - Do m.ResetPosition() - } - } Else { - // Headers too long, pick different file - } - - Set totalprops=propcount - Set done=0 - Set linecount=0 - } Catch ex { - Set st=$$$ERROR($$$GeneralError,"Error reading CSV") - } - - If $$$ISERR(st) Quit st - - Try { - While ('done)&&((pMaxLines=0)||(linecount",","),">C") - If ignorearray.%Get(propcount) { - If val'="" { - // If this row has data, stop processing the row - Set ignore=1 - Quit - } Else { - // If this row does not have data for this column, we can continue processing other columns - Continue - } - } - Set:$e(val,1)="""" val=$e(val,2,*) - Set:$e(val,*)="""" val=$e(val,1,*-1) - //w "Cell: "_val,! - Do dynamicObject.%Set(properties.Display.%Get(propcount),val) - } - If (totalprops=propcount) && ('ignore) { - Do dynamicArray.%Push(dynamicObject) - Set pos=m.End - } - } - - Set s=$e(s,pos,*) - If stream.AtEnd { - Set done=1 - } Else { - //If less than half available memory is used, double available memory - If $s<($zs/2*1024) { - Set $zs=$zs*2 - } - Set s=##class(AnalyzeThis.Utils).ReadStream(.stream,s) - Set s=$replace(s,$c(13),"\r") - Set s=$replace(s,$c(10),"\n") - Set m.Text=s - } - } - } Catch ex { - // Reached max len, pop one item from array - Set st=ex.AsStatus() - } - - If $$$ISERR(st) Quit st - - Set dataJSONStream=##class(%Stream.FileBinary).%New() - Do dynamicArray.%ToJSON(.dataJSONStream) - Do dataJSONStream.%Save() - Set dataJSONStreamId=dataJSONStream.%Id() - - Set propertyJSONStream=##class(%Stream.FileBinary).%New() - Do properties.%ToJSON(.propertyJSONStream) - Do propertyJSONStream.%Save() - Set propertyJSONStreamId=propertyJSONStream.%Id() - - Quit st -} - -/// Invoke class finder dialog. -ClientMethod browseClass(popupName) [ Language = javascript ] -{ - var mode="sourceclasses"; - zenLaunchPopupWindow('_DeepSee.UI.Dialog.finderDialog.cls?MODE='+encodeURIComponent(mode),popupName,'resizable,width=900,height=500'); -} -} +/// Created using the page template: Default +Class AnalyzeThis.UI.Dialog.CSVImport Extends %ZEN.Dialog.standardDialog [ System = 4 ] +{ + +Parameter DOMAIN = "AnalyzeThis"; + +Parameter Version = 1; + +Property INPUTTYPE As %ZEN.Datatype.string(ZENURL = "INPUTTYPE"); + +/// This is the temporary file name we saved on remote server, only saved when Input Type is LOCAL. +Property LOCALFILENAME As %ZEN.Datatype.string; + +Property LineSize As %Integer [ InitialExpression = 10000 ]; + +Property hasHeaders As %ZEN.Datatype.string [ InitialExpression = 1 ]; + +Property Source As %String; + +Property SourceClass As %String; + +Property CubeName As %String; + +Property propertyJSONStreamID As %ZEN.Datatype.string; + +Property dataJSONStreamID As %ZEN.Datatype.string; + +Property SourceType As %String; + +/// Id used to track progress. +Property trackingId As %ZEN.Datatype.string; + +/// This Style block contains page-specific CSS style definitions. +XData Style +{ + +} + +XData Contents [ XMLNamespace = "http://www.intersystems.com/zen" ] +{ + + +
+ + + +
+} + +/// This XML block defines the contents of this pane. +XData dialogBody [ XMLNamespace = "http://www.intersystems.com/zen" ] +{ + + +} + +/// Update the LineSize property +Method UpdateLineSize(value) [ Internal, ZenMethod ] +{ + Set ..LineSize=value +} + +/// Update the hasHeaders property +Method UpdateHasHeaders(value) [ Internal, ZenMethod ] +{ + Set ..hasHeaders=value +} + +/// Update the Source property +Method UpdateSource(value) [ Internal, ZenMethod ] +{ + Set ..Source=value + Do ..GenerateCubeName() +} + +/// Update the LOCALFILENAME property +Method UpdateLocalFile(value) [ Internal, ZenMethod ] +{ + Set ..LOCALFILENAME=value + + Set button=..%GetComponentById("btnSubmit") + Set button.disabled=(value="") + If value'="" { + Set button.controlClass="commandButton" + } Else { + Set button.controlClass="commandButtonDisabled" + } + + Do ..GenerateCubeName() +} + +/// Fire popup action to navigate to dashboard +Method seeDash() [ Internal, ZenMethod ] +{ + Set pcube=..CubeName + &JS +} + +/// Fire popup action to navigate to scorecard +Method seeScoreCard() [ Internal, ZenMethod ] +{ + Set pcube=..CubeName + &JS +} + +/// User changed to pick file between Remote and Local +ClientMethod doLocationChange(flag) [ Internal, Language = javascript ] +{ + if (flag == "LOCAL") { + zen("idRemoteFile").setHidden(true); + zen("LocalFile").setHidden(false); + // for local we cannot allow for Directory + // show Submit button + zen("btnNext").setHidden(true); + zen("btnSubmit").setHidden(false); + } else { + zen("idRemoteFile").setHidden(false); + zen("LocalFile").setHidden(true); + zen("btnNext").setHidden(false); + zen("btnSubmit").setHidden(true); + var Source=zen("FileName").getValue() + if (zen("FileName").getValue() != "") { + zen("btnNext").setProperty('disabled',Source==""); + zen("btnNext").setProperty('controlClass', (Source!="" ? 'commandButton' :'commandButtonDisabled')); + } + } +} + +/// Start the Import +ClientMethod doImportJSON() [ Language = javascript ] +{ + //Before processing, make sure all property names are unique + var count=1 + var ok=1 + var test=[]; + while (document.getElementById("propName"+count)!=null) { + document.getElementById("propName"+count).style="" + if (test[document.getElementById("propName"+count).value]!=null) { + document.getElementById("propName"+test[document.getElementById("propName"+count).value]).style.border="2px solid #FF0000" + document.getElementById("propName"+count).style.border="2px solid #FF0000" + ok=0 + } + test[document.getElementById("propName"+count).value]=count + count+=1 + } + + if (!ok) { + alert("Property names not unique."); + return + } + + zenPage.nextPage(); + zen("btnDone").setHidden(false); + zen("btnCancel").setHidden(true); + zenPage.updateState(); +} + +/// Update the state of the template buttons. +/// Subclasses should call this method when they need to +/// update the state of the footer buttons. +ClientMethod updateState() [ Language = javascript ] +{ + var btnBack=zen('btnBack'); + var btnNext=zen('btnNext'); + var btnFinish=zen('btnFinish'); + var btnGoStats=zen('btnGoStats'); + var btnHelp=zen('btnHelp'); + var btnSubmit=zen('btnSubmit'); + var multi=this.hasMultiplePages(); + + if (0) { + btnBack.setProperty('hidden',!this.canGoBack()); + btnBack.setProperty('disabled',!this.canGoBack()); + btnBack.setProperty('controlClass', (this.canGoBack() ? 'commandButton' :'commandButtonDisabled')); + } + if (btnNext) { + btnNext.setProperty('hidden',!this.canGoNext()); + btnNext.setProperty('disabled',!this.canGoNext()); + btnNext.setProperty('controlClass', (this.canGoNext() ? 'commandButton' :'commandButtonDisabled')); + } + // special for Import on this page. Hide regular Next and show Submit Next for LOCAL. + if (zen("InputType").getValue()=="LOCAL") { + btnSubmit.setProperty('disabled',true); + btnSubmit.setProperty('controlClass', 'commandButtonDisabled'); + } + if (btnFinish) { + btnFinish.setProperty('disabled',!this.canFinish()); + btnFinish.setProperty('hidden',!this.canFinish()); + btnFinish.setProperty('controlClass', (this.canFinish() ? 'commandButton' :'commandButtonDisabled')); + } +} + +/// Kicks off GenerateAll and starts the update timer +ClientMethod updateProgress() [ Language = javascript ] +{ + var file=this.Source + if (this.INPUTTYPE=='LOCAL') { + file=this.LOCALFILENAME + } + this.trackingId=this.GenerateAll(this.SourceType,file,this.LineSize,this.hasHeaders,this.CubeName,this.propertyJSONStreamID,this.dataJSONStreamID); + if (this.trackingId!='') { + zen('timer').timeout=500; + zen('timer').startTimer(); + } +} + +/// Update the status area. +ClientMethod updateStatus() [ Language = javascript ] +{ + var status=this.CheckStatus(this.trackingId); + var html=zen('htmlContent3'); + if (status != '') { + html.setContent(status); + zen('timer').timeout=250; + zen('timer').startTimer(); + } + else { + this.trackingId=''; + + if (zenPage.DashboardExists()) { + zen("btnGoDash").setProperty('disabled',0); + zen("btnGoDash").setProperty('hidden',0); + zen("btnGoDash").setProperty('controlClass','commandButton'); + + // If the dashboard was generated, we know it is safe to proceed to stats tab + zen("btnGoStats").setProperty('disabled',!this.canStat()); + zen("btnGoStats").setProperty('hidden',!this.canStat()); + zen("btnGoStats").setProperty('controlClass', (this.canStat() ? 'commandButton' :'commandButtonDisabled')); + } + + if (zenPage.ScoreCardExists()) { + zen("btnScoreDash").setProperty('disabled',0); + zen("btnScoreDash").setProperty('hidden',0); + zen("btnScoreDash").setProperty('controlClass','commandButton'); + } + } + return; +} + +ClientMethod showMsg(msg) [ Language = javascript ] +{ + zen("idRespond").setValue(msg); + zen("idRespond").setHidden(false); +} + +/// Get the current status of the background task. +ClassMethod CheckStatus(pTrackingId As %String) As %String [ ZenMethod ] +{ + Set tOutput="" + Set tIsComplete=0 + Set tIsErr=0 + + Merge tTracking=^AnalyzeThis.GenerateTracking(pTrackingId) + + Set currStep=$O(tTracking(""),-1) + If currStep="" Quit tOutput + For tStep=1:1:currStep { + + Set tPhase=$Case(tStep, + 1:"Preparing JSON", + 2:"Generating Source Class", + 3:"Importing Data", + 4:"Generating Cube", + 5:"Populating Cube", + :"Generating Sample Dashboard") + + Set tStatus=tTracking(tStep) + + If tStatus="Working..." { + // Leave message as "Working..." + } ElseIf $$$ISERR(tStatus) { + Set tStatus=$System.Status.GetErrorText(tStatus) + Set tIsErr=1 + } Else { + Set tStatus="Complete" + If tStep=6 { + Set tIsComplete=1 + } + } + + Set tOutput=tOutput _ "" + Set tOutput=tOutput _ ""_tPhase_"" + Set tOutput=tOutput _ ""_tStatus_"" + Set tOutput=tOutput _ "" + } + For tStep=currStep+1:1:6 { + //Set tOutput=tOutput _ "" + Set tOutput=tOutput _ "" + + Set tPhase=$Case(tStep, + 1:"Preparing JSON", + 2:"Generating Source Class", + 3:"Importing Data", + 4:"Generating Cube", + 5:"Populating Cube", + :"Generating Sample Dashboard") + + Set tOutput=tOutput _ ""_tPhase_"" + Set tOutput=tOutput _ "Not Yet Started" + Set tOutput=tOutput _ "" + } + + If (tOutput'="") { + Set tOutput="
"_$$$Text("Status")_ "
" _ tOutput _ "
" + } + + If (tIsComplete || tIsErr) { + Kill ^AnalyzeThis.GenerateTracking(pTrackingId) + } + Quit tOutput +} + +/// Start the build cube process in the background. +/// Return the tracking id. +ClassMethod GenerateAll(pSourceType, pSource, pLineSize, pHasHeaders, pCubeName, pPropertyJSONStreamId, pDataJSONStreamId) As %String [ ZenMethod ] +{ + Set tTrackingId="" + Set tJobTimeOut=10 + + Job ##class(AnalyzeThis.Generator).GenerateAll(pSourceType,pSource, pLineSize, pHasHeaders, pCubeName, pPropertyJSONStreamId, pDataJSONStreamId)::tJobTimeOut + If '$Test { + &js + Set tTrackingId="" + } + Else { + Set tTrackingId=$ZChild + } + Quit tTrackingId +} + +/// Using the current cube name, check if a dashboard was generated for this cube +Method DashboardExists() As %Status [ ZenMethod ] +{ + Set dashName="Generated/Samples for "_..CubeName_".dashboard" + Set st=##class(%DeepSee.Dashboard.Utils).%DashboardExists(dashName) + Quit st +} + +/// Using the current cube name, check if a scorecard was generated for this cube +Method ScoreCardExists() As %Status [ ZenMethod ] +{ + Set dashName="Generated/Sample ScoreCard for "_..CubeName_".dashboard" + Set st=##class(%DeepSee.Dashboard.Utils).%DashboardExists(dashName) + Quit st +} + +/// Display cube stats in the status area +ClientMethod drawStats() [ Language = javascript ] +{ + document.getElementById("statsTable").innerHTML="

Generating Stats...

" + document.getElementById("statsTable").innerHTML=zenPage.GetStats() + zenPage.updateState(); +} + +/// Generate stats display +Method GetStats() As %String [ ZenMethod ] +{ + Set st=$$$OK + + Set mdx="SELECT [Measures].[%COUNT] on 1 from ["_..CubeName_"]" + Set rs=##class(%DeepSee.ResultSet).%ExecuteDirect(mdx) + Set totalmembers=rs.%GetOrdinalValue(1) + Set html="

"_totalmembers_" facts have been built for Cube "_..CubeName_"

" + Set st=##class(%DeepSee.Utils).%GetDimensionList(..CubeName,.info) + Set d=$order(info("")) + While d'="" { + Set h=$order(info(d,"")) + While h'="" { + Set l=$order(info(d,h,"")) + While l'="" { + If $lg(info(d,h,l),1)="l" { + Set spec="["_$lg(info(d,h,l),2)_"].["_$lg(info(d,h,l),3)_"].["_$lg(info(d,h,l),4)_"]" + Set mdx="SELECT COUNT("_spec_".Members) on 1 from ["_..CubeName_"]" + Set rs=##class(%DeepSee.ResultSet).%ExecuteDirect(mdx) + Set members=rs.%GetOrdinalValue(1) + + Set mdx="SELECT ISNULL("_spec_".&[],0)/[Measures].[%COUNT] on 1 from ["_..CubeName_"]" + Set rs=##class(%DeepSee.ResultSet).%ExecuteDirect(mdx) + Set nullpercent=(((rs.%GetOrdinalValue(1)*10000)\1)/100) + Set html=html_"" + } + Set l=$order(info(d,h,l)) + } + Set h=$order(info(d,h)) + } + Set d=$order(info(d)) + } + Set html=html_"
Level NameNumber of MembersPercentage of Null Values
"_spec_""_members_""_nullpercent_"%
" + Quit html +} + +/// Using file properties, create the cube name +Method GenerateCubeName() [ ZenMethod ] +{ + If ..SourceType="CSV" { + Set file="" + If ..%GetValueById("InputType")="LOCAL" { + Set file=..LOCALFILENAME + } Else { + Set file=..Source + } + If $length(file,"\")>$length(file,"/") { + Set tName=$zstrip($replace($piece($piece(file,"\",*),".",1),"DEEPSEE",""),"*PCW") + } Else { + Set tName=$zstrip($replace($piece($piece(file,"/",*),".",1),"DEEPSEE",""),"*PCW") + } + } ElseIf ..SourceType="Class" { + Set tName=..Source + // Class should end in .cls, extract package + .cls extension to get class name + // If class does not end in .cls, handle appropriately + Set tCls=1 + If $zconvert($extract(tName,*-3,*),"U")'=".CLS" { + Set tCls=0 + } + Set tName=$piece(tName,".",*-tCls) + } Else { + // SouceType not supported to auto generate class name + Quit + } + + Set ..CubeName=tName + Set ..SourceClass="AnalyzeThis.Generated."_tName + Do ..%SetValueById("CubeName",tName) +} + +/// Validates and updates the cube name property +Method UpdateCubeName(tName) [ ZenMethod ] +{ + If ($$$UPPER($ZStrip(tName," + } + Set ..CubeName=tName + Set ..SourceClass="AnalyzeThis.Generated."_tName +} + +/// This method is fired up after onDrawContent is finished. Hide progress message and display error if exists. +ClientMethod drawDone(tab) [ Language = javascript ] +{ + + var result=this.GetStatus(tab); + var id="idProgress"+tab; + if (result.Status == "Done") { + if (result.Error != "") { + this.showMsg(result.Error); + zen("btnFinish").setProperty('disabled',true); + zen("btnFinish").setProperty('controlClass','commandButtonDisabled'); + } + } else { + zen("btnNext").setProperty('disabled',true); + zen("btnNext").setProperty('controlClass','commandButtonDisabled'); + zen("btnFinish").setProperty('disabled',true); + zen("btnFinish").setProperty('controlClass','commandButtonDisabled'); + } + zen(id).setHidden(true); +} + +/// Return true if this template can go to the next page (i.e., enable +/// the Next button).
+ClientMethod canGoNext() [ Language = javascript ] +{ + var tabGroup=zen('tabGroup'); + var tabNo=tabGroup.getCurrTabNo(); + var flag=false; + switch(tabNo) { + case 1: + flag=true; + if (this.INPUTTYPE == "LOCAL") flag=false; + break; + case 2: + break; + case 3: + break; + case 4: + break; + } + return flag; +} + +/// Return true if this template can go to the stats page (i.e., enable +/// the stats button).
+ClientMethod canStat() [ Language = javascript ] +{ + var tabGroup=zen('tabGroup'); + var tabNo=tabGroup.getCurrTabNo(); + var flag=false; + switch(tabNo) { + case 1: + break; + case 2: + break; + case 3: + flag=true; + break; + case 4: + break; + } + return flag; +} + +/// Return true if this template can go to the previous page (i.e., enable +/// the Back button).
+/// This is implemented by subclasses. +ClientMethod canGoBack() [ Language = javascript ] +{ + var tabGroup=zen('tabGroup'); + var tabNo=tabGroup.getCurrTabNo(); + var flag=true; + switch(tabNo) { + case 1: + flag=false; + break; + case 2: + break; + case 3: + break; + case 4: + break; + } + return flag +} + +/// Return true if this template can Finish (i.e., enable +/// the Finish button).
+ClientMethod canFinish() [ Language = javascript ] +{ + var tabGroup=zen('tabGroup'); + var tabNo=tabGroup.getCurrTabNo(); + var flag=false; + switch(tabNo) { + case 1: + break; + case 2: + flag=true; + break; + case 3: + break; + case 4: + break; + } + + return flag; +} + +ClientMethod hasMultiplePages() [ Language = javascript ] +{ + return true; +} + +/// Go to the next page of the template (if there is one).
+ClientMethod nextPage() [ Language = javascript ] +{ + var tabGroup=zen('tabGroup'); + var tabNo=tabGroup.getCurrTabNo(); + var html=zen("htmlContent"); + this.showMsg("",1); + switch(tabNo) { + case 1: + var SourceType=zen("SourceType").getValue(); + + if (SourceType=="CSV") { + var Source=zen("FileName").getValue(); + var inputtype="REMOTE" + if (this.LOCALFILENAME != "") { + Source=this.LOCALFILENAME; + inputtype="LOCAL"; + } + var ok=this.ValidateFile(Source); + if (ok == 0) { + this.showMsg($$$Text("File required."),1); + zen("FileName").focus(); + break; + } + if (ok == -1) { + this.showMsg($$$Text("You have entered a directory path without a file name."),1); + zen("FileName").focus(); + break; + } + if (ok == -2) { + this.showMsg($$$Text("File does not exist!"),1); + zen("FileName").focus(); + break; + } + if (ok == -3) { + this.showMsg($$$Text("File Name cannot start with a number!"),1); + zen("FileName").focus(); + break; + } + html.setProperty('seed',Source); + } else if (SourceType=="SQL") { + var sql=zen("SQLText").getValue(); + if (sql=="") { + this.showMsg($$$Text("SQL Query required."),1); + zen("SQLText").focus(); + break; + } + html.setProperty('seed',sql); + } else if (SourceType=="Class") { + var sourceClass=zen("sourceClass").getValue(); + if (sourceClass=="") { + this.showMsg($$$Text("Source Class required."),1); + zen("sourceClass").focus(); + break; + } + html.setProperty('seed',sourceClass); + } + + if (zen("CubeName").getValue()=="") { + this.showMsg($$$Text("Cube Name required."),1); + zen("CubeName").focus(); + break; + } + + var uniquename=zenPage.UniqueName(zen("CubeName").getValue()); + if (uniquename!="") { + this.showMsg($$$Text(uniquename),1); + zen("CubeName").focus(); + break; + } + + tabGroup.showNextTab(); + break; + case 2: + var html=zen("htmlContent3"); + html.setProperty('seed',1); + tabGroup.showNextTab(); + break; + case 3: + var html=zen("htmlContent4"); + html.setProperty('seed',1); + tabGroup.showNextTab(); + break; + case 4: + break; + } + zenPage.updateState(); +} + +/// Go to the previous page of the template (if there is one).
+/// This is implemented by subclasses. +ClientMethod previousPage() [ Language = javascript ] +{ + var tabGroup=zen('tabGroup'); + var tabNo=tabGroup.getCurrTabNo(); + switch(tabNo) { + case 1: + break; + case 2: + tabGroup.showPreviousTab(); + zenPage.updateState(); + if (zen("InputType").getValue()=="LOCAL") { + zen('btnNext').setProperty('hidden',true); + } + break; + case 3: + tabGroup.showPreviousTab(); + zenPage.updateState(); + break; + case 4: + tabGroup.showPreviousTab(); + zenPage.updateState(); + break; + } +} + +/// This client event, if present, is fired when the page is loaded. +ClientMethod onloadHandler() [ Internal, Language = javascript ] +{ + this.invokeSuper('onloadHandler',arguments); + if (this.InvalidParam) { + zen("btnNext").setHidden(true); + } + this.onstartHandler(); +} + +ClientMethod getDialogValue() [ Internal, Language = javascript ] +{ + return "" +} + +/// User clicked the Browse button. +ClientMethod browseSelect(name) [ Internal, Language = javascript ] +{ + var Dir=zen(name).getValue(); + zenLaunchPopupWindow('%ZEN.Dialog.fileSelect.zen?Dir='+encodeURIComponent(Dir)+'&wildcard=*.csv&showdirectoryonly=0','FileSelect','resizable,width=600,height=700'); +} + +/// Returning from file select OR qualifers dialog and setting the value into the appropriate field. +/// The id of the field that needs to be updated is saved in "dialogClicked". +ClientMethod onPopupAction(popupName, action, value) [ Internal, Language = javascript ] +{ + if (action=="ok") { + if (popupName=="sourceClass") { + zenPage.getComponentById(popupName).setValue(value); + zenPage.UpdateSource(value); + zenPage.updateState(); + } else if (popupName=="FileSelect") { + zen("FileName").setValue(value); + zenPage.UpdateSource(value); + zenPage.GenerateCubeName(); + zenPage.updateState(); + } + } +} + +/// This is called when the template is first displayed; +/// This provides a chance to load the last filetype, etc. +ClientMethod onstartHandler() [ Internal, Language = javascript ] +{ + this.onresizeHandler(); + // if this is from the submit of the Local file, load the content of the file on tab 2. + if (this.INPUTTYPE == "LOCAL") { + // set the first tab for LOCAL correctly + this.doLocationChange("LOCAL"); + if (this.LOCALFILENAME != "") { + var html=zen("htmlContent"); + html.setProperty('seed',this.LOCALFILENAME); + var tabGroup=zen('tabGroup'); + var tabNo=tabGroup.getCurrTabNo(); + if (tabNo == 1) { + tabGroup.showNextTab(); + } + zenPage.updateState(); + } + zen("btnNext").setHidden(true); + } +} + +/// Get the (localized) title string for the dialog. +/// This should be implemented in a subclass. +Method %OnGetTitle() As %String [ Internal ] +{ + Quit $$$TextHTML("Import Data") +} + +/// Get the (localized) subtitle string for the dialog. +/// This should be implemented in a subclass. +Method %OnGetSubtitle() As %String [ Internal ] +{ + Quit "" +} + +/// This callback is called after the server-side page +/// object and all of its children are created.
+/// Subclasses can override this to add, remove, or modify +/// items within the page object model, or to provide values +/// for controls. +Method %OnAfterCreatePage() As %Status [ Internal ] +{ + Set tSC=##super() + If $$$ISERR(tSC) Quit tSC + + // try and determine if user is on *same* machine as server + // if so, do not provide local upload option + Set tTCPAddr=$SYSTEM.TCPDevice.PeerAddr(0) + Set tClientAddr=$G(%request.CgiEnvs("REMOTE_ADDR")) + Set tIsLocal=((tClientAddr="127.0.0.1") ! (tClientAddr="::1")) & ((tTCPAddr="127.0.0.1") ! (tTCPAddr="::1")) + If tIsLocal { + Set ..%GetComponentById("InputType").hidden=1 + Set ..%GetComponentById("LocalFile").hidden=1 + } + Set ..SourceType=..%GetComponentById("SourceType").value + Set Source="" + Set ..%GetComponentById("FileName").value=Source + #; Set for Remote input types choice: ServerName or local machine + Set ..%GetComponentById("InputType").displayList=$zu(110)_","_$$$Text("My Local Machine") + If $G(%request.Data("INPUTTYPE",1))="LOCAL" { + Set %page.LOCALFILENAME=$G(%session.Data($Username,"Import","LOCALFileName")) + Set ..%GetComponentById("InputType").value="LOCAL" + } + If $G(%request.Data("LineSize",1)) '= "" { + Set %page.LineSize=$G(%request.Data("LineSize",1)) + Set ..%GetComponentById("LineSize").value=$G(%request.Data("LineSize",1)) + Set ..LineSize=$G(%request.Data("LineSize",1)) + } + If $G(%request.Data("CubeName",1)) '= "" { + Set %page.CubeName=$G(%request("CubeName",1)) + Set ..%GetComponentById("CubeName").value=$G(%request.Data("CubeName",1)) + Set ..CubeName=$G(%request.Data("CubeName",1)) + } + Quit $$$OK +} + +ClassMethod DrawSortPageTitle(pSeed As %String) As %Status [ Internal ] +{ + Set tPageTitle="Please sort each column as Dimension, Date, or Measure. "_ + "
A Measure might be something like revenue, or total discharges.
If it is a measure, please specify if it is a number or currency."_ + "
A Dimension would be things like zip code, state, or measure description
" + &html<
#(tPageTitle)# +

> + Quit 1 +} + +ClassMethod DrawPageTitle(pSeed As %String) As %Status [ Internal ] +{ + Set tPageTitle="Import Data" + &html<
#(tPageTitle)# +

> + Quit 1 +} + +/// Draw preview content of the CSV file for user to confirm properties +Method PropertyCheckJSON(pSeed) As %Status [ ZenMethod ] +{ + Set tSC=$$$OK + + Quit:pSeed="" tSC + + // Get JSON from CSV + Set file="" + If ..INPUTTYPE="LOCAL" { + Set file=..LOCALFILENAME + } Else { + Set file=..Source + } + If ..SourceType="SQL" { + Set tSC=##class(AnalyzeThis.Utils).SQLToJSON(pSeed,.propertiesJSONStreamId,.dataJSONStreamId,10) + } ElseIf ..SourceType="Class" { + // Get the SQL table name and then pass through SQL method + Set sqlQuery=##class(AnalyzeThis.Utils).ClassToQuery(pSeed) + Set tSC=##class(AnalyzeThis.Utils).SQLToJSON(sqlQuery,.propertiesJSONStreamId,.dataJSONStreamId,10) + } Else { + Set tSC=##class(AnalyzeThis.UI.Dialog.CSVImport).CSVToJSON(file,.propertiesJSONStreamId,.dataJSONStreamId,10,..hasHeaders) + } + + Quit:$$$ISERR(tSC) tSC + + Set ..propertyJSONStreamID=propertiesJSONStreamId + Set ..dataJSONStreamID=dataJSONStreamId + + Set properties=##class(%Stream.FileBinary).%OpenId(propertiesJSONStreamId) + Set propertyObj={}.%FromJSON(properties) + + Set data=##class(%Stream.FileBinary).%OpenId(dataJSONStreamId) + Set dataArray=[].%FromJSON(data) + + Quit:$$$ISERR(tSC) tSC + + &html<> + &html<> + Set iter=propertyObj.Display.%GetIterator() + While iter.%GetNext(.key,.val) { + If propertyObj.Ignore.%Get(key) { + Continue + } + Set format=0 + &html<> + } + &html<> + &html<> + Set iter=propertyObj.Display.%GetIterator() + While iter.%GetNext(.key,.val) { + If propertyObj.Ignore.%Get(key) { + Continue + } + &html<> + } + &html<> + Set iter=dataArray.%GetIterator() + While ((iter.%GetNext(.key,.val)) && (key<10)) { + If propertyObj.Ignore.%Get(key) { + Continue + } + &html<> + Set tPropNum=-1 + Set iter2=val.%GetIterator() + While iter2.%GetNext(.key2,.val2) { + &html<> + } + &html<> + } + &html<
Include?

#(val2)#
> + Set %session.Data($Username,"Import","ContentStatus") = "Done" + Quit tSC +} + +/// Changes formatting options based on the type +ClientMethod swapFormat(propID, value) [ Internal, Language = javascript ] +{ + if ((value=='%Date') || (value=='%TimeStamp')) { + document.getElementById("dateFormat"+propID).style.display="" + document.getElementById("intFormat"+propID).style.display="none" + + zenPage.autoDateFormat(propID); + } else if (value=='%Integer') { + document.getElementById("intFormat"+propID).style.display="" + document.getElementById("dateFormat"+propID).style.display="none" + } else { + document.getElementById("dateFormat"+propID).style.display="none" + document.getElementById("intFormat"+propID).style.display="none" + } +} + +/// Try and automatically select the date format +ClientMethod autoDateFormat(propID) [ Internal, Language = javascript ] +{ + var dates=[]; + for (var i=0;i<10;i++) { + var prop=document.getElementById("p"+propID+"r"+i); + if (prop==null) { + return + } + var temp=parseInt(zenPage.findDateType(prop.innerHTML)); + if (typeof dates[temp]=="undefined") { + dates[temp]=1; + } else { + dates[temp]+=1; + } + } + + var typemaxval=0; + var typemaxpos=0; + for (var i=0;itypemaxval) { + typemaxpos=i; + typemaxval=dates[i]; + } + } + + // Can enhance this later to keep type 0 and prevent user from going to next page if type=0 + if (typemaxpos==0) { + typemaxpos=1; + } + + document.getElementById("dateFormat"+propID).value=typemaxpos; + document.getElementById("dateFormat"+propID).onchange(); +} + +/// As changes are made to the properties, update the JSON stream +Method updateProp(key, which, value, stream) [ ZenMethod ] +{ + Set propertyJSON=##class(%Stream.FileBinary).%OpenId(stream) + Set propertyObj={}.%FromJSON(propertyJSON) + + Do propertyObj.%Get(which).%Set(key,value) + + Do propertyObj.%ToJSON(.propertyJSON) + Do propertyJSON.%Save() +} + +Method findDateType(pValue) As %Integer [ ZenMethod ] +{ + Quit ##class(AnalyzeThis.UI.Dialog.CSVImport).FindType(pValue) +} + +/// Given a value, find the best guess of type +ClassMethod FindType(value) As %Integer +{ + Set type=0 + + If (value?1.2N1"/"1.2N1"/"2.4N) { + // [N]N/[N]N/[NN]NN + Set type=1 + } ElseIf (value?2N1" "1(1(1"J",1"j")1(1"A",1"a")1(1"N",1"n"),1(1"F",1"f")1(1"E",1"e")1(1"B",1"b"),1(1"M",1"m")1(1"A",1"a")1(1"R",1"r"),1(1"A",1"a")1(1"P",1"p")1(1"R",1"r"),1(1"M",1"m")1(1"A",1"a")1(1"Y",1"y"),1(1"J",1"j")1(1"U",1"u")1(1"N",1"n"),1(1"J",1"j")1(1"U",1"u")1(1"L",1"l"),1(1"A",1"a")1(1"U",1"u")1(1"G",1"g"),1(1"S",1"s")1(1"E",1"e")1(1"P",1"p"),1(1"O",1"o")1(1"C",1"c")1(1"T",1"t"),1(1"N",1"n")1(1"O",1"o")1(1"V",1"v"),1(1"D",1"d")1(1"E",1"e")1(1"C",1"c"))1" "2.4N) { + // NN (Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec) [NN]NN + Set type=2 + } ElseIf (value?4N1"-"1.2N1"-"1.2N) { + // NNNN-[N]N-[N]N + Set type=3 + } ElseIf (value?2N1"/"2N1"/"2.4N) { + // NN/NN/[NN]NN + Set type=4 + } ElseIf ((value?1"18"2N)||(value?1"19"2N)||(value?1"20"2N)||(value?1"21"2N)) { + // 18NN, 19NN, 20NN, 21NN + Set type=5 + } ElseIf (value?5N) { + // NNNNN, $H format + Set type=30 + } + + Quit type +} + +/// tab = 2: Content - when DrawContent is finished, Status is "Done". +ClassMethod GetStatus(tab) As %ZEN.proxyObject [ Internal, ZenMethod ] +{ + Set proxy=##class(%ZEN.proxyObject).%New() + Set proxy.Status=$G(%session.Data($Username,"Import","ContentStatus")) + Set proxy.Error=$G(%session.Data($Username,"Import","ContentError")) + Quit proxy +} + +/// Validate file name. +ClassMethod ValidateFile(FILE) As %Integer [ Internal, ZenMethod ] +{ + If $ZStrip(FILE,"<>W")="" Quit 0 + #; Entered directory only! + If ##class(%File).DirectoryExists(FILE) Quit -1 + #; File does not exist!" + If '##class(%File).Exists(FILE) Quit -2 + Set tFILE=$PIECE(FILE,"\",*) + If ($$$UPPER($ZStrip(tFILE," +/// The necessary parameters are passed in URL. The temp file name is saved in %session and to be used in loading content. +ClassMethod %OnSubmit(pSubmit As %ZEN.Submit) As %Status [ Internal ] +{ + Set tStream=pSubmit.%GetStream("LocalFile") + Set dir=##class(%File).SubDirectoryName($zu(12),"Temp") + #; If temp directory does not exist, create it now + If ##class(%File).DirectoryExists(dir)=0 { + Do ##class(%File).CreateDirectory(dir) + } + #; Get proper delimeter since SubDirectoryName does not include delimeter + Set tDelim="\" + If $$$isUNIX Set tDelim="/" + If $$$isVMS Set tDelim="" + #; Give it a name so it won't be deleted automatically + Set DirFileName=dir_tDelim_pSubmit.%Data("CubeName")_".stream" + Set file=##class(%Stream.FileBinary).%New() + Set tSC=file.LinkToFile(DirFileName) + If tStream '= "" { + #; Copy the stream from local server + Set tSC=file.CopyFrom(tStream) + #; Save it to the file stream on the remote server + If tSC Set tSC=file.%Save() + } + Set %response.Context("INPUTTYPE")="LOCAL" + Set %response.Context("LineSize")=pSubmit.%Data("LineSize") + Set %response.Context("CubeName")=pSubmit.%Data("CubeName") + // remember popup info + Set %response.Context("$ZEN_POPUP")=1 + Set %response.Context("$ZEN_POPUPPARENT")=+$G(%request.Data("$ZEN_POPUPPARENT",1)) + Set %response.Context("$ZEN_SOFTMODAL")=..%OnUseSoftModals() + // save the temp local file name to be used in later to load content + Set %session.Data($Username,"Import","LOCALFileName")=DirFileName + + Quit tSC +} + +/// This client event, if present, is fired when the page is resized. +ClientMethod onresizeHandler() [ Internal, Language = javascript ] +{ + zenbody=document.getElementById("zenBody") + body=document.getElementById("body") + body.offsetHeight=body.parentNode.offsetHeight+'px' + body.style.height=body.parentNode.offsetHeight+'px' + body.offsetWidth=(zenbody.offsetWidth-20)+'px' + body.style.width=(zenbody.offsetWidth-20)+'px' +} + +Method SwitchSourceType() As %Status [ ZenMethod ] +{ + Set st=$$$OK + + Set sourceType=..%GetComponentById("SourceType") + Set sourceCSV=..%GetComponentById("SourceCSV") + Set sourceSQL=..%GetComponentById("SourceSQL") + Set sourceClass=..%GetComponentById("SourceClass") + Set sourceClassFile=..%GetComponentById("sourceClass") + + Set ..SourceType=sourceType.value + If sourceType.value="CSV" { + // Unhide CSV + Set sourceCSV.hidden=0 + + // Hide others + Set sourceSQL.hidden=1 + Set sourceClass.hidden=1 + + Set sourceClassFile.required=0 + } ElseIf sourceType.value="SQL" { + // Unhide SQL + Set sourceSQL.hidden=0 + + // Hide others + Set sourceCSV.hidden=1 + Set sourceClass.hidden=1 + + Set sourceClassFile.required=0 + } ElseIf sourceType.value="Class" { + // Unhide SQL + Set sourceClass.hidden=0 + + // Hide others + Set sourceCSV.hidden=1 + Set sourceSQL.hidden=1 + + Set sourceClassFile.required=1 + } + + Quit st +} + +/// Takes a CSV file and returns two stream IDs containing relating to a JSON object for the Properties and a JSON object for the Data +ClassMethod CSVToJSON(pFileName As %String = "C:\Users\psteiwer\Documents\simplecsv.csv", ByRef propertyJSONStreamId, ByRef dataJSONStreamId, pMaxLines As %Integer = 0, pHasHeaders As %Boolean = 1, pNewLine As %String = {$c(13,10)}) As %Status +{ + Set st=$$$OK + + Set stream=##class(%Stream.FileBinary).%New() + Do stream.LinkToFile(pFileName) + Set s=##class(AnalyzeThis.Utils).ReadStream(.stream) + Set pNewLine=##class(AnalyzeThis.Utils).DetermineNewLine(s) + Set s=$replace(s,$c(13),"\r") + Set s=$replace(s,$c(10),"\n") + Set pNewLine=$zconvert(pNewLine,"O","JS") + Set m=##class(%Regex.Matcher).%New(##class(AnalyzeThis.Utils).GetRegexLine(pNewLine)) + Set m.Text=s + Set m2=##class(%Regex.Matcher).%New(##class(AnalyzeThis.Utils).GetRegexProp(pNewLine)) + Set properties={} + Set propposition={} + Set displayarray=[] + Set typearray=[] + Set dateformatarray=[] + Set includearray=[] + Set intformatarray=[] + // ignorearray is an array that says if a column has an empty header + Set ignorearray=[] + Set dynamicArray=[] + Set pos=0 + + Try { + If ##class(AnalyzeThis.Utils).GetNextRegex(.m,.line) { + Set m2.Text=line + Set propcount=-1 + While ##class(AnalyzeThis.Utils).GetNextRegex(.m2,.prop,0) { + Set propcount=$i(propcount) + If pHasHeaders { + Set prop=$zstrip($zstrip(prop,">",","),">C") + Set:$e(prop,1)="""" prop=$e(prop,2,*) + Set:$e(prop,*)="""" prop=$e(prop,1,*-1) + Set prop=$zstrip(prop,"*P") + } Else { + Set prop="Property"_propcount + } + Do displayarray.%Push(prop) + Do typearray.%Push("%String") + Do dateformatarray.%Push("") + Do intformatarray.%Push("") + Do includearray.%Push("true") + Do ignorearray.%Push(prop="") + Do propposition.%Set(prop,propcount) + } + Do properties.%Set("Properties",propposition) + Do properties.%Set("Display",displayarray) + Do properties.%Set("Type",typearray) + Do properties.%Set("DateFormat",dateformatarray) + Do properties.%Set("IntFormat",intformatarray) + Do properties.%Set("Include",includearray) + Do properties.%Set("Ignore",ignorearray) + + If pHasHeaders { + Set pos=m.End-1 + } Else { + Do m.ResetPosition() + } + } Else { + // Headers too long, pick different file + } + + Set totalprops=propcount + Set done=0 + Set linecount=0 + } Catch ex { + Set st=$$$ERROR($$$GeneralError,"Error reading CSV") + } + + If $$$ISERR(st) Quit st + + Try { + While ('done)&&((pMaxLines=0)||(linecount",","),">C") + If ignorearray.%Get(propcount) { + If val'="" { + // If this row has data, stop processing the row + Set ignore=1 + Quit + } Else { + // If this row does not have data for this column, we can continue processing other columns + Continue + } + } + Set:$e(val,1)="""" val=$e(val,2,*) + Set:$e(val,*)="""" val=$e(val,1,*-1) + //w "Cell: "_val,! + Do dynamicObject.%Set(properties.Display.%Get(propcount),val) + } + If (totalprops=propcount) && ('ignore) { + Do dynamicArray.%Push(dynamicObject) + Set pos=m.End + } + } + + Set s=$e(s,pos,*) + If stream.AtEnd { + Set done=1 + } Else { + //If less than half available memory is used, double available memory + If $s<($zs/2*1024) { + Set $zs=$zs*2 + } + Set s=##class(AnalyzeThis.Utils).ReadStream(.stream,s) + Set s=$replace(s,$c(13),"\r") + Set s=$replace(s,$c(10),"\n") + Set m.Text=s + } + } + } Catch ex { + // Reached max len, pop one item from array + Set st=ex.AsStatus() + } + + If $$$ISERR(st) Quit st + + Set dataJSONStream=##class(%Stream.FileBinary).%New() + Do dynamicArray.%ToJSON(.dataJSONStream) + Do dataJSONStream.%Save() + Set dataJSONStreamId=dataJSONStream.%Id() + + Set propertyJSONStream=##class(%Stream.FileBinary).%New() + Do properties.%ToJSON(.propertyJSONStream) + Do propertyJSONStream.%Save() + Set propertyJSONStreamId=propertyJSONStream.%Id() + + Quit st +} + +/// Invoke class finder dialog. +ClientMethod browseClass(popupName) [ Language = javascript ] +{ + var mode="sourceclasses"; + zenLaunchPopupWindow('_DeepSee.UI.Dialog.finderDialog.cls?MODE='+encodeURIComponent(mode),popupName,'resizable,width=900,height=500'); +} + +} diff --git a/AnalyzeThis/Utils.cls b/AnalyzeThis/Utils.cls index 7ad8dad..521e682 100644 --- a/AnalyzeThis/Utils.cls +++ b/AnalyzeThis/Utils.cls @@ -1,222 +1,222 @@ -Class AnalyzeThis.Utils -{ - -ClassMethod GetVersion() As %String -{ - Quit "v1.1.3" -} - -ClassMethod GetNextRegex(ByRef pMatch,ByRef pResult,pIncludeTerm As %Boolean = 1) As %Boolean -{ - Set pResult="" - - If '$IsObject(pMatch) { - Quit 0 - } - - Set tStart=$Case(pMatch.End,-2:1,:pMatch.End) - Set tBool=pMatch.Locate() - Quit:tBool=0 tBool - Set tEnd=pMatch.End - - Set pResult=$e(pMatch.Text,tStart,$select(pIncludeTerm=1:pMatch.End,1:pMatch.Start)-1) - - Quit tBool -} - -ClassMethod GetRegexLine(pNewLine As %String) As %String -{ - Quit $replace(pNewLine,"\","\\")_"(?=(?:(?:\\.|[^""\\])*""(?:\\.|[^""\\])*"")*(?:\\.|[^""\\])*\Z)" - -} - -ClassMethod GetRegexProp(pNewLine As %String) As %String -{ - Quit "("_$replace(pNewLine,"\","\\")_"|,)"_"(?=(?:(?:\\.|[^""\\])*""(?:\\.|[^""\\])*"")*(?:\\.|[^""\\])*\Z)" -} - -ClassMethod DetermineNewLine(pString As %String) As %String -{ - Set tNewLine=$c(13)_$c(10) - - If ($find(pString,$c(10))=0)&&($find(pString,$c(13))>0) { - Set tNewLine=$c(13) - } ElseIf ($find(pString,$c(10))>0)&&($find(pString,$c(13))=0) { - Set tNewLine=$c(10) - } ElseIf ($find(pString,$c(10))>($find(pString,$c(13))+1)) { - // Assume that headers do not have $c(13) or $c(10) - Set tNewLine=$c(13) - } - - Quit tNewLine -} - -ClassMethod ReadStream(ByRef pStream, pString As %String = "") As %String -{ - Set tRet=pString_pStream.Read() - Set tQuotes=$length(tRet,"""")-1 - - If tQuotes#2 { - // This means there is an odd number of quotes. - // Continue to read until a single quote is found - Set tDone=0 - While 'tDone { - Set tNext=pStream.Read(1) - Set tRet=tRet_tNext - If tNext="""" { - Set tNext=pStream.Read(1) - Set tRet=tRet_tNext - If tNext'="""" { - // We are done, quotes are balanced - Set tDone=1 - } - } - } - } - - Quit tRet -} - -/// Given a cube name, refresh the data from the data source -ClassMethod RefreshCube(pCubeName As %String, pLineSize As %Integer = 0, pVerbose As %Integer = 0) As %Status -{ - Set st=$$$OK - Set version=$O(^AnalyzeThis.ImportCSV(pCubeName,"")) - If version'="" { - Set sourceType=$O(^AnalyzeThis.ImportCSV(pCubeName,version,"")) - If sourceType="CSV" { - Set fileLocation=^AnalyzeThis.ImportCSV(pCubeName,version,sourceType) - Write:pVerbose "Converting CSV to JSON...",! - Set st=##class(AnalyzeThis.UI.Dialog.CSVImport).CSVToJSON(fileLocation,.tPropertyJSONStreamId,.tDataJSONStreamId,pLineSize) - If $$$ISERR(st) Quit st - Write:pVerbose "CSV successfully converted",!,"Refreshing data from JSON...",! - Set st=##class(AnalyzeThis.Utils).RefreshJSONData(tDataJSONStreamId,pCubeName) - if $$$ISERR(st) Quit st - write:pVerbose "Data successfully refreshed",!,"Building the cube...",! - Set st=$system.DeepSee.BuildCube(pCubeName) - if $$$ISERR(st) Quit st - write:pVerbose "Cube successfully built",! - } - } - Quit st -} - -/// Given a Property Stream ID, Data Stream ID, and a CubeName - populate the source class with data from the stream -ClassMethod RefreshJSONData(dataJSONStreamId As %String, pCubeName As %String) As %Status -{ - Set tSC=$$$OK - - Set tClassName="AnalyzeThis.Generated."_pCubeName - Set tSC=$classmethod(tClassName,"%DeleteExtent") - - // Get property name array - Do ##class(AnalyzeThis.Utils).GetPropertyNames(tClassName,.propNames) - - Set dataJSON=##class(%Stream.FileBinary).%OpenId(dataJSONStreamId) - Set dataArray=[].%FromJSON(dataJSON) - Set iterArray=dataArray.%GetIterator() - While iterArray.%GetNext(.key,.val) { - Set obj=$classmethod(tClassName,"%New") - Set iterObj=dataArray.%Get(key).%GetIterator() - Set propCount=0 - While iterObj.%GetNext(.key2,.val2) { - Set $property(obj,propNames($i(propCount)))=val2 - } - Set tSC= obj.%Save() - } - - Quit tSC -} - -/// Given a class name, get the property names by order of SqlColumnNumber -ClassMethod GetPropertyNames(pClassName As %String, Output pPropNames) As %Status -{ - Set sql="SELECT Name FROM %Dictionary.PropertyDefinition WHERE parent=? ORDER BY SqlColumnNumber" - Set sqlrs=##class(%SQL.Statement).%ExecDirect(,sql,pClassName) - - While sqlrs.%Next() { - Set pPropNames($i(pPropNames))=sqlrs.Name - } - - Quit $$$OK -} - -ClassMethod SQLToJSON(pSourceQuery As %String, ByRef propertyJSONStreamId, ByRef dataJSONStreamId, pMaxLines As %Integer = 0) As %Status -{ - Set tSC=$$$OK - - // Build Prop JSON - Set sqlrs=##class(%SQL.Statement).%ExecDirect(,pSourceQuery) - If sqlrs.%SQLCODE'=0 { - Quit $$$ERROR(5001,"SQLToJSON Error. SQLCODE: "_$SYSTEM.SQL.SQLCODE(sqlrs.%SQLCODE)) - } - Set metadata=sqlrs.%GetMetadata() - Set properties={} - Set propposition={} - Set displayarray=[] - Set typearray=[] - Set dateformatarray=[] - Set includearray=[] - Set intformatarray=[] - // ignorearray is an array that says if a column has an empty header - Set ignorearray=[] - Set propcount=-1 - For i=1:1:metadata.columnCount { - Set propcount=$i(propcount) - Set prop=metadata.columns.GetAt(i).colName - Do displayarray.%Push(prop) - Do typearray.%Push("%String") - Do dateformatarray.%Push("") - Do intformatarray.%Push("") - Do includearray.%Push("true") - Do ignorearray.%Push(prop="") - Do propposition.%Set(prop,propcount) - } - Do properties.%Set("Properties",propposition) - Do properties.%Set("Display",displayarray) - Do properties.%Set("Type",typearray) - Do properties.%Set("DateFormat",dateformatarray) - Do properties.%Set("IntFormat",intformatarray) - Do properties.%Set("Include",includearray) - Do properties.%Set("Ignore",ignorearray) - - Set propertyJSONStream=##class(%Stream.FileBinary).%New() - Do properties.%ToJSON(.propertyJSONStream) - Do propertyJSONStream.%Save() - Set propertyJSONStreamId=propertyJSONStream.%Id() - - // Build Data JSON - Set dynamicArray=[] - While sqlrs.%Next() { - Set dynamicObject={} - For i=0:1:propcount { - Do dynamicObject.%Set(properties.Display.%Get(i),sqlrs.%GetData(i+1)) - } - Do dynamicArray.%Push(dynamicObject) - - Set rowcount=$i(rowcount) - If (pMaxLines>0)&&(rowcount>pMaxLines) { - Quit - } - } - - Set dataJSONStream=##class(%Stream.FileBinary).%New() - Do dynamicArray.%ToJSON(.dataJSONStream) - Do dataJSONStream.%Save() - Set dataJSONStreamId=dataJSONStream.%Id() - - Quit tSC -} - -ClassMethod ClassToQuery(pClass) As %String -{ - // Check for .cls extension - Set tCls=1 - If $zconvert($extract(pClass,*-3,*),"U")'=".CLS" { - Set tCls=0 - } - Set sqlTableName=##class(%DeepSee.Utils).%GetSQLTableName($piece(pClass,".",1,*-tCls)) - Set sqlQuery="SELECT * FROM "_sqlTableName - Quit sqlQuery -} -} \ No newline at end of file +Class AnalyzeThis.Utils +{ + +ClassMethod GetVersion() As %String +{ + Quit "v1.1.3" +} + +ClassMethod GetNextRegex(ByRef pMatch, ByRef pResult, pIncludeTerm As %Boolean = 1) As %Boolean +{ + Set pResult="" + + If '$IsObject(pMatch) { + Quit 0 + } + + Set tStart=$Case(pMatch.End,-2:1,:pMatch.End) + Set tBool=pMatch.Locate() + Quit:tBool=0 tBool + Set tEnd=pMatch.End + + Set pResult=$e(pMatch.Text,tStart,$select(pIncludeTerm=1:pMatch.End,1:pMatch.Start)-1) + + Quit tBool +} + +ClassMethod GetRegexLine(pNewLine As %String) As %String +{ + Quit $replace(pNewLine,"\","\\")_"(?=(?:(?:\\.|[^""\\])*""(?:\\.|[^""\\])*"")*(?:\\.|[^""\\])*\Z)" +} + +ClassMethod GetRegexProp(pNewLine As %String) As %String +{ + Quit "("_$replace(pNewLine,"\","\\")_"|,)"_"(?=(?:(?:\\.|[^""\\])*""(?:\\.|[^""\\])*"")*(?:\\.|[^""\\])*\Z)" +} + +ClassMethod DetermineNewLine(pString As %String) As %String +{ + Set tNewLine=$c(13)_$c(10) + + If ($find(pString,$c(10))=0)&&($find(pString,$c(13))>0) { + Set tNewLine=$c(13) + } ElseIf ($find(pString,$c(10))>0)&&($find(pString,$c(13))=0) { + Set tNewLine=$c(10) + } ElseIf ($find(pString,$c(10))>($find(pString,$c(13))+1)) { + // Assume that headers do not have $c(13) or $c(10) + Set tNewLine=$c(13) + } + + Quit tNewLine +} + +ClassMethod ReadStream(ByRef pStream, pString As %String = "") As %String +{ + Set tRet=pString_pStream.Read() + Set tQuotes=$length(tRet,"""")-1 + + If tQuotes#2 { + // This means there is an odd number of quotes. + // Continue to read until a single quote is found + Set tDone=0 + While 'tDone { + Set tNext=pStream.Read(1) + Set tRet=tRet_tNext + If tNext="""" { + Set tNext=pStream.Read(1) + Set tRet=tRet_tNext + If tNext'="""" { + // We are done, quotes are balanced + Set tDone=1 + } + } + } + } + + Quit tRet +} + +/// Given a cube name, refresh the data from the data source +ClassMethod RefreshCube(pCubeName As %String, pLineSize As %Integer = 0, pVerbose As %Integer = 0) As %Status +{ + Set st=$$$OK + Set version=$O(^AnalyzeThis.ImportCSV(pCubeName,"")) + If version'="" { + Set sourceType=$O(^AnalyzeThis.ImportCSV(pCubeName,version,"")) + If sourceType="CSV" { + Set fileLocation=^AnalyzeThis.ImportCSV(pCubeName,version,sourceType) + Write:pVerbose "Converting CSV to JSON...",! + Set st=##class(AnalyzeThis.UI.Dialog.CSVImport).CSVToJSON(fileLocation,.tPropertyJSONStreamId,.tDataJSONStreamId,pLineSize) + If $$$ISERR(st) Quit st + Write:pVerbose "CSV successfully converted",!,"Refreshing data from JSON...",! + Set st=##class(AnalyzeThis.Utils).RefreshJSONData(tDataJSONStreamId,pCubeName) + if $$$ISERR(st) Quit st + write:pVerbose "Data successfully refreshed",!,"Building the cube...",! + Set st=$system.DeepSee.BuildCube(pCubeName) + if $$$ISERR(st) Quit st + write:pVerbose "Cube successfully built",! + } + } + Quit st +} + +/// Given a Property Stream ID, Data Stream ID, and a CubeName - populate the source class with data from the stream +ClassMethod RefreshJSONData(dataJSONStreamId As %String, pCubeName As %String) As %Status +{ + Set tSC=$$$OK + + Set tClassName="AnalyzeThis.Generated."_pCubeName + Set tSC=$classmethod(tClassName,"%DeleteExtent") + + // Get property name array + Do ##class(AnalyzeThis.Utils).GetPropertyNames(tClassName,.propNames) + + Set dataJSON=##class(%Stream.FileBinary).%OpenId(dataJSONStreamId) + Set dataArray=[].%FromJSON(dataJSON) + Set iterArray=dataArray.%GetIterator() + While iterArray.%GetNext(.key,.val) { + Set obj=$classmethod(tClassName,"%New") + Set iterObj=dataArray.%Get(key).%GetIterator() + Set propCount=0 + While iterObj.%GetNext(.key2,.val2) { + Set $property(obj,propNames($i(propCount)))=val2 + } + Set tSC= obj.%Save() + } + + Quit tSC +} + +/// Given a class name, get the property names by order of SqlColumnNumber +ClassMethod GetPropertyNames(pClassName As %String, Output pPropNames) As %Status +{ + Set sql="SELECT Name FROM %Dictionary.PropertyDefinition WHERE parent=? ORDER BY SqlColumnNumber" + Set sqlrs=##class(%SQL.Statement).%ExecDirect(,sql,pClassName) + + While sqlrs.%Next() { + Set pPropNames($i(pPropNames))=sqlrs.Name + } + + Quit $$$OK +} + +ClassMethod SQLToJSON(pSourceQuery As %String, ByRef propertyJSONStreamId, ByRef dataJSONStreamId, pMaxLines As %Integer = 0) As %Status +{ + Set tSC=$$$OK + + // Build Prop JSON + Set sqlrs=##class(%SQL.Statement).%ExecDirect(,pSourceQuery) + If sqlrs.%SQLCODE'=0 { + Quit $$$ERROR(5001,"SQLToJSON Error. SQLCODE: "_$SYSTEM.SQL.SQLCODE(sqlrs.%SQLCODE)) + } + Set metadata=sqlrs.%GetMetadata() + Set properties={} + Set propposition={} + Set displayarray=[] + Set typearray=[] + Set dateformatarray=[] + Set includearray=[] + Set intformatarray=[] + // ignorearray is an array that says if a column has an empty header + Set ignorearray=[] + Set propcount=-1 + For i=1:1:metadata.columnCount { + Set propcount=$i(propcount) + Set prop=metadata.columns.GetAt(i).colName + Do displayarray.%Push(prop) + Do typearray.%Push("%String") + Do dateformatarray.%Push("") + Do intformatarray.%Push("") + Do includearray.%Push("true") + Do ignorearray.%Push(prop="") + Do propposition.%Set(prop,propcount) + } + Do properties.%Set("Properties",propposition) + Do properties.%Set("Display",displayarray) + Do properties.%Set("Type",typearray) + Do properties.%Set("DateFormat",dateformatarray) + Do properties.%Set("IntFormat",intformatarray) + Do properties.%Set("Include",includearray) + Do properties.%Set("Ignore",ignorearray) + + Set propertyJSONStream=##class(%Stream.FileBinary).%New() + Do properties.%ToJSON(.propertyJSONStream) + Do propertyJSONStream.%Save() + Set propertyJSONStreamId=propertyJSONStream.%Id() + + // Build Data JSON + Set dynamicArray=[] + While sqlrs.%Next() { + Set dynamicObject={} + For i=0:1:propcount { + Do dynamicObject.%Set(properties.Display.%Get(i),sqlrs.%GetData(i+1)) + } + Do dynamicArray.%Push(dynamicObject) + + Set rowcount=$i(rowcount) + If (pMaxLines>0)&&(rowcount>pMaxLines) { + Quit + } + } + + Set dataJSONStream=##class(%Stream.FileBinary).%New() + Do dynamicArray.%ToJSON(.dataJSONStream) + Do dataJSONStream.%Save() + Set dataJSONStreamId=dataJSONStream.%Id() + + Quit tSC +} + +ClassMethod ClassToQuery(pClass) As %String +{ + // Check for .cls extension + Set tCls=1 + If $zconvert($extract(pClass,*-3,*),"U")'=".CLS" { + Set tCls=0 + } + Set sqlTableName=##class(%DeepSee.Utils).%GetSQLTableName($piece(pClass,".",1,*-tCls)) + Set sqlQuery="SELECT * FROM "_sqlTableName + Quit sqlQuery +} + +} From 2ef4c1056bf0d4072bf262050f695dfefbcaf8ac Mon Sep 17 00:00:00 2001 From: psteiwer Date: Mon, 30 Mar 2020 15:28:20 -0400 Subject: [PATCH 07/11] Copy RefreshData method into generated class --- AnalyzeThis/Generator.cls | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/AnalyzeThis/Generator.cls b/AnalyzeThis/Generator.cls index 0c78aeb..4ae5420 100644 --- a/AnalyzeThis/Generator.cls +++ b/AnalyzeThis/Generator.cls @@ -182,6 +182,10 @@ ClassMethod JSONToClass(propertiesJSON As %String, pClassName As %String) As %St Do class.Properties.Insert(prop) //} } + + // Add Refresh Data method directoy to class + Set tSC=##class(AnalyzeThis.Generator).GenerateRefreshMethod(.class) + Quit:$$$ISERR(tSC) tSC // Set global name directly to allow for global mapping of AnalyzeThis.* Set param=##class(%Dictionary.ParameterDefinition).%New() @@ -196,6 +200,20 @@ ClassMethod JSONToClass(propertiesJSON As %String, pClassName As %String) As %St Quit tSC } +ClassMethod GenerateRefreshMethod(pClass As %Dictionary.ClassDefinition) As %Status +{ + Set tSC=$$$OK + + Set tMethod=##class(%Dictionary.MethodDefinition).%OpenId("AnalyzeThis.Utils||RefreshCube") + Set tMethod=tMethod.%ConstructClone() + Set tMethod.Name="ReloadData" + Set tMethod.FormalSpec=$Replace(tMethod.FormalSpec,"pCubeName:%String","pCubeName:%String="""_$Piece(pClass.Name,".",*)_"""") + + Set tSC=pClass.Methods.Insert(tMethod) + + Quit tSC +} + /// Given a Property Stream ID, Data Stream ID, and a CubeName - populate the source class with data from the stream ClassMethod ImportJSONData(propertyJSONStreamId As %String, dataJSONStreamId As %String, pCubeName As %String, pCleanFirst As %Boolean = 1) As %Status { From 7d7fa903238c48a271217ad5b18905fd946d4535 Mon Sep 17 00:00:00 2001 From: psteiwer Date: Mon, 30 Mar 2020 15:36:03 -0400 Subject: [PATCH 08/11] Add option to refresh from new file --- AnalyzeThis/Utils.cls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/AnalyzeThis/Utils.cls b/AnalyzeThis/Utils.cls index 521e682..d57835a 100644 --- a/AnalyzeThis/Utils.cls +++ b/AnalyzeThis/Utils.cls @@ -77,16 +77,16 @@ ClassMethod ReadStream(ByRef pStream, pString As %String = "") As %String } /// Given a cube name, refresh the data from the data source -ClassMethod RefreshCube(pCubeName As %String, pLineSize As %Integer = 0, pVerbose As %Integer = 0) As %Status +ClassMethod RefreshCube(pCubeName As %String, pLineSize As %Integer = 0, pVerbose As %Integer = 0, pFileName As %String = "") As %Status { Set st=$$$OK Set version=$O(^AnalyzeThis.ImportCSV(pCubeName,"")) If version'="" { Set sourceType=$O(^AnalyzeThis.ImportCSV(pCubeName,version,"")) If sourceType="CSV" { - Set fileLocation=^AnalyzeThis.ImportCSV(pCubeName,version,sourceType) + Set:pFileName="" pFileName=^AnalyzeThis.ImportCSV(pCubeName,version,sourceType) Write:pVerbose "Converting CSV to JSON...",! - Set st=##class(AnalyzeThis.UI.Dialog.CSVImport).CSVToJSON(fileLocation,.tPropertyJSONStreamId,.tDataJSONStreamId,pLineSize) + Set st=##class(AnalyzeThis.UI.Dialog.CSVImport).CSVToJSON(pFileName,.tPropertyJSONStreamId,.tDataJSONStreamId,pLineSize) If $$$ISERR(st) Quit st Write:pVerbose "CSV successfully converted",!,"Refreshing data from JSON...",! Set st=##class(AnalyzeThis.Utils).RefreshJSONData(tDataJSONStreamId,pCubeName) From 4a7b4dd481e9c45bbdd37fa0c13ad2629a78dfb2 Mon Sep 17 00:00:00 2001 From: psteiwer Date: Fri, 3 Apr 2020 12:39:40 -0400 Subject: [PATCH 09/11] Accept URL for file source Fixes #45 --- AnalyzeThis/UI/Dialog/CSVImport.cls | 13 +++++++++++-- AnalyzeThis/Utils.cls | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/AnalyzeThis/UI/Dialog/CSVImport.cls b/AnalyzeThis/UI/Dialog/CSVImport.cls index 97fbd26..1bedf58 100644 --- a/AnalyzeThis/UI/Dialog/CSVImport.cls +++ b/AnalyzeThis/UI/Dialog/CSVImport.cls @@ -1149,6 +1149,10 @@ ClassMethod GetStatus(tab) As %ZEN.proxyObject [ Internal, ZenMethod ] ClassMethod ValidateFile(FILE) As %Integer [ Internal, ZenMethod ] { If $ZStrip(FILE,"<>W")="" Quit 0 + + // Skip validation if it begins with HTTP + Quit:$$$UPPER($E(FILE,1,4))="HTTP" 1 + #; Entered directory only! If ##class(%File).DirectoryExists(FILE) Quit -1 #; File does not exist!" @@ -1268,8 +1272,13 @@ ClassMethod CSVToJSON(pFileName As %String = "C:\Users\psteiwer\Documents\simple { Set st=$$$OK - Set stream=##class(%Stream.FileBinary).%New() - Do stream.LinkToFile(pFileName) + If $$$UPPER($e(pFileName,1,4))="HTTP" { + Set st=##class(AnalyzeThis.Utils).URLToStream(pFileName,.stream) + } Else { + Set stream=##class(%Stream.FileBinary).%New() + Do stream.LinkToFile(pFileName) + } + Set s=##class(AnalyzeThis.Utils).ReadStream(.stream) Set pNewLine=##class(AnalyzeThis.Utils).DetermineNewLine(s) Set s=$replace(s,$c(13),"\r") diff --git a/AnalyzeThis/Utils.cls b/AnalyzeThis/Utils.cls index d57835a..891bb7d 100644 --- a/AnalyzeThis/Utils.cls +++ b/AnalyzeThis/Utils.cls @@ -219,4 +219,23 @@ ClassMethod ClassToQuery(pClass) As %String Quit sqlQuery } +ClassMethod URLToStream(pURL, ByRef pStream) As %Status +{ + Set tSC=$$$OK + + Set pStream=##class(%Stream.FileCharacter).%New() + + Set tRequest=##class(%Net.HttpRequest).%New() + Set tRequest.Https=1 + Set tRequest.Port=443 + Set tRequest.SSLConfiguration="AnalyzeThis" + Set tSC=tRequest.Get(pURL) + Quit:$$$ISERR(tSC) tSC + + Set tSC=pStream.Write(tRequest.HttpResponse.Data.Read(tRequest.HttpResponse.Data.SizeGet())) + Quit:$$$ISERR(tSC) tSC + + Quit tSC +} + } From 8a7ba4bac1fc65fd1c927355e4634ff42aebb83a Mon Sep 17 00:00:00 2001 From: psteiwer Date: Fri, 3 Apr 2020 12:46:18 -0400 Subject: [PATCH 10/11] Dynamically create SSL configuration if needed --- AnalyzeThis/Utils.cls | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/AnalyzeThis/Utils.cls b/AnalyzeThis/Utils.cls index 891bb7d..9c694e0 100644 --- a/AnalyzeThis/Utils.cls +++ b/AnalyzeThis/Utils.cls @@ -228,7 +228,7 @@ ClassMethod URLToStream(pURL, ByRef pStream) As %Status Set tRequest=##class(%Net.HttpRequest).%New() Set tRequest.Https=1 Set tRequest.Port=443 - Set tRequest.SSLConfiguration="AnalyzeThis" + Set tRequest.SSLConfiguration=..GetSSLConfiguration("AnalyzeThis") Set tSC=tRequest.Get(pURL) Quit:$$$ISERR(tSC) tSC @@ -238,4 +238,16 @@ ClassMethod URLToStream(pURL, ByRef pStream) As %Status Quit tSC } +ClassMethod GetSSLConfiguration(pConfig) As %String +{ + New $NAMESPACE + Set $NAMESPACE="%SYS" + + If '##class(Security.SSLConfigs).Exists(pConfig) { + Do ##class(Security.SSLConfigs).Create(pConfig) + } + + Quit pConfig +} + } From 88427d5db4b21d77661c4f6a644e6453ef61e519 Mon Sep 17 00:00:00 2001 From: psteiwer Date: Fri, 3 Apr 2020 12:50:11 -0400 Subject: [PATCH 11/11] Update version string --- AnalyzeThis/Utils.cls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AnalyzeThis/Utils.cls b/AnalyzeThis/Utils.cls index 9c694e0..a05b8a0 100644 --- a/AnalyzeThis/Utils.cls +++ b/AnalyzeThis/Utils.cls @@ -3,7 +3,7 @@ Class AnalyzeThis.Utils ClassMethod GetVersion() As %String { - Quit "v1.1.3" + Quit "v1.2.0" } ClassMethod GetNextRegex(ByRef pMatch, ByRef pResult, pIncludeTerm As %Boolean = 1) As %Boolean