diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java index 93954afd4..8bbf22996 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java @@ -27,10 +27,7 @@ import java.sql.SQLException; import java.sql.Types; import java.util.*; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; +import java.util.concurrent.*; import javax.annotation.Nonnull; import com.gemstone.gemfire.DataSerializer; @@ -41,6 +38,7 @@ import com.gemstone.gemfire.cache.TransactionException; import com.gemstone.gemfire.cache.control.RebalanceOperation; import com.gemstone.gemfire.cache.control.ResourceManager; +import com.gemstone.gemfire.cache.execute.FunctionException; import com.gemstone.gemfire.cache.execute.FunctionService; import com.gemstone.gemfire.distributed.DistributedMember; import com.gemstone.gemfire.distributed.internal.ServerLocation; @@ -70,6 +68,7 @@ import com.pivotal.gemfirexd.internal.engine.ddl.catalog.messages.GfxdSystemProcedureMessage; import com.pivotal.gemfirexd.internal.engine.ddl.resolver.GfxdPartitionByExpressionResolver; import com.pivotal.gemfirexd.internal.engine.ddl.wan.messages.AbstractGfxdReplayableMessage; +import com.pivotal.gemfirexd.internal.engine.diag.SnappyTableStatsVTI; import com.pivotal.gemfirexd.internal.engine.distributed.AckResultCollector; import com.pivotal.gemfirexd.internal.engine.distributed.ByteArrayDataOutput; import com.pivotal.gemfirexd.internal.engine.distributed.GfxdDistributionAdvisor; @@ -86,6 +85,7 @@ import com.pivotal.gemfirexd.internal.engine.store.CustomRowsResultSet; import com.pivotal.gemfirexd.internal.engine.store.GemFireContainer; import com.pivotal.gemfirexd.internal.engine.store.GemFireStore; +import com.pivotal.gemfirexd.internal.engine.ui.SnappyRegionStats; import com.pivotal.gemfirexd.internal.iapi.db.PropertyInfo; import com.pivotal.gemfirexd.internal.iapi.error.PublicAPI; import com.pivotal.gemfirexd.internal.iapi.error.StandardException; @@ -121,8 +121,12 @@ import com.pivotal.gemfirexd.internal.shared.common.sanity.SanityManager; import com.pivotal.gemfirexd.internal.snappy.LeadNodeSmartConnectorOpContext; import com.pivotal.gemfirexd.load.Import; +import com.pivotal.gemfirexd.internal.engine.ui.SnappyRegionStatsCollectorFunction; +import com.pivotal.gemfirexd.internal.engine.ui.SnappyRegionStatsCollectorResult; +//import com.pivotal.gemfirexd.internal.engine.distributed; import io.snappydata.thrift.ServerType; import io.snappydata.thrift.internal.ClientBlob; +import org.apache.commons.collections.bag.SynchronizedSortedBag; /** * GemFireXD built-in system procedures that will get executed on every @@ -690,7 +694,7 @@ public boolean getNext(DataValueDescriptor[] template) private static final ResultColumnDescriptor[] encryptColumnInfo = { EmbedResultSetMetaData.getResultColumnDescriptor("ENCRYPTED_PASSWORD", Types.VARCHAR, - false, Limits.DB2_VARCHAR_MAXWIDTH), + false, Limits.DB2_VARCHAR_MAXWIDTH) }; @@ -3242,4 +3246,114 @@ private static void rollBackAndThrowSQLException(Connection conn, } throw se; } -} + + /** + * given a row/column table, method returns size of table with samplePercentage=100%, + * if given an external table and sample percentage, method creates a column table with + * sample percentage amount and extrapolates memory for loading whole(100% data) + * external table as column table. + * + * Cases(Exceptions) not handled: + * Case 1: given a row/column table and isExternalTable=TRUE then it doesn't + * show an error instead it creates another column table with sample amount. + * + * Case 2: given an external table with isExternalTable=FALSE or + * given any table which is not present then it gives error but need to show + * an error message as table not found. + * + * @param tableName table name with schema(fully qualified table name) + * @param isExternalTable true for external table else false + * @param samplePercentage sampling percentage (Integer :0 to 100) + * @param tableSize returns resultset + * @throws SQLException + */ + public static void GET_TABLE_SIZE(String tableName, Boolean isExternalTable, int samplePercentage, ResultSet[] tableSize) throws SQLException { + if (GemFireXDUtils.TraceSysProcedures) { + SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_SYS_PROCEDURES, + "executing GET_TABLE_SIZE "); + } + List result = new java.util.ArrayList<>(); + Set dataServers = GfxdMessage.getAllDataStores(); + try { + + if (dataServers != null && dataServers.size() > 0) { + Map args = new HashMap(); + if (isExternalTable) { + float sample = (float) samplePercentage / 100; + String sampleTable = tableName + "_sample"; + Connection conn = getDefaultConn(); + conn.createStatement().execute("drop table if exists " + sampleTable + " ;"); + conn.createStatement().execute("create table " + sampleTable + " using column as(select * from " + tableName + " where rand() < " + sample + " );"); + args.put("TABLE_NAME", sampleTable); + } else { + args.put("TABLE_NAME", tableName); + samplePercentage = 100; + } + result = (ArrayList) FunctionService.onMembers(dataServers) + .withArgs(args) + .execute(SnappyRegionStatsCollectorFunction.ID).getResult(1, TimeUnit.SECONDS); + + List stats = ((SnappyRegionStatsCollectorResult) result.get(0)).getRegionStats(); + SnappyRegionStats statsResult = stats.get(0); + long sampleTableSize = statsResult.getTotalSize(); + long sampleInMemoryTableSize = statsResult.getSizeInMemory(); + + final long totalTableSize = (sampleTableSize * 100) / samplePercentage; + final long inMemoryTableSize = (sampleInMemoryTableSize * 100) / samplePercentage; + + Boolean resultFetched = true; + final CustomRowsResultSet.FetchDVDRows fetchRows = + new CustomRowsResultSet.FetchDVDRows() { + Boolean resultFetched = false; + + @Override + public boolean getNext(DataValueDescriptor[] template) + throws SQLException, StandardException { + if (!resultFetched) { + if (totalTableSize > 104857.6) { + float totalTableSizeInMB = (float) totalTableSize / 1048576; + template[0].setValue(tableName + " = " + totalTableSizeInMB + " MB"); + } else if (totalTableSize > 102.4) { + float totalTableSizeInKB = (float) totalTableSize / 1024; + template[0].setValue(tableName + " = " + totalTableSizeInKB + " KB"); + } else { + template[0].setValue(tableName + " = " + totalTableSize + " Bytes"); + } + + if (inMemoryTableSize > 104857.6) { + float inMemoryTableSizeInMB = (float) inMemoryTableSize / 1048576; + template[1].setValue(tableName + " = " + inMemoryTableSizeInMB + " MB"); + } else if (inMemoryTableSize > 102.4) { + float inMemoryTableSizeInKB = (float) inMemoryTableSize / 1024; + template[1].setValue(tableName + " = " + inMemoryTableSizeInKB + " KB"); + } else { + template[1].setValue(tableName + " = " + inMemoryTableSize + " Bytes"); + } + resultFetched = true; + return true; + } + return false; + } + }; + if (!result.isEmpty()) { + tableSize[0] = new CustomRowsResultSet(fetchRows, tableSizeInfo); + } else { + tableSize[0] = null; + } + } + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (StandardException se) { + se.printStackTrace(); + } + } + + private static final ResultColumnDescriptor[] tableSizeInfo = { + EmbedResultSetMetaData.getResultColumnDescriptor("TOTAL TABLE SIZE", Types.VARCHAR, + false, Limits.DB2_VARCHAR_MAXWIDTH), + EmbedResultSetMetaData.getResultColumnDescriptor("IN MEMORY TABLE SIZE", Types.VARCHAR, + false, Limits.DB2_VARCHAR_MAXWIDTH) + }; + + +} \ No newline at end of file diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java index d7351369d..c25ea8bfb 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java @@ -31,6 +31,9 @@ import com.gemstone.gemfire.management.internal.SystemManagementService; import com.pivotal.gemfirexd.internal.engine.Misc; import com.pivotal.gemfirexd.internal.engine.access.index.GfxdIndexManager; +import com.pivotal.gemfirexd.internal.engine.ddl.DDLConflatable; +import com.pivotal.gemfirexd.internal.engine.ddl.callbacks.CallbackProcedures; +import com.pivotal.gemfirexd.internal.engine.distributed.utils.GemFireXDUtils; import com.pivotal.gemfirexd.internal.engine.store.GemFireContainer; import com.pivotal.gemfirexd.internal.iapi.error.StandardException; import com.pivotal.gemfirexd.tools.sizer.GemFireXDInstrumentation; @@ -66,55 +69,74 @@ public void execute(FunctionContext context) { com.pivotal.gemfirexd.internal.snappy.CallbackFactoryProvider.getClusterCallbacks(). publishColumnTableStats(); SnappyRegionStatsCollectorResult result = new SnappyRegionStatsCollectorResult(); - Map cachBatchStats = new HashMap<>(); + Map cachedBatchStats = new HashMap<>(); ArrayList otherStats = new ArrayList<>(); + try { + final SystemManagementService managementService = + (SystemManagementService) ManagementService.getManagementService(Misc.getGemFireCache()); + if (context.getArguments() != null && context.getArguments() instanceof HashMap + && ((HashMap) context.getArguments()).containsKey("TABLE_NAME") + && !((HashMap) context.getArguments()).get("TABLE_NAME").isEmpty()) { + String tname = ((HashMap) context.getArguments()).get("TABLE_NAME"); + String[] names = tname.split("\\."); + if (names.length == 2) { - try { - List containers = Misc.getMemStore().getAllContainers(); + GemFireContainer container = CallbackProcedures.getContainerForTable(names[0], names[1]); + getSnappyRegionStats(otherStats, cachedBatchStats, container, managementService); + StoreCallbacks callback = CallbackFactoryProvider.getStoreCallbacks(); + String columnBatchTableName = callback.columnBatchTableName(otherStats.get(0).getTableName()); + try { + container = CallbackProcedures.getContainerForTable(names[0], + columnBatchTableName.substring(columnBatchTableName.indexOf(".") + 1)); + getSnappyRegionStats(otherStats, cachedBatchStats, container, managementService); + } catch (StandardException se) { + } - final SystemManagementService managementService = - (SystemManagementService)ManagementService.getManagementService(Misc.getGemFireCache()); - - for (GemFireContainer container : containers) { - if (container.isApplicationTable()) { - LocalRegion r = container.getRegion(); - if (managementService != null && r != null ) { - RegionMXBean bean = managementService.getLocalRegionMBean(r.getFullPath()); - if (bean != null && !(r.getFullPath().startsWith( - "/" + Misc.SNAPPY_HIVE_METASTORE + '/'))) { - SnappyRegionStats dataCollector = collectDataFromBean(r, bean); - if (dataCollector.isColumnTable()) { - cachBatchStats.put(dataCollector.getTableName(), dataCollector); - } else { - otherStats.add(dataCollector); + } // TODO else get default or current schemaname and proceed + } else { + List containers = Misc.getMemStore().getAllContainers(); + + for (GemFireContainer container : containers) { + //TODO replace block with method getSnappyRegionStats() + if (container.isApplicationTable()) { + LocalRegion r = container.getRegion(); + if (managementService != null && r != null) { + RegionMXBean bean = managementService.getLocalRegionMBean(r.getFullPath()); + if (bean != null && !(r.getFullPath().startsWith( + "/" + Misc.SNAPPY_HIVE_METASTORE + '/'))) { + SnappyRegionStats dataCollector = collectDataFromBean(r, bean); + if (dataCollector.isColumnTable()) { + cachedBatchStats.put(dataCollector.getTableName(), dataCollector); + } else { + otherStats.add(dataCollector); + } } } - } /* if(!LocalRegion.isMetaTable(r.getFullPath())){ result.addAllIndexStat(getIndexStatForContainer(container)); } */ + } } } - if (Misc.reservoirRegionCreated) { for (SnappyRegionStats tableStats : otherStats) { String tableName = tableStats.getTableName(); StoreCallbacks callback = CallbackFactoryProvider.getStoreCallbacks(); String columnBatchTableName = callback.columnBatchTableName(tableName); - if (cachBatchStats.containsKey(columnBatchTableName)) { + if (cachedBatchStats.containsKey(columnBatchTableName)) { String reservoirRegionName = Misc.getReservoirRegionNameForSampleTable("APP", tableName); PartitionedRegion pr = Misc.getReservoirRegionForSampleTable(reservoirRegionName); if (managementService != null && pr != null) { RegionMXBean reservoirBean = managementService.getLocalRegionMBean(pr.getFullPath()); if (reservoirBean != null) { SnappyRegionStats rStats = collectDataFromBeanImpl(pr, reservoirBean, true); - SnappyRegionStats cStats = cachBatchStats.get(columnBatchTableName); - cachBatchStats.put(columnBatchTableName, cStats.getCombinedStats(rStats)); + SnappyRegionStats cStats = cachedBatchStats.get(columnBatchTableName); + cachedBatchStats.put(columnBatchTableName, cStats.getCombinedStats(rStats)); } } } @@ -122,22 +144,45 @@ public void execute(FunctionContext context) { } // Create one entry per Column Table by combining the results of row buffer and column table + for (SnappyRegionStats tableStats : otherStats) { StoreCallbacks callback = CallbackFactoryProvider.getStoreCallbacks(); String columnBatchTableName = callback.columnBatchTableName(tableStats.getTableName()); - if (cachBatchStats.containsKey(columnBatchTableName)) { - result.addRegionStat(tableStats.getCombinedStats(cachBatchStats.get(columnBatchTableName))); + if (cachedBatchStats.containsKey(columnBatchTableName)) { + result.addRegionStat(tableStats.getCombinedStats(cachedBatchStats.get(columnBatchTableName))); } else { result.addRegionStat(tableStats); } } } catch (CacheClosedException ignored) { + } catch (StandardException ignored) { } finally { context.getResultSender().lastResult(result); } } + private void getSnappyRegionStats(ArrayList otherStats, + Map cacheBatchStats, + GemFireContainer container, + SystemManagementService managementService) { + if (container.isApplicationTable()) { + LocalRegion r = container.getRegion(); + if (managementService != null && r != null) { + RegionMXBean bean = managementService.getLocalRegionMBean(r.getFullPath()); + if (bean != null && !(r.getFullPath().startsWith( + "/" + Misc.SNAPPY_HIVE_METASTORE + '/'))) { + SnappyRegionStats dataCollector = collectDataFromBean(r, bean); + if (dataCollector.isColumnTable()) { + cacheBatchStats.put(dataCollector.getTableName(), dataCollector); + } else { + otherStats.add(dataCollector); + } + } + } + } + } + private SnappyRegionStats collectDataFromBean(LocalRegion lr, RegionMXBean bean) { return collectDataFromBeanImpl(lr, bean, false); } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java index ab4479373..81b95083e 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java @@ -2092,6 +2092,19 @@ private void createGfxdSystemProcedures(TransactionController tc, arg_names, arg_types, 0, 1, RoutineAliasInfo.READS_SQL_DATA, null, newlyCreatedRoutines, tc, GFXD_SYS_PROC_CLASSNAME, false); } + { + // GET_TABLE_SIZE + String[] arg_names = new String[]{"TABLE_NAME","IS_EXTERNAL_TABLE","SAMPLE_PERCENTAGE"}; + TypeDescriptor[] arg_types = new TypeDescriptor[]{ + DataTypeDescriptor.getCatalogType(Types.VARCHAR), + DataTypeDescriptor.getCatalogType(Types.BOOLEAN), + DataTypeDescriptor.getCatalogType(Types.INTEGER) + }; + super.createSystemProcedureOrFunction("GET_TABLE_SIZE", sysUUID, + arg_names, arg_types, 0, 1, RoutineAliasInfo.NO_SQL, null, + newlyCreatedRoutines, tc, GFXD_SYS_PROC_CLASSNAME, false); + } + } @SuppressWarnings({ "unchecked", "rawtypes" })